"""Mutation matrix over the lint rules. For each rule, neuter it so it yields nothing, run the suite, and record which tests fail. A row with zero failures means the rule is either dead code or has no test guarding it, and you cannot tell which without looking. Run with: python mutation_check.py """ from __future__ import annotations import os import re import shutil import subprocess import sys import tempfile from pathlib import Path HERE = Path(__file__).resolve().parent RULE_DEF = re.compile(r"^def (rule_[a-z_]+)\(", re.MULTILINE) def rule_names(source: str) -> list[str]: return RULE_DEF.findall(source) def mutate(source: str, rule: str) -> str: """Insert an immediate return into one rule, leaving it a generator.""" pattern = re.compile(rf"^(def {re.escape(rule)}\(.*:\n)", re.MULTILINE) mutated, count = pattern.subn(r"\1 return # MUTATED\n", source) if count != 1: raise SystemExit(f"anchor for {rule} matched {count} times, expected 1") if mutated == source: raise SystemExit(f"mutation for {rule} was a no-op") return mutated def run_suite(workdir: Path) -> tuple[int, str]: # Every mutated lint.py is the same length, and CPython validates a cached # .pyc on (mtime, size). Inside one second that pair does not change, so the # interpreter would silently re-import the previous mutation's bytecode and # the whole matrix would report another rule's result. env = {**os.environ, "PYTHONDONTWRITEBYTECODE": "1"} proc = subprocess.run( [sys.executable, "-B", "-m", "pytest", "-q", "--no-header", "-p", "no:cacheprovider"], cwd=workdir, capture_output=True, text=True, env=env, ) return proc.returncode, proc.stdout + proc.stderr def failed_tests(output: str) -> list[str]: return sorted({m.group(1) for m in re.finditer(r"^FAILED (\S+)", output, re.MULTILINE)}) def main() -> int: source = (HERE / "lint.py").read_text() rules = rule_names(source) if not rules: raise SystemExit("no rules found - the anchor is wrong, not the code") with tempfile.TemporaryDirectory() as tmp: base = Path(tmp) shutil.copy(HERE / "test_lint.py", base / "test_lint.py") # Baseline must be green, or every verdict below is meaningless. (base / "lint.py").write_text(source) rc, out = run_suite(base) if rc != 0: print("BASELINE IS NOT GREEN - fix that before trusting any row\n") print(out[-2000:]) return 1 baseline_count = re.search(r"(\d+) passed", out) print(f"baseline: {baseline_count.group(1) if baseline_count else '?'} passed\n") zero_rows: list[str] = [] print(f"{'rule':<38} {'rc':>3} result") print("-" * 78) for rule in rules: (base / "lint.py").write_text(mutate(source, rule)) rc, out = run_suite(base) fails = failed_tests(out) if not fails and rc != 0: verdict = "CRASHED (covered)" elif not fails: verdict = "*** ZERO - dead code or unguarded ***" zero_rows.append(rule) else: verdict = f"{len(fails)} failed: " + ", ".join(f.split("::")[-1] for f in fails) print(f"{rule:<38} {rc:>3} {verdict}") print() if zero_rows: print(f"{len(zero_rows)} rule(s) with no failing test: {', '.join(zero_rows)}") return 1 print("every rule is guarded by at least one test") return 0 if __name__ == "__main__": raise SystemExit(main())