mutation_check.py

"""Mutation matrix over the lint rules.

For each rule, neuter it so it yields nothing, run the suite, and record which
tests fail. A row with zero failures means the rule is either dead code or has
no test guarding it, and you cannot tell which without looking.

Run with: python mutation_check.py
"""

from __future__ import annotations

import os
import re
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path

HERE = Path(__file__).resolve().parent
RULE_DEF = re.compile(r"^def (rule_[a-z_]+)\(", re.MULTILINE)


def rule_names(source: str) -> list[str]:
    return RULE_DEF.findall(source)


def mutate(source: str, rule: str) -> str:
    """Insert an immediate return into one rule, leaving it a generator."""
    pattern = re.compile(rf"^(def {re.escape(rule)}\(.*:\n)", re.MULTILINE)
    mutated, count = pattern.subn(r"\1    return  # MUTATED\n", source)
    if count != 1:
        raise SystemExit(f"anchor for {rule} matched {count} times, expected 1")
    if mutated == source:
        raise SystemExit(f"mutation for {rule} was a no-op")
    return mutated


def run_suite(workdir: Path) -> tuple[int, str]:
    # Every mutated lint.py is the same length, and CPython validates a cached
    # .pyc on (mtime, size). Inside one second that pair does not change, so the
    # interpreter would silently re-import the previous mutation's bytecode and
    # the whole matrix would report another rule's result.
    env = {**os.environ, "PYTHONDONTWRITEBYTECODE": "1"}
    proc = subprocess.run(
        [sys.executable, "-B", "-m", "pytest", "-q", "--no-header", "-p", "no:cacheprovider"],
        cwd=workdir, capture_output=True, text=True, env=env,
    )
    return proc.returncode, proc.stdout + proc.stderr


def failed_tests(output: str) -> list[str]:
    return sorted({m.group(1) for m in re.finditer(r"^FAILED (\S+)", output, re.MULTILINE)})


def main() -> int:
    source = (HERE / "lint.py").read_text()
    rules = rule_names(source)
    if not rules:
        raise SystemExit("no rules found - the anchor is wrong, not the code")

    with tempfile.TemporaryDirectory() as tmp:
        base = Path(tmp)
        shutil.copy(HERE / "test_lint.py", base / "test_lint.py")

        # Baseline must be green, or every verdict below is meaningless.
        (base / "lint.py").write_text(source)
        rc, out = run_suite(base)
        if rc != 0:
            print("BASELINE IS NOT GREEN - fix that before trusting any row\n")
            print(out[-2000:])
            return 1
        baseline_count = re.search(r"(\d+) passed", out)
        print(f"baseline: {baseline_count.group(1) if baseline_count else '?'} passed\n")

        zero_rows: list[str] = []
        print(f"{'rule':<38} {'rc':>3}  result")
        print("-" * 78)
        for rule in rules:
            (base / "lint.py").write_text(mutate(source, rule))
            rc, out = run_suite(base)
            fails = failed_tests(out)
            if not fails and rc != 0:
                verdict = "CRASHED (covered)"
            elif not fails:
                verdict = "*** ZERO - dead code or unguarded ***"
                zero_rows.append(rule)
            else:
                verdict = f"{len(fails)} failed: " + ", ".join(f.split("::")[-1] for f in fails)
            print(f"{rule:<38} {rc:>3}  {verdict}")

    print()
    if zero_rows:
        print(f"{len(zero_rows)} rule(s) with no failing test: {', '.join(zero_rows)}")
        return 1
    print("every rule is guarded by at least one test")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())