File size: 6,785 Bytes
ecc81b3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 | """Mutation testing: break the code on purpose, check the suite notices.
python scripts/mutate.py # run the catalog
python scripts/mutate.py --list # show it without running anything
DEBUG.md §B ranks this the highest-yield technique in the project, and it has
been a *manual* discipline the whole time — which means it runs when somebody
remembers. This makes it a machine's job.
Each entry names a real invariant, a one-line edit that violates it, and the
tests that must fail as a result. A mutation that survives is a hole in the
suite: the code can be wrong in that specific way and nothing says so. That is
the finding, and it is reported as a failure of *the tests*, not of the code.
Every mutation here corresponds to a bug that was either found in this project
or deliberately designed against — see the `why` field. Restoring the file is
guaranteed by a finally-block; the script also refuses to start on a dirty
working tree, because a crash mid-mutation must never be confusable with your
own uncommitted work.
"""
from __future__ import annotations
import argparse
import subprocess
import sys
from dataclasses import dataclass
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
SRC = ROOT / "src" / "torch_dimensions"
@dataclass
class Mutation:
name: str
path: Path
old: str
new: str
tests: str
"""The `-k` expression, or a test path, that must fail once mutated."""
why: str
def describe(self) -> str:
return f"{self.name:34s} {self.path.relative_to(ROOT)} -> {self.tests}"
CATALOG = [
Mutation(
name="absolute cancellation epsilon",
path=SRC / "compose" / "kernel.py",
old="den = torch.where(den.abs() <= _REL * den_abs, torch.ones_like(den), den)",
new="den = torch.where(den.abs() <= 1e-12, torch.ones_like(den), den)",
tests="tests/test_kernel.py -k cancel or explode",
why="DEBUG.md #12: an absolute epsilon has no idea what scale the kernel works at; "
"float32 blew up 7,000x under one.",
),
Mutation(
name="mask only on entry, not per layer",
path=SRC / "compose" / "kernel.py",
old=" if valid is not None:\n x = x * valid",
new=" if valid is not None and False:\n x = x * valid",
tests="tests/test_kernel.py -k absent or influence",
why="Zeroing once at entry is sufficient only at rank 1; a contraction leaves absent "
"cells holding legitimate scratch that the next axis then counts.",
),
Mutation(
name="direction flips per layer, not per cycle",
path=SRC / "plan.py",
old="Step(axes[i % n], axes[i % n] in bidi and (i // n) % 2 == 1) for i in range(n_layers)",
new="Step(axes[i % n], axes[i % n] in bidi and i % 2 == 1) for i in range(n_layers)",
tests="tests/test_plan.py -k alias or direction or cycle",
why="DEBUG.md #4: with an even axis count the two periods phase-lock and every axis is "
"silently pinned one way. This bug exists in published research code.",
),
Mutation(
name="lattice mask returns a view",
path=SRC / "lattice.py",
old="return base.reshape(*lead, *self.shape, 1).to(dtype, copy=True)",
new="return base.reshape(*lead, *self.shape, 1).to(dtype)",
tests="tests/test_lattice.py -k view or copy or alias",
why="DEBUG.md #13: a caller writing into 'their' mask reached into the lattice.",
),
Mutation(
name="flat_idx indexes on the wrong device",
path=SRC / "lattice.py",
old="out[..., self.flat_idx.to(x.device), :] = x",
new="out[..., self.flat_idx, :] = x",
tests="tests/test_device.py -k device_mismatch or gather_scatter",
why="DEBUG.md #18: torch tolerates CPU indices into a device tensor but not the "
"reverse, so one direction raised and the other did not.",
),
Mutation(
name="conformance skips count as passes",
path=SRC / "testing.py",
old='rep.results.append(Result(name, "skip", str(s)))',
new='rep.results.append(Result(name, "pass", str(s)))',
tests="tests/test_conformance.py -k skip",
why="A skipped check that reads as a pass is how a whole family goes unverified while "
"the report looks green.",
),
Mutation(
name="plan/n_layers disagreement goes silent",
path=SRC / "models" / "base.py",
old="if n_layers != 1 and n_layers != len(plan):",
new="if False:",
tests="tests/test_scan.py tests/test_conformance.py -k n_layers or plan_wins",
why="DEBUG.md #10: a model quietly shallower than requested.",
),
]
def run(mutation: Mutation, verbose: bool) -> bool:
"""Apply, run the named tests, restore. True when the mutation was caught."""
source = mutation.path.read_text()
if mutation.old not in source:
print(" STALE: the code no longer contains the mutated line — update the catalog")
return False
try:
mutation.path.write_text(source.replace(mutation.old, mutation.new, 1))
cmd = [sys.executable, "-m", "pytest", "-x", "-q", *mutation.tests.split()]
result = subprocess.run(cmd, cwd=ROOT, capture_output=not verbose, text=True)
finally:
mutation.path.write_text(source)
return result.returncode != 0
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--list", action="store_true")
ap.add_argument("--verbose", action="store_true")
ap.add_argument("--only", help="substring of a mutation name")
args = ap.parse_args()
if args.list:
for m in CATALOG:
print(m.describe())
return 0
dirty = subprocess.run(
["git", "status", "--porcelain", "src"], cwd=ROOT, capture_output=True, text=True
).stdout.strip()
if dirty:
print("refusing to run with uncommitted changes under src/:\n" + dirty)
print("a crash mid-mutation must not be confusable with your own edits")
return 2
chosen = [m for m in CATALOG if not args.only or args.only in m.name]
survived = []
for m in chosen:
print(f"\n== {m.name}\n {m.why}")
caught = run(m, args.verbose)
print(f" {'caught' if caught else 'SURVIVED — the suite does not notice this'}")
if not caught:
survived.append(m)
print(f"\n{len(chosen) - len(survived)}/{len(chosen)} mutations caught")
if survived:
print("\nsurviving mutations — each is a hole in the test suite, not in the code:")
for m in survived:
print(f" - {m.name}: {m.why}")
return 1
return 0
if __name__ == "__main__":
sys.exit(main())
|