scripts/validate_mutations.py

"""Sanity-check the synthetic mutator by running it across many seeds
and confirming each produced ground-truth change can be re-detected
by the DiffSpecialist."""

import random
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))

from scenarios.layer2_synthetic.mutator import synth_mutate
from scenarios.library import REAL_SCENARIOS
from server.specialists.diff_specialist import DiffSpecialist


def main():
    differ = DiffSpecialist()
    base = REAL_SCENARIOS["stripe_v18_invoice_field_rename"]()
    n = 200
    matched = 0
    total = 0
    for s in range(n):
        rng = random.Random(s)
        v2, gt = synth_mutate(base.v1_spec, rng, "medium")
        if not gt:
            continue
        detected = differ.diff(base.v1_spec, v2)
        gt_locations = {(g["location"], g["change_type"]) for g in gt}
        det_locations = {(d.location, d.change_type) for d in detected}
        matched += len(gt_locations & det_locations)
        total += len(gt_locations)
    rate = matched / total if total else 0.0
    print(f"Mutator-vs-DiffSpecialist agreement: {matched}/{total} ({rate:.2%})")


if __name__ == "__main__":
    main()