release_tools/diff_solved.py

#!/usr/bin/env python3
"""diff_solved.py — solved-set regression diff between two flagship versions.

Catches the failure mode where a refactor / new orchestrator cap silently drops
problems a previous version solved (e.g. a stage starved of budget). Run it on
every new flagship version before it becomes the submission.

Usage:
    python3 diff_solved.py <old.json> <new.json>

Each input is either:
    * a coverage file with a "solved_ids": [...] list, or
    * an eq2 manifest with "results": [{"id":..., "solved":true}, ...]

Reports per-tier gained / lost, the net delta, and EXITS NONZERO if anything
regressed (any id solved by old but not new) so it can gate a release.
"""
from __future__ import annotations

import json
import sys
from pathlib import Path

TIERS = ("normal", "hard1", "hard2", "hard3")


def solved_ids(path: Path) -> set:
    d = json.loads(path.read_text())
    if isinstance(d, dict) and "solved_ids" in d:
        return set(d["solved_ids"])
    if isinstance(d, dict) and "results" in d:
        return {r["id"] for r in d["results"] if r.get("solved")}
    sys.exit(f"FATAL: {path} has neither 'solved_ids' nor 'results'.")


def tier(pid: str) -> str:
    return pid.split("_")[0]


def by_tier(ids: set) -> dict:
    out = {t: 0 for t in TIERS}
    for i in ids:
        out[tier(i)] = out.get(tier(i), 0) + 1
    return out


def main() -> None:
    if len(sys.argv) != 3:
        sys.exit(__doc__)
    old_p, new_p = Path(sys.argv[1]), Path(sys.argv[2])
    old, new = solved_ids(old_p), solved_ids(new_p)
    gained = new - old
    lost = old - new

    print(f"OLD {old_p.name}: {len(old)} solved   NEW {new_p.name}: {len(new)} solved   "
          f"net {len(new) - len(old):+d}\n")
    print(f"{'tier':<8}{'old':>6}{'new':>6}{'gained':>8}{'lost':>6}")
    ot, nt, gt, lt = by_tier(old), by_tier(new), by_tier(gained), by_tier(lost)
    for t in TIERS:
        print(f"{t:<8}{ot[t]:>6}{nt[t]:>6}{gt[t]:>8}{lt[t]:>6}")
    print(f"{'TOTAL':<8}{len(old):>6}{len(new):>6}{len(gained):>8}{len(lost):>6}\n")

    if gained:
        print(f"GAINED ({len(gained)}): {sorted(gained)[:40]}{' ...' if len(gained) > 40 else ''}")
    if lost:
        print(f"\n*** REGRESSION — {len(lost)} problem(s) LOST: {sorted(lost)} ***")
        print("NEW version solves fewer than OLD in these ids. Investigate before releasing.")
        sys.exit(1)
    print("\nNo regressions: NEW is a superset of OLD. Safe.")
    sys.exit(0)


if __name__ == "__main__":
    main()