dee/core/compiler_validation.py

"""Check the compiler against landmark editing designs.

The question anyone serious asks within two minutes is "how do you know it's
right?", and "it is deterministic and well tested" is an engineering answer to
a scientific question. This is the scientific answer: run the compiler against
designs the field has already settled and show it arrives at the same verdict.

THE CLAIM THIS SUPPORTS, STATED PRECISELY
-----------------------------------------
It reproduces **known-correct decisions**. It does NOT discover new biology,
and the two must never be blurred — the first is defensible and the second
would be a lie a reviewer could take apart in one question.

TWO KINDS OF CASE, COUNTED SEPARATELY
-------------------------------------
`basis="chemistry"`   the expected verdict follows from the genetic code and
                      the two base-editor chemistries. Anyone can re-derive
                      it, so it needs no citation and counts as validation.

`basis="literature"`  the expectation is a claim about what a real programme
                      actually did. It requires a citation. Without one the
                      case is reported UNVERIFIED and does **not** count as a
                      pass — it is a named gap, which is more useful than a
                      silent absence.

That split is the honesty mechanism. "12 of 12 validated" means nothing if
half were asserted from memory; here the report always carries both numbers
and the unverified ones by name.

Pure logic: no network, no GPU, no model. It runs in CI on every commit, so a
change that breaks a landmark verdict fails the build rather than being
noticed in a demo.
"""

from __future__ import annotations

import json
import pathlib
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional

from dee.core import compiler as _compiler

__all__ = ["CaseResult", "ValidationReport", "load_cases", "run_validation",
           "report_to_dict"]

CORPUS_PATH = (pathlib.Path(__file__).resolve().parent.parent
               / "data" / "landmark_designs.json")


@dataclass
class CaseResult:
    case_id: str
    name: str
    basis: str
    status: str            # "pass" | "fail" | "unverified"
    expected: Dict[str, Any] = field(default_factory=dict)
    observed: Dict[str, Any] = field(default_factory=dict)
    mismatches: List[str] = field(default_factory=list)
    why: str = ""
    citation: str = ""


@dataclass
class ValidationReport:
    results: List[CaseResult]

    @property
    def chemistry_total(self) -> int:
        return sum(1 for r in self.results if r.basis == "chemistry")

    @property
    def chemistry_passed(self) -> int:
        return sum(1 for r in self.results
                   if r.basis == "chemistry" and r.status == "pass")

    @property
    def unverified(self) -> List[str]:
        return [r.name for r in self.results if r.status == "unverified"]

    @property
    def failures(self) -> List[CaseResult]:
        return [r for r in self.results if r.status == "fail"]

    @property
    def headline(self) -> str:
        """One sentence that cannot be misread as more than it is."""
        n, tot = self.chemistry_passed, self.chemistry_total
        base = (f"{n} of {tot} landmark decisions reproduced "
                "(derivable from the genetic code and base-editor chemistry)")
        if self.unverified:
            base += (f"; {len(self.unverified)} further case(s) awaiting a "
                     "citation and not counted")
        return base + "."


def load_cases(path: Optional[pathlib.Path] = None) -> List[Dict[str, Any]]:
    p = path or CORPUS_PATH
    if not p.exists():
        return []
    data = json.loads(p.read_text())
    return [c for c in (data.get("cases") or []) if isinstance(c, dict)]


def _expand_allele(value: Any) -> str:
    """Alleles may be a literal string or {"repeat": "A", "times": 400}.

    The compact form exists because a 400-base deletion is a legitimate case
    and pasting 400 characters into JSON makes the corpus unreadable. It is
    NOT a shorthand the compiler understands — an earlier corpus wrote "A400"
    expecting it to mean 400 A's, and the compiler correctly read it as an
    allele containing a digit and refused. The harness caught that, which is
    the harness working.
    """
    if isinstance(value, dict):
        base = str(value.get("repeat") or "")
        times = int(value.get("times") or 0)
        return base * times
    return str(value or "")


def _observe(case: Dict[str, Any]) -> Dict[str, Any]:
    """Run the compiler on a case and flatten what it decided."""
    call = _compiler.classify_lesion(_expand_allele(case.get("wt_base")),
                                     _expand_allele(case.get("patient_base")))
    corr = call.correction
    return {
        "compiles": call.compiles,
        "route": call.route,
        "editor_family": corr.editor_family if corr else None,
        "strand": corr.strand if corr else None,
        "diagnostics": [d.code for d in call.diagnostics],
    }


def _compare(expected: Dict[str, Any], observed: Dict[str, Any]) -> List[str]:
    """Only the keys the case actually asserts. A case that says nothing about
    strand is not failed for the strand."""
    out: List[str] = []
    for key in ("compiles", "route", "editor_family", "strand"):
        if key in expected and observed.get(key) != expected[key]:
            out.append(f"{key}: expected {expected[key]!r}, "
                       f"got {observed.get(key)!r}")
    want_diag = expected.get("diagnostic")
    if want_diag and want_diag not in (observed.get("diagnostics") or []):
        out.append(f"diagnostic {want_diag!r} not raised "
                   f"(raised {observed.get('diagnostics')!r})")
    return out


def run_validation(cases: Optional[List[Dict[str, Any]]] = None
                   ) -> ValidationReport:
    """Every case, with literature cases short-circuited unless cited."""
    results: List[CaseResult] = []
    for case in (cases if cases is not None else load_cases()):
        basis = str(case.get("basis") or "literature")
        why = case.get("why")
        why_text = " ".join(why) if isinstance(why, list) else str(why or "")
        citation = str(case.get("citation") or "").strip()
        expected = case.get("expect") or {}

        # A literature claim without a citation is not evidence, and must not
        # be run as though it were — an uncited expectation is just a guess
        # with a filename.
        if basis == "literature" and not citation:
            results.append(CaseResult(
                case_id=str(case.get("id") or ""), name=str(case.get("name") or ""),
                basis=basis, status="unverified", expected=expected,
                why=why_text, citation=citation))
            continue

        if not expected:
            results.append(CaseResult(
                case_id=str(case.get("id") or ""), name=str(case.get("name") or ""),
                basis=basis, status="unverified", expected={},
                why=why_text or "No expectation recorded.", citation=citation))
            continue

        observed = _observe(case)
        mismatches = _compare(expected, observed)
        results.append(CaseResult(
            case_id=str(case.get("id") or ""), name=str(case.get("name") or ""),
            basis=basis, status="pass" if not mismatches else "fail",
            expected=expected, observed=observed, mismatches=mismatches,
            why=why_text, citation=citation))
    return ValidationReport(results)


def report_to_dict(report: ValidationReport) -> Dict[str, Any]:
    return {
        "headline": report.headline,
        "chemistry_passed": report.chemistry_passed,
        "chemistry_total": report.chemistry_total,
        "unverified": report.unverified,
        "all_passed": not report.failures,
        "cases": [
            {"id": r.case_id, "name": r.name, "basis": r.basis,
             "status": r.status, "expected": r.expected, "observed": r.observed,
             "mismatches": r.mismatches, "why": r.why, "citation": r.citation}
            for r in report.results
        ],
    }