"""Check the compiler against landmark editing designs. The question anyone serious asks within two minutes is "how do you know it's right?", and "it is deterministic and well tested" is an engineering answer to a scientific question. This is the scientific answer: run the compiler against designs the field has already settled and show it arrives at the same verdict. THE CLAIM THIS SUPPORTS, STATED PRECISELY ----------------------------------------- It reproduces **known-correct decisions**. It does NOT discover new biology, and the two must never be blurred — the first is defensible and the second would be a lie a reviewer could take apart in one question. TWO KINDS OF CASE, COUNTED SEPARATELY ------------------------------------- `basis="chemistry"` the expected verdict follows from the genetic code and the two base-editor chemistries. Anyone can re-derive it, so it needs no citation and counts as validation. `basis="literature"` the expectation is a claim about what a real programme actually did. It requires a citation. Without one the case is reported UNVERIFIED and does **not** count as a pass — it is a named gap, which is more useful than a silent absence. That split is the honesty mechanism. "12 of 12 validated" means nothing if half were asserted from memory; here the report always carries both numbers and the unverified ones by name. Pure logic: no network, no GPU, no model. It runs in CI on every commit, so a change that breaks a landmark verdict fails the build rather than being noticed in a demo. """ from __future__ import annotations import json import pathlib from dataclasses import dataclass, field from typing import Any, Dict, List, Optional from dee.core import compiler as _compiler __all__ = ["CaseResult", "ValidationReport", "load_cases", "run_validation", "report_to_dict"] CORPUS_PATH = (pathlib.Path(__file__).resolve().parent.parent / "data" / "landmark_designs.json") @dataclass class CaseResult: case_id: str name: str basis: str status: str # "pass" | "fail" | "unverified" expected: Dict[str, Any] = field(default_factory=dict) observed: Dict[str, Any] = field(default_factory=dict) mismatches: List[str] = field(default_factory=list) why: str = "" citation: str = "" @dataclass class ValidationReport: results: List[CaseResult] @property def chemistry_total(self) -> int: return sum(1 for r in self.results if r.basis == "chemistry") @property def chemistry_passed(self) -> int: return sum(1 for r in self.results if r.basis == "chemistry" and r.status == "pass") @property def unverified(self) -> List[str]: return [r.name for r in self.results if r.status == "unverified"] @property def failures(self) -> List[CaseResult]: return [r for r in self.results if r.status == "fail"] @property def headline(self) -> str: """One sentence that cannot be misread as more than it is.""" n, tot = self.chemistry_passed, self.chemistry_total base = (f"{n} of {tot} landmark decisions reproduced " "(derivable from the genetic code and base-editor chemistry)") if self.unverified: base += (f"; {len(self.unverified)} further case(s) awaiting a " "citation and not counted") return base + "." def load_cases(path: Optional[pathlib.Path] = None) -> List[Dict[str, Any]]: p = path or CORPUS_PATH if not p.exists(): return [] data = json.loads(p.read_text()) return [c for c in (data.get("cases") or []) if isinstance(c, dict)] def _expand_allele(value: Any) -> str: """Alleles may be a literal string or {"repeat": "A", "times": 400}. The compact form exists because a 400-base deletion is a legitimate case and pasting 400 characters into JSON makes the corpus unreadable. It is NOT a shorthand the compiler understands — an earlier corpus wrote "A400" expecting it to mean 400 A's, and the compiler correctly read it as an allele containing a digit and refused. The harness caught that, which is the harness working. """ if isinstance(value, dict): base = str(value.get("repeat") or "") times = int(value.get("times") or 0) return base * times return str(value or "") def _observe(case: Dict[str, Any]) -> Dict[str, Any]: """Run the compiler on a case and flatten what it decided.""" call = _compiler.classify_lesion(_expand_allele(case.get("wt_base")), _expand_allele(case.get("patient_base"))) corr = call.correction return { "compiles": call.compiles, "route": call.route, "editor_family": corr.editor_family if corr else None, "strand": corr.strand if corr else None, "diagnostics": [d.code for d in call.diagnostics], } def _compare(expected: Dict[str, Any], observed: Dict[str, Any]) -> List[str]: """Only the keys the case actually asserts. A case that says nothing about strand is not failed for the strand.""" out: List[str] = [] for key in ("compiles", "route", "editor_family", "strand"): if key in expected and observed.get(key) != expected[key]: out.append(f"{key}: expected {expected[key]!r}, " f"got {observed.get(key)!r}") want_diag = expected.get("diagnostic") if want_diag and want_diag not in (observed.get("diagnostics") or []): out.append(f"diagnostic {want_diag!r} not raised " f"(raised {observed.get('diagnostics')!r})") return out def run_validation(cases: Optional[List[Dict[str, Any]]] = None ) -> ValidationReport: """Every case, with literature cases short-circuited unless cited.""" results: List[CaseResult] = [] for case in (cases if cases is not None else load_cases()): basis = str(case.get("basis") or "literature") why = case.get("why") why_text = " ".join(why) if isinstance(why, list) else str(why or "") citation = str(case.get("citation") or "").strip() expected = case.get("expect") or {} # A literature claim without a citation is not evidence, and must not # be run as though it were — an uncited expectation is just a guess # with a filename. if basis == "literature" and not citation: results.append(CaseResult( case_id=str(case.get("id") or ""), name=str(case.get("name") or ""), basis=basis, status="unverified", expected=expected, why=why_text, citation=citation)) continue if not expected: results.append(CaseResult( case_id=str(case.get("id") or ""), name=str(case.get("name") or ""), basis=basis, status="unverified", expected={}, why=why_text or "No expectation recorded.", citation=citation)) continue observed = _observe(case) mismatches = _compare(expected, observed) results.append(CaseResult( case_id=str(case.get("id") or ""), name=str(case.get("name") or ""), basis=basis, status="pass" if not mismatches else "fail", expected=expected, observed=observed, mismatches=mismatches, why=why_text, citation=citation)) return ValidationReport(results) def report_to_dict(report: ValidationReport) -> Dict[str, Any]: return { "headline": report.headline, "chemistry_passed": report.chemistry_passed, "chemistry_total": report.chemistry_total, "unverified": report.unverified, "all_passed": not report.failures, "cases": [ {"id": r.case_id, "name": r.name, "basis": r.basis, "status": r.status, "expected": r.expected, "observed": r.observed, "mismatches": r.mismatches, "why": r.why, "citation": r.citation} for r in report.results ], }