recovery/tests/test_checked_reports.py

import json
from pathlib import Path

REPORTS = Path(__file__).parents[1] / "reports"


def load(name: str) -> dict:
    return json.loads((REPORTS / name).read_text())


def model(report: dict, split: str, name: str) -> dict:
    return next(item for item in report["splits"][split]["models"] if item["name"] == name)


def test_checked_sample_and_manifests():
    audit = load("dev-10k-audit.json")
    assert audit["dataset_revision"] == "11e49b7ece33d62afd7f65bc05ce60ad37f9ba7b"
    assert (
        audit["source_sha256"] == "d50f4cae794298f32c39f75a9e8603528d20e50e4fa757025def412f5b2a78d8"
    )
    assert audit["sample_rows"] == 10_000
    assert audit["external_split_audits"]["protein_similarity_50"]["protein_cluster_overlap"] == 0


def test_combined_baseline_beats_train_mean_on_every_checked_split():
    report = load("dev-10k-baselines.json")
    assert report["target"] == "pAffinity"
    for split_name, split in report["splits"].items():
        combined = model(report, split_name, "combined_char_ridge")["test"]
        mean = split["mean_baseline"]["test"]
        assert combined["rmse"] < mean["rmse"]
        assert combined["pearson_r"] > 0
        assert combined["spearman_r"] > 0


def test_similarity_split_passes_initial_recovery_gate():
    report = load("dev-10k-baselines.json")
    result = model(report, "protein_similarity_50", "combined_char_ridge")["test"]
    assert result["pearson_r"] > 0.30
    assert result["spearman_r"] > 0.30


def test_v2_model_evaluation_is_honest_and_consistent():
    report = load("bindingdb-v2-model-evaluation.json")
    assert report["unique_pairs"] == 2198
    assert report["observations"] == 2327
    assert report["v1_forensic_reference"]["pearson_r"] == -0.9107
    # Every held-out split except the degenerate cold_both has a large positive
    # Pearson correlation, categorically better than the failed v1 artifact.
    for name, split in report["splits"].items():
        if name != "cold_both":
            assert split["frozen_head"]["pearson_r"] > 0
            assert split["vs_v1_pearson_positive"] is True
    # The head does not beat the best control everywhere, so the checked
    # conclusion must remain "do not publish a checkpoint yet".
    wins = report["summary"]["splits_where_head_beats_best_control"]
    assert wins == sum(s["head_beats_best_control_rmse"] for s in report["splits"].values())
    assert wins < report["summary"]["total_splits"]
    # The hardest generalization test must stay non-degenerate-flagged.
    assert report["splits"]["cold_both"]["frozen_head"]["r2"] < 0


def test_balm_auxiliary_gate_rejects_negative_transfer():
    report = load("bindingdb-v2-balm-auxiliary-gate.json")
    assert report["primary"]["unique_pairs"] == 2198
    assert report["auxiliary"]["input_pairs"] == 20_477
    assert report["auxiliary"]["excluded_pair_overlap"] == 739
    assert report["auxiliary"]["base_eligible_pairs"] == 19_081
    original = report["variants"]["bindingdb_only"]["best_control"]["rmse"]
    for name, variant in report["variants"].items():
        if name != "bindingdb_only":
            assert variant["best_control"]["rmse"] > original
    assert report["gate"]["any_auxiliary_variant_beats_original"] is False
    assert report["gate"]["decision"].startswith("reject BALM")


def test_joint_finetune_gate_rejects_checkpoint():
    report = load("bindingdb-v2-joint-finetune-cold-protein.json")
    assert report["decision"] == "reject_checkpoint_no_publish"
    assert report["training"]["gpu"] == "NVIDIA L4"
    assert report["training"]["test_evaluations"] == 1
    assert report["training"]["selected_epoch"] == 3
    assert report["data"]["sample_sha256"] == (
        "4b022ee8e3f1a4d00499bcaea218bad730f0181fedd7dda7d6a5a3ea370bb7ea"
    )
    assert report["data"]["manifest_sha256"] == (
        "fb0a4dc63b8074018a1b7f02b863b8fa0cf6117930c3298b7c80133867cb2798"
    )
    joint_rmse = report["metrics"]["test"]["rmse"]
    assert joint_rmse < report["comparisons"]["frozen_head"]["rmse"]
    assert joint_rmse > report["comparisons"]["best_control"]["rmse"]
    assert report["gate"] == {
        "required_test_rmse_below": 1.1702173443155717,
        "actual_test_rmse": 1.3472529698141609,
        "passed": False,
        "run_additional_paid_splits": False,
        "publish_weights": False,
    }
    assert report["artifacts"]["checkpoint_in_git"] is False
    assert report["artifacts"]["checkpoint_published"] is False
    assert report["artifacts"]["checkpoint_validation"]["all_finite"] is True


def test_chimera_development_gate_rejects_uncertain_residual():
    report = load("bindingdb-v2-chimera-development-gate.json")
    assert report["status"] == "development_only_no_test_evaluation"
    assert report["decision"] == "reject"
    assert report["test_evaluations"] == 0
    assert report["population"] == {
        "sample_rows": 2327,
        "development_rows": 2057,
        "test_rows_excluded": 270,
        "development_pairs": 1978,
        "development_proteins": 399,
        "development_clusters": 210,
        "source_partitions": ["train", "validation"],
        "excluded_partitions": ["test"],
    }
    assert report["best_variant"] == "protein_ridge_residual"
    control_rmse = report["control"]["metrics"]["rmse"]
    variant = report["variants"]["protein_ridge_residual"]
    assert variant["metrics"]["rmse"] < control_rmse
    assert variant["rmse_improvement"] > 0.05
    assert variant["bootstrap"]["ci_2_5"] < 0
    assert variant["bootstrap"]["probability_improvement"] > 0.95
    assert variant["gate"]["bootstrap_lower_bound_positive"] is False
    assert variant["gate"]["passed"] is False
    assert report["passed_variants"] == []
    assert min(variant["fold_rmse_improvements"]) >= -0.10
    assert report["artifacts"]["predictions_sha256"] == (
        "e064bd81224be5c05a01df6ee7baa79295ec012303644e1dcdc13832755041ab"
    )


def test_source_atlas_contract_is_exact_and_homology_filtered():
    report = load("bindingdb-v2-source-atlas-contract.json")
    assert report["source"]["records"] == 87315
    eligibility = report["eligibility"]
    assert eligibility["accepted_total"] == 75331
    assert eligibility["accepted_by_task"] == {
        "EC50": 3731,
        "IC50": 47829,
        "Kd": 2432,
        "Ki": 21339,
    }
    assert (
        eligibility["accepted_total"] + sum(eligibility["rejections"].values())
        == report["source"]["records"]
    )
    assert report["entities"]["source_record_ids"] == eligibility["accepted_total"]
    clustering = report["joint_protein_clustering"]
    assert clustering["identity"] == 0.5
    assert clustering["coverage"] == 0.8
    assert clustering["proteins"] == report["entities"]["proteins"]
    reserved = report["reserved_test_homology_filter"]
    assert reserved["reserved_test_clusters"] == 34
    assert reserved["development_rows_excluded_for_test_cluster_homology"] == 292
    assert reserved["safe_development_rows"] == 1765
    assert sum(reserved["safe_atlas_by_task"].values()) == reserved["safe_atlas_records"]


def test_source_atlas_transfer_gate_rejects_negative_transfer():
    report = load("bindingdb-v2-source-atlas-transfer-gate.json")
    assert report["status"] == "development_only_no_test_evaluation"
    assert report["decision"] == "reject_neural_pretraining_not_justified"
    assert report["test_evaluations"] == 0
    assert report["reserved_test_clusters"] == 34
    assert report["development_rows"] == 1765
    assert report["safe_atlas_records"] == 71473
    kd_rmse = report["kd_only"]["metrics"]["rmse"]
    multitask_rmse = report["multitask"]["metrics"]["rmse"]
    assert multitask_rmse > kd_rmse
    assert report["gate"]["actual_rmse_improvement"] < 0
    assert report["bootstrap"]["ci_97_5"] < 0
    assert report["bootstrap"]["probability_improvement"] == 0
    assert report["gate"]["passed"] is False
    assert all(value < 0 for value in report["fold_rmse_improvements"])
    assert report["artifacts"]["predictions_sha256"] == (
        "7e54551f654dc14f443d06cc0bff4b787f2b7e7fa37188850e79f0d45e54bf4a"
    )


def test_gtopdb_external_holdout_is_independent_cold_both_and_sealed():
    report = load("gtopdb-2026.2-external-holdout.json")
    assert report["decision"] == "freeze_external_cold_both_holdout"
    assert report["source"]["release"] == "2026.2"
    audit = report["candidate_audit"]
    assert audit["interaction_rows"] == 24_599
    assert audit["strict_rows"] == 261
    assert audit["bindingdb_publication_overlap_rows"] == 0
    assert audit["bindingdb_pair_overlap_rows"] == 3
    assert audit["independent_rows"] == 258
    assert report["uniprot_snapshot"]["returned_reviewed_entries"] == 161
    clustering = report["joint_clustering"]
    assert clustering["minimum_sequence_identity"] == 0.5
    assert clustering["minimum_bidirectional_coverage"] == 0.8
    cold_both = report["novelty_categories"]["cold_both"]
    assert cold_both == {
        "observations": 103,
        "pairs": 96,
        "proteins": 78,
        "ligands": 79,
        "scaffolds": 78,
        "pubmed_ids": 77,
    }
    freeze = report["freeze_gate"]
    assert freeze["accepted"] is True
    assert freeze["all_minimums_passed"] is True
    assert freeze["labels_isolated"] is True
    assert freeze["model_predictions_generated"] == 0
    assert report["evaluation_policy"]["status"] == "sealed"


def test_pocket_atlas_gate_rejects_structure_without_touching_holdouts():
    report = load("bindingdb-v2-pocket-atlas-development-gate.json")
    assert report["decision"] == "reject_structure_candidate_no_further_training"
    assert report["benchmark_test_evaluations"] == 0
    assert report["external_holdout_evaluations"] == 0
    assert report["population"] == {
        "sample_rows": 2327,
        "source_development_rows": 2057,
        "development_rows_excluded_for_test_cluster_homology": 292,
        "safe_development_rows": 1765,
        "safe_development_pairs": 1717,
        "safe_development_proteins": 278,
        "safe_development_clusters": 187,
        "benchmark_test_rows_excluded": 270,
        "benchmark_test_clusters_reserved": 34,
    }
    assert report["alphafold_snapshot"]["canonical_models"] == 145
    assert report["structure_features"]["successful_structures"] == 145
    assert report["structure_features"]["failed_structures"] == 0
    control = report["control"]["metrics"]["rmse"]
    esm = report["variants"]["protein_embedding_ridge_residual"]["metrics"]["rmse"]
    structure = report["variants"]["structure_ridge_residual"]["metrics"]["rmse"]
    combined = report["variants"]["protein_embedding_plus_structure_ridge_residual"]["metrics"][
        "rmse"
    ]
    assert esm < combined < control < structure
    gate = report["predeclared_structure_gate"]
    assert gate["actual_incremental_rmse_improvement_vs_esm"] < 0
    assert gate["incremental_cluster_bootstrap"]["ci_2_5"] < 0
    assert gate["incremental_cluster_bootstrap"]["ci_97_5"] > 0
    assert gate["no_catastrophic_fold"] is False
    assert gate["passed"] is False
    verification = report["verification"]
    assert verification["test_pairs_in_predictions"] == 0
    assert verification["test_clusters_in_predictions"] == 0
    assert verification["source_reproduction"] == {
        "mapping_exact_match": True,
        "structure_manifest_exact_match": True,
    }
    assert report["artifacts"]["checkpoint_created"] is False
    assert report["artifacts"]["weights_published"] is False


def test_primary_kd_audit_and_similarity_split():
    audit = load("balm-kd-audit.json")
    clusters = load("balm-kd-protein-clusters.json")
    assert audit["target"]["name"] == "pKd"
    assert audit["target"]["aggregation"].startswith("median")
    assert audit["source_rows"] == 24_700
    assert audit["valid_canonical_pairs"] == 20_477
    assert audit["source_sha256"] == (
        "d0f8691a96876f09766e574c5600df55f0d0b1c11cfd1705faf841d924f89f20"
    )
    assert clusters["protein_cluster_overlap"] == 0
    assert clusters["clusters"] == 739


def test_kd_descriptor_model_beats_all_controls_on_every_split():
    report = load("balm-kd-descriptor-baselines.json")
    assert report["target"] == "pKd"
    for split_name, split in report["splits"].items():
        models = {entry["name"]: entry["test"] for entry in split["models"]}
        combined = models["combined_morgan_protein_lightgbm"]
        assert combined["rmse"] < split["mean_baseline"]["test"]["rmse"], split_name
        assert combined["rmse"] < models["ligand_morgan_descriptors_lightgbm"]["rmse"], split_name
        assert combined["rmse"] < models["protein_aac_dipeptide_lightgbm"]["rmse"], split_name
        assert combined["pearson_r"] > 0
        assert combined["spearman_r"] > 0


def test_actual_encoder_overfit_sanity_passed():
    report = load("overfit-actual-embeddings.json")
    assert report["passed"] is True
    assert report["target"] == "pAffinity"
    assert report["rmse_target"] <= report["acceptance_max_rmse_target"]


def test_bindingdb_source_aware_audit_is_complete():
    audit = load("bindingdb-source-aware-audit.json")
    assert audit["source_database"] == "BindingDB"
    assert audit["source_release"] == "202607"
    assert audit["source_rows"] == 93_712
    assert audit["rows_with_emitted_measurements"] == 83_654
    assert audit["emitted_records"]["total"] == 87_315
    assert audit["emitted_records"]["by_type"]["Kd"] == 2_615
    assert audit["gold_exact_kd"] == {
        "records": 2_432,
        "unique_pairs": 2_198,
        "unique_proteins": 442,
        "unique_assays": 245,
        "pkd_min": 1.1146161610532512,
        "pkd_max": 13.327902142064282,
        "pkd_below_3_review_flag": 47,
        "pkd_above_12_review_flag": 3,
        "review_flags_are_not_filters": True,
    }
    assert (
        sum(audit["rejections"]["row_level"].values()) + audit["rows_with_emitted_measurements"]
        == audit["source_rows"]
    )
    assert audit["duplicates"]["duplicate_reactant_set_ids"] == 122
    assert audit["duplicates"]["extra_reactant_set_rows"] == 190
    assert audit["outputs"]["source_records.jsonl"]["sha256"] == (
        "1e9963217b9ed65528ca477adcba66beea1fa341a1c1c6d41cd789332ab85bbb"
    )
    assert audit["outputs"]["gold_exact_kd.jsonl"]["sha256"] == (
        "432507f29163acf37c7c74e62c3bba47b1dff02a38e34945c1b06cac14c7282e"
    )