File size: 4,145 Bytes
718badd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
"""Accession-recovery control over an actually downloaded complete reference.

Inputs are canonical sequences extracted from that same frozen reference.
This measures engine/routing/accession recovery, not novel-protein accuracy.
"""
from __future__ import annotations
import argparse
import asyncio
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
import subprocess
import sys


async def run(root, output, panel_path):
    from app.tools.blast import BlastTool
    from app.tools.blast_local import _reference
    panel = json.loads(panel_path.read_text())
    criteria = panel["predeclared_success_criteria"]
    manifest = _reference(str(root.resolve()), (root / "manifest.json").stat().st_mtime_ns)
    if "complete UniProtKB/Swiss-Prot" not in manifest["scope"]:
        raise RuntimeError("Full-reference benchmark requires a complete official snapshot")
    output.mkdir(parents=True, exist_ok=True)
    outcomes = []
    for control in panel["queries"]:
        accession = control["expected_accessions"][0]
        sequence = "".join(subprocess.run(["blastdbcmd", "-db", str(root / "swissprot"),
            "-entry", accession, "-outfmt", "%s"], capture_output=True, text=True,
            check=True, timeout=30).stdout.split())
        if len(sequence) < 20:
            raise RuntimeError("Reference control sequence is missing")
        (output / f"{control['id']}.fasta").write_text(f">{accession}\n{sequence}\n")
        result = await BlastTool().run_uncached({"sequence": sequence, "program": "blastp",
            "database": "uniprotkb_swissprot", "max_hits": panel["program_settings"]["max_hits_requested"]})
        if result.get("source") != "local-blast+" or result.get("search_complete") is not True:
            raise RuntimeError("Production path did not execute the declared local reference")
        (output / f"{control['id']}.xml").write_text(result["raw_result_xml"])
        hits = result["hits"][:criteria["max_recovery_rank"]]
        rank = next((i+1 for i, hit in enumerate(hits)
                     if hit["accession"] in control["expected_accessions"]), None)
        outcomes.append({"id": control["id"], "expected_accessions": control["expected_accessions"],
            "essential": control["essential"], "query_length": len(sequence),
            "query_sha256": hashlib.sha256(sequence.encode()).hexdigest(),
            "recovered_rank": rank, "recovered": rank is not None,
            "top_hits": [{"accession": h["accession"], "identity_pct": h["identity_pct"],
                          "bit_score": h["bit_score"], "evalue": h["evalue"]} for h in hits]})
    fraction = sum(row["recovered"] for row in outcomes) / len(outcomes)
    essential = all(row["recovered"] for row in outcomes if row["essential"])
    passed = (len(outcomes) >= criteria["min_queries"] and
              fraction >= criteria["recovery_threshold_frac"] and
              (essential or not criteria["essential_required"]))
    report = {"benchmark": "local-swissprot-production-accession-recovery/v1",
        "status": "passed" if passed else "failed", "executed": True,
        "recorded_at": datetime.now(timezone.utc).isoformat(),
        "reference": manifest, "n_queries": len(outcomes), "n_recovered": sum(o["recovered"] for o in outcomes),
        "essential_recovered": essential, "criteria": criteria, "queries": outcomes,
        "limitations": ["Canonical queries originate in the searched snapshot: accession-recovery positive controls only.",
                        "No claim of novel sequence function, homology-transfer accuracy, or external NCBI service availability."]}
    (output / "latest.json").write_text(json.dumps(report, indent=2) + "\n")
    print("LOCAL_BLAST_BENCHMARK=" + json.dumps(report, sort_keys=True))
    return 0 if passed else 1


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("reference", type=Path)
    parser.add_argument("output", type=Path)
    parser.add_argument("panel", type=Path)
    args = parser.parse_args()
    sys.exit(asyncio.run(run(args.reference, args.output, args.panel)))