source/src/bgc_retrieval/external_prepare.py

"""Prepare the pinned BGC clustering benchmark without altering upstream files."""

from __future__ import annotations

import ast
from pathlib import Path
from typing import Any

import pandas as pd
from Bio import SeqIO
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord

from .artifacts import file_record, sha256_file, write_json_immutable


def build_product_mappings(source_tsv: str | Path) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
    source = pd.read_csv(source_tsv, sep="\t")
    required = {"npaid", "compound_inchikey", "genus", "mibig_ids"}
    if missing := required.difference(source.columns):
        raise ValueError(f"NPAtlas source is missing columns: {sorted(missing)}")
    rows: list[dict[str, str]] = []
    for record in source.itertuples(index=False):
        try:
            mibig_ids = ast.literal_eval(record.mibig_ids) if isinstance(record.mibig_ids, str) else []
        except (SyntaxError, ValueError) as error:
            raise ValueError(f"Invalid mibig_ids for {record.npaid}") from error
        for bgc_id in mibig_ids:
            rows.append(
                {
                    "bgc_id": str(bgc_id),
                    "product_id": str(record.compound_inchikey),
                    "genus": str(record.genus),
                    "npaid": str(record.npaid),
                }
            )
    expanded = pd.DataFrame(rows).drop_duplicates()
    metadata_rows: list[dict[str, Any]] = []
    gold_rows: list[dict[str, str]] = []
    exclusions: list[dict[str, str]] = []
    for bgc_id, group in expanded.groupby("bgc_id"):
        products = sorted(group["product_id"].dropna().unique())
        genera = sorted(group["genus"].dropna().unique())
        metadata_rows.append(
            {
                "bgc_id": bgc_id,
                "product_ids": ";".join(products),
                "product_count": len(products),
                "genera": ";".join(genera),
                "genus_count": len(genera),
            }
        )
        if len(products) == 1 and len(genera) == 1:
            gold_rows.append(
                {
                    "bgc_id": bgc_id,
                    "product_group_id": f"inchikey:{products[0]}",
                    "product_id": products[0],
                    "mibig_reference_id": bgc_id,
                    "genus": genera[0],
                    "source": "BGC-clustering-benchmark@bb8500d60f90",
                }
            )
        else:
            reason = "multiple_products" if len(products) != 1 else "multiple_or_missing_genera"
            exclusions.append(
                {"bgc_id": bgc_id, "reason": reason, "values": ";".join(products)}
            )
    return pd.DataFrame(metadata_rows), pd.DataFrame(gold_rows), pd.DataFrame(exclusions)


def _feature_translation(feature: Any, nucleotide_record: Any) -> str:
    translations = feature.qualifiers.get("translation", [])
    if translations:
        return str(translations[0]).replace(" ", "").rstrip("*")
    table = int(feature.qualifiers.get("transl_table", [11])[0])
    nucleotide = feature.extract(nucleotide_record.seq)
    return str(nucleotide.translate(table=table, to_stop=False)).rstrip("*")


def extract_benchmark_proteins(
    genbank_dir: str | Path,
    fasta_path: str | Path,
    atlas_path: str | Path,
    provenance_path: str | Path,
    source_commit: str,
) -> dict[str, Any]:
    fasta_output = Path(fasta_path)
    atlas_output = Path(atlas_path)
    provenance_output = Path(provenance_path)
    for output in (fasta_output, atlas_output, provenance_output):
        if output.exists():
            raise FileExistsError(f"Refusing to overwrite external preparation output: {output}")
        output.parent.mkdir(parents=True, exist_ok=True)

    protein_records: list[SeqRecord] = []
    atlas_rows: list[dict[str, Any]] = []
    excluded_files: list[dict[str, str]] = []
    source_files = sorted(Path(genbank_dir).glob("*.gbk"))
    for genbank_path in source_files:
        bgc_id = genbank_path.stem
        protein_ids: list[str] = []
        organisms: set[str] = set()
        ordinal = 0
        try:
            with genbank_path.open("r", encoding="utf-8") as handle:
                nucleotide_records = list(SeqIO.parse(handle, "genbank"))
            for nucleotide_record in nucleotide_records:
                organism = str(nucleotide_record.annotations.get("organism", "unknown"))
                organisms.add(organism)
                for feature in nucleotide_record.features:
                    if feature.type != "CDS":
                        continue
                    sequence = _feature_translation(feature, nucleotide_record)
                    if not sequence:
                        continue
                    gene_id = f"{bgc_id}__cds{ordinal:05d}"
                    protein_ids.append(gene_id)
                    protein_records.append(
                        SeqRecord(Seq(sequence), id=gene_id, description=f"source={bgc_id}")
                    )
                    ordinal += 1
        except Exception as error:
            excluded_files.append({"bgc_id": bgc_id, "reason": f"parse_error:{type(error).__name__}"})
            continue
        if not protein_ids:
            excluded_files.append({"bgc_id": bgc_id, "reason": "no_translated_cds"})
            continue
        atlas_rows.append(
            {
                "bgc_id": bgc_id,
                "protein_ids": ";".join(protein_ids),
                "num_proteins": len(protein_ids),
                "organisms": ";".join(sorted(organisms)),
            }
        )

    SeqIO.write(protein_records, fasta_output, "fasta-2line")
    pd.DataFrame(atlas_rows).to_csv(atlas_output, index=False)
    provenance = {
        "schema_version": 1,
        "source_commit": source_commit,
        "source_files": len(source_files),
        "usable_bgcs": len(atlas_rows),
        "proteins": len(protein_records),
        "excluded_files": excluded_files,
        "fasta_sha256": sha256_file(fasta_output),
        "atlas_sha256": sha256_file(atlas_output),
        "source_manifest": [file_record(path, genbank_dir) for path in source_files],
    }
    write_json_immutable(provenance_output, provenance)
    return provenance


def write_mapping_outputs(
    source_tsv: str | Path,
    mapping_path: str | Path,
    metadata_path: str | Path,
    exclusions_path: str | Path,
) -> dict[str, int]:
    metadata, mapping, exclusions = build_product_mappings(source_tsv)
    for path in (mapping_path, metadata_path, exclusions_path):
        output = Path(path)
        output.parent.mkdir(parents=True, exist_ok=True)
        if output.exists():
            raise FileExistsError(f"Refusing to overwrite external mapping output: {output}")
    mapping.to_csv(mapping_path, index=False)
    metadata.to_csv(metadata_path, index=False)
    exclusions.to_csv(exclusions_path, index=False)
    return {
        "all_mapped_bgcs": len(metadata),
        "unambiguous_gold_bgcs": len(mapping),
        "ambiguous_exclusions": len(exclusions),
    }