src/uploads/validation.py

"""Validate-before-availability for a quarantined upload (ADR-0011 items 3, 4, 5).

A staged file stays quarantined until it clears the SAME grounding gate a
registered dataset clears:

    1. manifest schema validation  (src.datasets.manifest_schema.validate_manifest)
    2. semantic against-data checks (src.workflows.manifest_data_validation)

Item 4 ("never execute uploaded content") is honoured structurally: the file is
opened ONLY by the vetted scanpy loader — the same loader registered data uses —
never ``exec``/``eval``/``pickle``. Passing both checks flips the record to
``validated``; failing keeps it quarantined with the errors surfaced.
"""
from __future__ import annotations

from datetime import datetime
from pathlib import Path
from typing import Any

from src.uploads.records import (
    STATUS_QUARANTINED,
    STATUS_VALIDATED,
    UploadRecord,
    persist_record,
)


def _report(status: str, errors: list[str], **extra: Any) -> dict:
    return {"status": status, "errors": errors, **extra}


def validate_upload(record: UploadRecord) -> tuple[UploadRecord, dict]:
    """Run the manifest-schema + against-data gate on a quarantined upload.

    Returns ``(record, report)``. The record's ``status`` becomes ``validated``
    only when both gates pass; otherwise it stays ``quarantined`` and the reasons
    are in ``record.errors`` / ``report['errors']``.
    """
    if record.status != STATUS_QUARANTINED:
        return record, _report(
            "error",
            [f"Upload is '{record.status}', not 'quarantined' — nothing to validate."],
        )
    if not record.staged_path or not Path(record.staged_path).is_file():
        record.errors = [f"Staged file missing: {record.staged_path}"]
        persist_record(record)
        return record, _report("error", record.errors)

    # ── Gate 1: manifest schema validity ─────────────────────────────────────
    from src.datasets.manifest_schema import DatasetManifest, validate_manifest

    schema_result = validate_manifest(record.manifest)
    if not schema_result.valid:
        record.errors = [f"manifest schema: {e}" for e in schema_result.errors]
        persist_record(record)
        return record, _report(
            "error", record.errors, schema_warnings=schema_result.warnings
        )

    manifest_obj = DatasetManifest.from_dict(record.manifest)

    # ── Gate 2: against-data validation via the VETTED loader only (item 4) ──
    suffix = Path(record.staged_path).suffix.lower()
    if suffix != ".h5ad":
        # The gate accepts tabular for staging, but auto against-data validation
        # is h5ad-only this phase (every served dataset is an h5ad). Stay
        # quarantined and say so honestly rather than pretend it validated.
        record.errors = [
            f"Auto-validation for '{suffix}' uploads is not implemented yet — "
            "convert to .h5ad or validate manually. File stays quarantined."
        ]
        persist_record(record)
        return record, _report("error", record.errors)

    try:
        import numpy as np
        import scanpy as sc

        from src.workflows.manifest_data_validation import validate_manifest_against_data

        adata = sc.read_h5ad(record.staged_path)  # vetted loader — never exec

        X = adata.X
        if hasattr(X, "toarray"):
            X = X.toarray()
        X_flat = X.flatten()
        if len(X_flat) > 50_000:
            rng = np.random.default_rng(0)
            X_flat = rng.choice(X_flat, size=50_000, replace=False)

        data_report = validate_manifest_against_data(
            X_flat, list(adata.var.index), adata.obs.copy(), manifest_obj
        )
    except Exception as exc:  # noqa: BLE001 — a load/validate failure is a rejection
        record.errors = [f"against-data validation failed to run: {exc}"]
        persist_record(record)
        return record, _report("error", record.errors)

    if not data_report.get("overall_valid"):
        record.errors = list(data_report.get("errors", [])) or [
            "against-data validation reported the manifest inconsistent with the file."
        ]
        persist_record(record)
        return record, _report(
            "error", record.errors, against_data=data_report,
            schema_warnings=schema_result.warnings,
        )

    # ── Passed both gates ────────────────────────────────────────────────────
    record.status = STATUS_VALIDATED
    record.validated_at = datetime.now().isoformat(timespec="seconds")
    record.errors = []
    persist_record(record)
    return record, _report(
        "pass", [], against_data=data_report, schema_warnings=schema_result.warnings
    )