scripts/execute_alpha5_experiments.py

"""Execute Alpha.5 experiments EXP-DET-0001 and EXP-STOR-0001.

This script:
1. Runs the determinism A/B/C experiment (EXP-DET-0001)
2. Runs the storage persistence experiment (EXP-STOR-0001)
3. Records manifests for both experiments
4. Generates DATA-* and EVID-* artifacts
5. Rebuilds Research Catalog and Evidence Matrix
"""

from __future__ import annotations

import hashlib
import json
import subprocess
import sys
import time
from pathlib import Path
from typing import Any, cast

REPO_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO_ROOT))

from src.dashboard.verification import compute_source_tree_digest  # noqa: E402
from src.research.evidence_engine import EvidenceEngine  # noqa: E402
from src.research.experiment_recorder import (  # noqa: E402
    ExperimentRecorder,
    get_software_info,
)
from src.research.registry import ResearchRegistry  # noqa: E402
from src.research.report_builder import ReportBuilder  # noqa: E402

EXPERIMENTS_DIR = REPO_ROOT / "research" / "experiments"
EVIDENCE_DIR = REPO_ROOT / "research" / "registry" / "evidence"
DATA_DIR = REPO_ROOT / "research" / "generated" / "data"


# Map experiments to their canonical runtime configuration files.
EXPERIMENT_CONFIGS: dict[str, Path] = {
    "EXP-DET-0001": REPO_ROOT / "configs" / "poc_alpha5_live.yaml",
    "EXP-STOR-0001": REPO_ROOT / "configs" / "poc_alpha5_live.yaml",
}


def _sha256_file(path: Path) -> str:
    """Return the SHA-256 hex digest of a file."""
    h = hashlib.sha256()
    with open(path, "rb") as f:
        for chunk in iter(lambda: f.read(8192), b""):
            h.update(chunk)
    return h.hexdigest()


def _brain5d_version() -> str:
    """Read Brain-5D version from pyproject.toml, fallback to import."""
    pyproject = REPO_ROOT / "pyproject.toml"
    if pyproject.exists():
        try:
            text = pyproject.read_text(encoding="utf-8")
            for line in text.splitlines():
                if line.strip().startswith("version"):
                    _, _, value = line.partition("=")
                    return value.strip().strip('"').strip("'")
        except Exception:
            pass
    return get_software_info().get("brain5d_version", "unknown")


def _run_pytest(test_path: str, timeout: int = 120) -> dict[str, Any]:
    """Run a pytest file and return pass/fail with output."""
    command = [sys.executable, "-m", "pytest", test_path, "-q", "--tb=short"]
    start = time.time()
    result = subprocess.run(
        command,
        capture_output=True,
        text=True,
        timeout=timeout,
        cwd=str(REPO_ROOT),
    )
    duration = time.time() - start
    return {
        "command": f"python -m pytest {test_path} -q --tb=short",
        "returncode": result.returncode,
        "passed": result.returncode == 0,
        "stdout": result.stdout,
        "stderr": result.stderr,
        "duration_seconds": duration,
    }


def _run_determinism_experiment() -> dict[str, Any]:
    """Run EXP-DET-0001: A/B/C restore determinism."""
    result = _run_pytest("tests/test_restore_determinism_abc.py", timeout=300)
    return result


def _run_storage_experiment() -> dict[str, Any]:
    """Run EXP-STOR-0001: B5D storage roundtrip persistence."""
    result = _run_pytest("tests/test_b5d_storage.py", timeout=300)
    return result


def _update_manifest(
    experiment_id: str,
    result: dict[str, Any],
    data_path: Path,
    claim_id: str,
    hypothesis_id: str,
    research_question_id: str,
) -> None:
    """Update experiment manifest with full provenance."""
    config_path = EXPERIMENT_CONFIGS.get(experiment_id)
    config_sha256 = (
        _sha256_file(config_path) if config_path and config_path.exists() else ""
    )

    duration_seconds = float(result["duration_seconds"])
    passed = bool(result["passed"])
    returncode = int(result["returncode"])
    stdout = str(result["stdout"])
    stderr = str(result["stderr"])

    recorder = ExperimentRecorder(experiment_id)
    recorder.record_software_version("brain5d_version", _brain5d_version())
    recorder.record_config(
        config_path=str(config_path) if config_path else "",
        sha256=config_sha256,
    )
    recorder.record_research_links(
        research_questions=[research_question_id],
        hypotheses=[hypothesis_id],
    )
    recorder.record_results(
        passed=passed,
        returncode=returncode,
        duration_seconds=duration_seconds,
        stdout_summary=stdout[:2000],
        stderr_summary=stderr[:2000],
    )
    recorder.record_runtime(duration_seconds=duration_seconds)
    recorder.record_artifact("test_output", data_path.relative_to(REPO_ROOT).as_posix())
    recorder.mark_completed()
    recorder.save()


def _generate_data_artifact(experiment_id: str, result: dict[str, Any]) -> Path:
    """Generate a DATA-* artifact for an experiment."""
    DATA_DIR.mkdir(parents=True, exist_ok=True)
    year = time.localtime().tm_year
    existing = list(DATA_DIR.glob(f"DATA-{year}-*.json"))
    data_id = f"DATA-{year}-{len(existing) + 1:02d}"
    data_path = DATA_DIR / f"{data_id}.json"
    data_record = {
        "data_id": data_id,
        "experiment_id": experiment_id,
        "generated": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
        "tree_digest": compute_source_tree_digest(REPO_ROOT),
        "test_command": result.get("command", ""),
        "results": {
            "passed": result["passed"],
            "returncode": result["returncode"],
            "duration_seconds": result["duration_seconds"],
        },
        "raw_outputs": {
            "stdout": result["stdout"],
            "stderr": result["stderr"],
        },
    }
    data_path.write_text(
        json.dumps(data_record, indent=2, ensure_ascii=False), encoding="utf-8"
    )
    return data_path


def _create_evidence(
    experiment_id: str,
    claim_id: str,
    hypothesis_id: str,
    data_path: Path,
    summary: str,
    status: str,
) -> str:
    """Create an EVID-* record linking experiment to claim/hypothesis."""
    registry = ResearchRegistry(REPO_ROOT / "research" / "registry")
    registry.load_all()
    engine = EvidenceEngine(registry)
    evidence_id = engine.evaluate_experiment(
        experiment_id=experiment_id,
        claim_id=claim_id,
        hypothesis_id=hypothesis_id,
        result_summary=summary,
        status=status,
    )
    evidence_path = EVIDENCE_DIR / f"{evidence_id}.json"
    evidence: Any = json.loads(evidence_path.read_text(encoding="utf-8"))
    if not isinstance(evidence, dict):
        raise TypeError(f"invalid evidence record: {evidence_path}")
    evidence_record = cast(dict[str, Any], evidence)
    raw_artifacts = evidence_record.get("artifacts")
    if not isinstance(raw_artifacts, dict):
        artifacts = {}
    else:
        artifacts = cast(dict[str, Any], raw_artifacts)
    artifacts["data_files"] = [data_path.relative_to(REPO_ROOT).as_posix()]
    evidence_record["artifacts"] = artifacts
    evidence_path.write_text(
        json.dumps(evidence_record, indent=2, ensure_ascii=False), encoding="utf-8"
    )
    return evidence_id


def _rebuild_reports(refresh_registry: bool = False) -> None:
    """Rebuild Research Catalog and Evidence Matrix."""
    registry = ResearchRegistry(REPO_ROOT / "research" / "registry")
    registry.load_all()
    builder = ReportBuilder(registry)
    catalog = builder.build_research_catalog()
    matrix = builder.build_evidence_matrix()
    (REPO_ROOT / "research" / "generated" / "RESEARCH_CATALOG.md").write_text(
        catalog, encoding="utf-8"
    )
    (REPO_ROOT / "research" / "generated" / "EVIDENCE_MATRIX.md").write_text(
        matrix, encoding="utf-8"
    )
    print("Rebuilt RESEARCH_CATALOG.md and EVIDENCE_MATRIX.md")


def main() -> int:
    print("=" * 60)
    print("Alpha.5 Experiment Execution")
    print("=" * 60)

    # EXP-DET-0001: Determinism A/B/C
    print("\n[EXP-DET-0001] Running determinism A/B/C experiment...")
    det_result = _run_determinism_experiment()
    print(
        f"  passed={det_result['passed']}, duration={det_result['duration_seconds']:.1f}s"
    )
    det_data = _generate_data_artifact("EXP-DET-0001", det_result)
    _update_manifest(
        "EXP-DET-0001",
        det_result,
        det_data,
        claim_id="CLAIM-DET-001",
        hypothesis_id="H-SNN-003-A",
        research_question_id="RQ-DET-001",
    )
    print(f"  DATA artifact: {det_data.name}")
    det_evid = _create_evidence(
        "EXP-DET-0001",
        claim_id="CLAIM-DET-001",
        hypothesis_id="H-SNN-003-A",
        data_path=det_data,
        summary="A/B/C restore determinism verified: process-restart restore produces identical structural and dynamic state.",
        status="supports" if det_result["passed"] else "inconclusive",
    )
    print(f"  EVID artifact: {det_evid}")

    # EXP-STOR-0001: Storage persistence
    print("\n[EXP-STOR-0001] Running storage persistence experiment...")
    stor_result = _run_storage_experiment()
    print(
        f"  passed={stor_result['passed']}, duration={stor_result['duration_seconds']:.1f}s"
    )
    stor_data = _generate_data_artifact("EXP-STOR-0001", stor_result)
    _update_manifest(
        "EXP-STOR-0001",
        stor_result,
        stor_data,
        claim_id="CLAIM-STOR-001",
        hypothesis_id="H-STOR-001-A",
        research_question_id="RQ-STORAGE-001",
    )
    print(f"  DATA artifact: {stor_data.name}")
    stor_evid = _create_evidence(
        "EXP-STOR-0001",
        claim_id="CLAIM-STOR-001",
        hypothesis_id="H-STOR-001-A",
        data_path=stor_data,
        summary="B5D storage roundtrip verified: full network state serializes and deserializes without loss.",
        status="supports" if stor_result["passed"] else "inconclusive",
    )
    print(f"  EVID artifact: {stor_evid}")

    # Rebuild reports after evidence has updated the registry
    print("\n[Reports] Rebuilding Research Catalog and Evidence Matrix...")
    _rebuild_reports(refresh_registry=True)

    all_passed = bool(det_result["passed"]) and bool(stor_result["passed"])
    print("\n" + "=" * 60)
    print(
        "Alpha.5 experiments executed successfully."
        if all_passed
        else "Alpha.5 experiment execution failed."
    )
    print("=" * 60)
    return 0 if all_passed else 1


if __name__ == "__main__":
    sys.exit(main())