tinymind-native-8b-remote-handoff/bundle/evaluation/perfection_gate.py

"""System perfection gate for TinyMind.

This gate separates two ideas that must not be mixed:

1. operational integrity: local files, schemas, tests, and safety gates pass;
2. absolute perfection/world-best: impossible to assert without exhaustive
   external evidence and continuous monitoring.
"""

from __future__ import annotations

from datetime import datetime, timezone
import json
from pathlib import Path
import subprocess
import sys


DEFAULT_REPORTS = {
    "hyper_pure_manifest": "reports/hyper_pure_knowledge/hyper_pure_manifest.json",
    "lineage_graph": "reports/hyper_pure_lineage/hyper_pure_lineage_graph.json",
    "world_class_eval": "reports/world_class_eval/world_class_eval_report.json",
    "coherence": "reports/axiomweave_coherence/layer_coherence_report.json",
    "extreme_memory": "reports/extreme_memory_10m/extreme_memory_report.json",
    "compact_intelligence": "reports/compact_intelligence/compact_intelligence_dossier.json",
}

DEFAULT_TESTS = [
    "tests/test_hyper_pure_refinery.py",
    "tests/test_lineage_weaver.py",
    "tests/test_world_class_eval.py",
    "tests/test_layer_coherence.py",
    "tests/test_extreme_memory_archive.py",
    "tests/test_elastic_answer.py",
    "tests/test_openai_compatible_api.py",
    "tests/test_axiomweave_architecture.py",
    "tests/test_axiom_lang.py",
]


def _load(path: str | Path) -> dict:
    return json.loads(Path(path).read_text(encoding="utf-8"))


def _check_exists(paths: dict[str, str]) -> list[dict]:
    rows = []
    for name, path in paths.items():
        p = Path(path)
        rows.append({"name": name, "path": str(p), "passed": p.exists() and p.stat().st_size > 0})
    return rows


def _run_tests(test_paths: list[str], timeout_s: int = 240) -> dict:
    cmd = [sys.executable, "-m", "pytest", *test_paths, "-q"]
    started = datetime.now(timezone.utc).isoformat()
    proc = subprocess.run(cmd, cwd=Path(__file__).resolve().parents[1], capture_output=True, text=True, timeout=timeout_s)
    return {
        "command": cmd,
        "started_at": started,
        "exit_code": proc.returncode,
        "passed": proc.returncode == 0,
        "stdout_tail": proc.stdout[-4000:],
        "stderr_tail": proc.stderr[-4000:],
    }


def build_perfection_gate(
    out_dir: str | Path,
    reports: dict[str, str] | None = None,
    run_tests: bool = True,
    test_paths: list[str] | None = None,
) -> dict:
    reports = reports or dict(DEFAULT_REPORTS)
    test_paths = test_paths or list(DEFAULT_TESTS)
    existence = _check_exists(reports)
    loaded = {name: _load(path) for name, path in reports.items() if Path(path).exists()}

    gates = {
        "hyper_pure_dataset": bool(loaded.get("hyper_pure_manifest", {}).get("gate", {}).get("passed")),
        "lineage_integrity": bool(loaded.get("lineage_graph", {}).get("gate", {}).get("passed")),
        "world_eval_packet_ready": bool(loaded.get("world_class_eval", {}).get("claim_gate", {}).get("can_claim_production_ready_eval_packet")),
        "coherence_no_zero_work": bool(loaded.get("coherence", {}).get("no_zero_work_gate", {}).get("passed")),
        "ten_million_exact_memory": bool(loaded.get("extreme_memory", {}).get("passkey_recall", {}).get("passed"))
        and int(loaded.get("extreme_memory", {}).get("measured_tokens", 0)) >= 10_000_000,
        "compact_claim_gate_honest": loaded.get("compact_intelligence", {}).get("world_best_claim_allowed") is False,
    }
    tests = _run_tests(test_paths) if run_tests else {"passed": None, "skipped": True}
    required = {
        "reports_exist": all(row["passed"] for row in existence),
        **gates,
        "tests_pass": tests.get("passed") is True if run_tests else True,
    }
    operational_integrity = all(required.values())
    weak_axes = loaded.get("world_class_eval", {}).get("summary", {}).get("weak_axes", [])
    external_blockers = loaded.get("world_class_eval", {}).get("summary", {}).get("hard_blockers", [])
    report = {
        "schema_version": "tinymind-system-perfection-gate-v1",
        "created_at": datetime.now(timezone.utc).isoformat(),
        "report_existence": existence,
        "required_gates": required,
        "test_run": tests,
        "operational_integrity": {
            "passed": operational_integrity,
            "meaning": "Local system evidence, tests, and safety gates are internally consistent.",
        },
        "perfection_claim": {
            "absolute_100_percent_error_free_allowed": False,
            "world_best_or_flawless_claim_allowed": False,
            "reason": "No finite test suite can prove all possible errors absent; external ranks and continuous production monitoring are still required.",
            "current_weak_axes": weak_axes,
            "external_blockers": external_blockers,
        },
        "release_readiness": {
            "local_research_release_ready": operational_integrity,
            "production_world_claim_ready": False,
        },
    }
    out = Path(out_dir)
    out.mkdir(parents=True, exist_ok=True)
    json_path = out / "system_perfection_gate.json"
    md_path = out / "system_perfection_gate.md"
    report["json_path"] = str(json_path)
    report["markdown_path"] = str(md_path)
    json_path.write_text(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True), encoding="utf-8")
    md_path.write_text(_markdown(report), encoding="utf-8")
    return report


def _markdown(report: dict) -> str:
    lines = [
        "# TinyMind System Perfection Gate",
        "",
        f"- Operational integrity: {report['operational_integrity']['passed']}",
        f"- Local research release ready: {report['release_readiness']['local_research_release_ready']}",
        f"- Absolute 100% error-free claim allowed: {report['perfection_claim']['absolute_100_percent_error_free_allowed']}",
        f"- World-best/flawless claim allowed: {report['perfection_claim']['world_best_or_flawless_claim_allowed']}",
        "",
        "## Required Gates",
        "",
    ]
    for key, value in report["required_gates"].items():
        lines.append(f"- {key}: {value}")
    lines.extend(["", "## Weak Axes", ""])
    for axis in report["perfection_claim"]["current_weak_axes"]:
        lines.append(f"- {axis}")
    if not report["perfection_claim"]["current_weak_axes"]:
        lines.append("- None")
    return "\n".join(lines) + "\n"