release/v1/tests/test_validate_logbook_release.py

from __future__ import annotations

import importlib.util
import json
import sys
from pathlib import Path
from typing import Any

import pytest


SCRIPT_PATH = (
    Path(__file__).resolve().parents[1] / "scripts" / "validate_logbook_release.py"
)
SPEC = importlib.util.spec_from_file_location(
    "chebyshev_validate_logbook_release", SCRIPT_PATH
)
assert SPEC is not None and SPEC.loader is not None
VALIDATOR = importlib.util.module_from_spec(SPEC)
sys.modules[SPEC.name] = VALIDATOR
SPEC.loader.exec_module(VALIDATOR)


def write_json(path: Path, value: Any) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8")


def cell(
    cell_id: str,
    cell_type: str,
    title: str,
    body: str,
    **metadata: Any,
) -> str:
    value = {
        "type": cell_type,
        "id": cell_id,
        "created_at": "2026-07-16T00:00:00+00:00",
        "title": title,
        **metadata,
    }
    return (
        "\n\n---\n<!-- trackio-cell\n"
        + json.dumps(value, sort_keys=True)
        + "\n-->\n"
        + body.rstrip()
        + "\n"
    )


def figure_body(label: str, *, raw: bool = True) -> str:
    value = f"````html\n<div>{label}</div>\n````\n"
    if raw:
        value += f"\n````raw\nlabel,value\n{label},1\n````\n"
    return value


def executive_summary_body() -> str:
    return """The bounded result is ready for release.

## Scope & cost

| Item | This reproduction | Full replication |
| --- | --- | --- |
| Scope | Five challenge claims | Every paper result |
| Hardware | CPU-bound HF Jobs | Physical Aero 2 hardware |
| Compute time | Under 24 hours | Not estimated |
| Cost | Recorded below | Not estimated |
| Outcome | Audited | Outside scope |
"""


def create_valid_logbook(tmp_path: Path, *, postpublish: bool = False) -> tuple[Path, Path]:
    project_root = tmp_path / "project"
    logbook_root = project_root / ".trackio" / "logbook"
    pages = logbook_root / "pages"
    figures_page = pages / "figures" / "page.md"
    conclusion_page = pages / "conclusion" / "page.md"
    figures_page.parent.mkdir(parents=True)
    conclusion_page.parent.mkdir(parents=True)

    (pages / "index.md").write_text("# Release logbook\n", encoding="utf-8")
    figure_text = "# Figures\n"
    for index in range(1, 5):
        figure_text += cell(
            f"cell_figure_{index}",
            "figure",
            f"Evidence figure {index}",
            figure_body(f"figure-{index}"),
        )
    figures_page.write_text(figure_text, encoding="utf-8")

    bundle = project_root / "release" / "bundle.zip"
    bundle.parent.mkdir(parents=True)
    bundle.write_bytes(b"release bundle")
    bundle_uri = (
        "https://huggingface.co/buckets/example/release#bundle.zip"
        if postpublish
        else "trackio-local-path://release/bundle.zip"
    )
    conclusion_text = "# Conclusion\n"
    conclusion_text += cell(
        "cell_executive",
        "markdown",
        "Executive summary",
        executive_summary_body(),
        pinned=True,
        pinned_at="2026-07-16T00:10:00+00:00",
    )
    conclusion_text += cell(
        "cell_poster",
        "figure",
        "Reproduction poster",
        "````html\n<img src=\"data:image/png;base64,AA==\" alt=\"poster\" />\n````\n",
        pinned=True,
        pinned_at="2026-07-16T00:11:00+00:00",
    )
    conclusion_text += cell(
        "cell_bundle",
        "artifact",
        "Reproduction bundle",
        f"**Artifact** release bundle\n\n{bundle_uri}\n",
        path="release/bundle.zip",
        artifact_type="dataset",
    )
    conclusion_page.write_text(conclusion_text, encoding="utf-8")

    manifest = {
        "schema_version": 1,
        "title": "Release logbook",
        "agent_view_tokens": 999,
        "root": {
            "slug": "index",
            "title": "Release logbook",
            "file": "pages/index.md",
            "children": [
                {
                    "slug": "figures",
                    "title": "Figures",
                    "file": "pages/figures/page.md",
                    "children": [],
                },
                {
                    "slug": "conclusion",
                    "title": "Conclusion",
                    "file": "pages/conclusion/page.md",
                    "children": [],
                },
            ],
        },
    }
    write_json(logbook_root / "logbook.json", manifest)
    return project_root, logbook_root


def validate(project_root: Path, logbook_root: Path, mode: str = "prepublish") -> dict:
    return VALIDATOR.validate_logbook_release(logbook_root, project_root, mode)


def error_codes(result: dict) -> set[str]:
    return {error["code"] for error in result["errors"]}


@pytest.mark.parametrize("mode", ["prepublish", "postpublish"])
def test_valid_release_passes_in_both_modes(tmp_path: Path, mode: str) -> None:
    project, logbook = create_valid_logbook(
        tmp_path, postpublish=mode == "postpublish"
    )

    result = validate(project, logbook, mode)

    assert result["valid"] is True
    assert result["status"] == "success"
    assert result["counts"] == {
        "declared_pages": 3,
        "cells": 7,
        "figures": 5,
        "nonposter_figures": 4,
        "artifact_cells": 1,
        "local_artifact_uris": 0 if mode == "postpublish" else 1,
        "pins": 2,
        "errors": 0,
    }


def test_cli_prints_json_and_uses_exit_two_on_failure(
    tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
    project, logbook = create_valid_logbook(tmp_path)
    page = logbook / "pages" / "figures" / "page.md"
    page.write_text(page.read_text(encoding="utf-8") + "\nTODO\n", encoding="utf-8")

    exit_code = VALIDATOR.main(
        [
            "--logbook-root",
            str(logbook),
            "--project-root",
            str(project),
            "--mode",
            "prepublish",
        ]
    )
    output = json.loads(capsys.readouterr().out)

    assert exit_code == 2
    assert output["status"] == "failed"
    assert "placeholder_token" in error_codes(output)


def test_cli_success_exit_zero(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
    project, logbook = create_valid_logbook(tmp_path)

    exit_code = VALIDATOR.main(
        [
            "--logbook-root",
            str(logbook),
            "--project-root",
            str(project),
            "--mode",
            "prepublish",
        ]
    )
    output = json.loads(capsys.readouterr().out)

    assert exit_code == 0
    assert output["valid"] is True


def test_rejects_undeclared_and_missing_pages(tmp_path: Path) -> None:
    project, logbook = create_valid_logbook(tmp_path)
    orphan = logbook / "pages" / "orphan" / "page.md"
    orphan.parent.mkdir()
    orphan.write_text("# Orphan\n", encoding="utf-8")
    (logbook / "pages" / "figures" / "page.md").unlink()

    result = validate(project, logbook)

    assert {"undeclared_page", "missing_declared_page"} <= error_codes(result)


def test_rejects_declared_path_traversal(tmp_path: Path) -> None:
    project, logbook = create_valid_logbook(tmp_path)
    manifest_path = logbook / "logbook.json"
    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
    manifest["root"]["children"][0]["file"] = "pages/../outside.md"
    write_json(manifest_path, manifest)

    result = validate(project, logbook)

    assert "unsafe_page_file" in error_codes(result)


def test_rejects_malformed_cell_metadata_json(tmp_path: Path) -> None:
    project, logbook = create_valid_logbook(tmp_path)
    page = logbook / "pages" / "figures" / "page.md"
    text = page.read_text(encoding="utf-8")
    text = text.replace(
        '"title": "Evidence figure 1"',
        '"title": "Evidence figure 1", BROKEN',
        1,
    )
    page.write_text(text, encoding="utf-8")

    result = validate(project, logbook)

    assert "invalid_cell_metadata_json" in error_codes(result)


def test_rejects_duplicate_json_key_in_cell_metadata(tmp_path: Path) -> None:
    project, logbook = create_valid_logbook(tmp_path)
    page = logbook / "pages" / "figures" / "page.md"
    text = page.read_text(encoding="utf-8")
    text = text.replace(
        '"title": "Evidence figure 1"',
        '"title": "Evidence figure 1", "title": "Duplicate"',
        1,
    )
    page.write_text(text, encoding="utf-8")

    result = validate(project, logbook)

    assert "invalid_cell_metadata_json" in error_codes(result)


def test_rejects_malformed_trackio_marker(tmp_path: Path) -> None:
    project, logbook = create_valid_logbook(tmp_path)
    page = logbook / "pages" / "figures" / "page.md"
    text = page.read_text(encoding="utf-8").replace(
        "<!-- trackio-cell\n", "<!-- trackio-cell \n", 1
    )
    page.write_text(text, encoding="utf-8")

    result = validate(project, logbook)

    assert "malformed_cell_marker" in error_codes(result)


def test_rejects_globally_duplicate_cell_ids(tmp_path: Path) -> None:
    project, logbook = create_valid_logbook(tmp_path)
    page = logbook / "pages" / "figures" / "page.md"
    page.write_text(
        page.read_text(encoding="utf-8").replace("cell_figure_2", "cell_figure_1"),
        encoding="utf-8",
    )

    result = validate(project, logbook)

    assert "duplicate_cell_id" in error_codes(result)


@pytest.mark.parametrize("token", ["__C4_REPRO_RESULT__", "TBD", "TODO", "RUNNING"])
def test_rejects_placeholder_tokens(tmp_path: Path, token: str) -> None:
    project, logbook = create_valid_logbook(tmp_path)
    page = logbook / "pages" / "figures" / "page.md"
    page.write_text(
        page.read_text(encoding="utf-8") + f"\n{token}\n", encoding="utf-8"
    )

    result = validate(project, logbook)

    assert "placeholder_token" in error_codes(result)


def test_ignores_tokens_inside_opaque_figure_payloads(tmp_path: Path) -> None:
    project, logbook = create_valid_logbook(tmp_path)
    page = logbook / "pages" / "figures" / "page.md"
    text = page.read_text(encoding="utf-8")
    text = text.replace(
        "<div>figure-1</div>",
        '<script>const vendor = "__NPO__ TODO file://plotly-worker";</script>',
        1,
    )
    text = text.replace(
        "figure-1,1",
        "figure-1,TODO file://reference __OPAQUE_DATA__",
        1,
    )
    page.write_text(text, encoding="utf-8")

    result = validate(project, logbook)

    assert result["valid"] is True


@pytest.mark.parametrize(
    "old,new,expected_code",
    [
        ("Evidence figure 1", "Evidence figure 1 TODO", "placeholder_token"),
        ("# Figures", "# Figures\nfile://outside-payload", "forbidden_file_uri"),
    ],
)
def test_still_scans_figure_metadata_and_outer_text(
    tmp_path: Path, old: str, new: str, expected_code: str
) -> None:
    project, logbook = create_valid_logbook(tmp_path)
    page = logbook / "pages" / "figures" / "page.md"
    page.write_text(
        page.read_text(encoding="utf-8").replace(old, new, 1),
        encoding="utf-8",
    )

    result = validate(project, logbook)

    assert expected_code in error_codes(result)


def test_rejects_nonposter_figure_without_raw_reference(tmp_path: Path) -> None:
    project, logbook = create_valid_logbook(tmp_path)
    page = logbook / "pages" / "figures" / "page.md"
    text = page.read_text(encoding="utf-8")
    text = text.replace(
        "\n````raw\nlabel,value\nfigure-1,1\n````\n",
        "\n",
        1,
    )
    page.write_text(text, encoding="utf-8")

    result = validate(project, logbook)

    assert "missing_figure_raw" in error_codes(result)


def test_rejects_wrong_nonposter_figure_count(tmp_path: Path) -> None:
    project, logbook = create_valid_logbook(tmp_path)
    page = logbook / "pages" / "figures" / "page.md"
    page.write_text(
        page.read_text(encoding="utf-8")
        + cell(
            "cell_extra_figure",
            "figure",
            "Unexpected fifth evidence figure",
            figure_body("extra"),
        ),
        encoding="utf-8",
    )

    result = validate(project, logbook)

    assert "nonposter_figure_count" in error_codes(result)
    assert result["counts"]["figures"] == 6


def test_rejects_figure_without_html_payload(tmp_path: Path) -> None:
    project, logbook = create_valid_logbook(tmp_path)
    page = logbook / "pages" / "figures" / "page.md"
    text = page.read_text(encoding="utf-8").replace(
        "````html\n<div>figure-1</div>\n````",
        "````html\n   \n````",
        1,
    )
    page.write_text(text, encoding="utf-8")

    result = validate(project, logbook)

    assert "missing_figure_html" in error_codes(result)


@pytest.mark.parametrize(
    "old,new",
    [
        ("Executive summary", "Executive Summary"),
        ("Reproduction poster", "Poster"),
        ("Reproduction bundle", "Bundle"),
    ],
)
def test_requires_exact_conclusion_titles(tmp_path: Path, old: str, new: str) -> None:
    project, logbook = create_valid_logbook(tmp_path)
    page = logbook / "pages" / "conclusion" / "page.md"
    page.write_text(
        page.read_text(encoding="utf-8").replace(f'"{old}"', f'"{new}"', 1),
        encoding="utf-8",
    )

    result = validate(project, logbook)

    assert "required_conclusion_cell" in error_codes(result)


@pytest.mark.parametrize("token", VALIDATOR.EXECUTIVE_SUMMARY_TOKENS)
def test_requires_all_executive_summary_scope_cost_fields(
    tmp_path: Path, token: str
) -> None:
    project, logbook = create_valid_logbook(tmp_path)
    page = logbook / "pages" / "conclusion" / "page.md"
    page.write_text(
        page.read_text(encoding="utf-8").replace(token, "REMOVED", 1),
        encoding="utf-8",
    )

    result = validate(project, logbook)

    assert "executive_summary_content" in error_codes(result)


def test_pin_order_matches_trackio_pinned_at_sort(tmp_path: Path) -> None:
    project, logbook = create_valid_logbook(tmp_path)
    page = logbook / "pages" / "conclusion" / "page.md"
    text = page.read_text(encoding="utf-8")
    text = text.replace("00:10:00+00:00", "00:12:00+00:00", 1)
    text = text.replace("00:11:00+00:00", "00:09:00+00:00", 1)
    page.write_text(text, encoding="utf-8")

    result = validate(project, logbook)

    assert "pin_order" in error_codes(result)


def test_requires_exactly_two_real_trackio_pins(tmp_path: Path) -> None:
    project, logbook = create_valid_logbook(tmp_path)
    page = logbook / "pages" / "conclusion" / "page.md"
    text = page.read_text(encoding="utf-8")
    text = text.replace('"pinned": true, ', "", 1)
    text = text.replace('"pinned_at": "2026-07-16T00:10:00+00:00", ', "", 1)
    page.write_text(text, encoding="utf-8")

    result = validate(project, logbook)

    assert "pin_count" in error_codes(result)


def test_requires_valid_trackio_pin_metadata(tmp_path: Path) -> None:
    project, logbook = create_valid_logbook(tmp_path)
    page = logbook / "pages" / "conclusion" / "page.md"
    page.write_text(
        page.read_text(encoding="utf-8").replace(
            '"pinned": true', '"pinned": "true"', 1
        ),
        encoding="utf-8",
    )

    result = validate(project, logbook)

    assert "invalid_pin_metadata" in error_codes(result)


def test_prepublish_requires_local_artifact_file_to_exist(tmp_path: Path) -> None:
    project, logbook = create_valid_logbook(tmp_path)
    (project / "release" / "bundle.zip").unlink()

    result = validate(project, logbook)

    assert "missing_local_artifact" in error_codes(result)


def test_prepublish_rejects_local_artifact_outside_project(tmp_path: Path) -> None:
    project, logbook = create_valid_logbook(tmp_path)
    outside = tmp_path / "outside.zip"
    outside.write_bytes(b"outside")
    page = logbook / "pages" / "conclusion" / "page.md"
    page.write_text(
        page.read_text(encoding="utf-8").replace(
            "trackio-local-path://release/bundle.zip",
            "trackio-local-path://../outside.zip",
        ),
        encoding="utf-8",
    )

    result = validate(project, logbook)

    assert "unsafe_local_artifact_path" in error_codes(result)


@pytest.mark.parametrize(
    "uri,expected_code",
    [
        ("trackio-local-path://release/bundle.zip", "unpublished_local_artifact"),
        ("trackio-artifact://release/bundle:v1", "unpublished_trackio_artifact"),
    ],
)
def test_postpublish_forbids_unresolved_trackio_artifact_uris(
    tmp_path: Path, uri: str, expected_code: str
) -> None:
    project, logbook = create_valid_logbook(tmp_path)
    page = logbook / "pages" / "conclusion" / "page.md"
    page.write_text(
        page.read_text(encoding="utf-8").replace(
            "trackio-local-path://release/bundle.zip", uri
        ),
        encoding="utf-8",
    )

    result = validate(project, logbook, "postpublish")

    assert expected_code in error_codes(result)
    assert "artifact_without_public_url" in error_codes(result)


@pytest.mark.parametrize("value", [100_000, True, "999"])
def test_rejects_invalid_or_oversized_agent_view_tokens(
    tmp_path: Path, value: Any
) -> None:
    project, logbook = create_valid_logbook(tmp_path)
    manifest_path = logbook / "logbook.json"
    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
    manifest["agent_view_tokens"] = value
    write_json(manifest_path, manifest)

    result = validate(project, logbook)

    assert error_codes(result) & {
        "invalid_agent_view_tokens",
        "agent_view_tokens_limit",
    }


@pytest.mark.parametrize("language", ["python", "json"])
def test_rejects_titled_python_and_json_code_attachments(
    tmp_path: Path, language: str
) -> None:
    project, logbook = create_valid_logbook(tmp_path)
    page = logbook / "pages" / "figures" / "page.md"
    page.write_text(
        page.read_text(encoding="utf-8")
        + cell(
            "cell_code_attachment",
            "code",
            "Oversized attachment",
            f"````{language} title=payload.{language}\n{{}}\n````\n",
        ),
        encoding="utf-8",
    )

    result = validate(project, logbook)

    assert "titled_code_attachment" in error_codes(result)


def test_invalid_mode_fails_closed(tmp_path: Path) -> None:
    project, logbook = create_valid_logbook(tmp_path)

    result = validate(project, logbook, "draft")

    assert result["valid"] is False
    assert "invalid_mode" in error_codes(result)