skillwiki/tests/test_harness_codex_cli.py

from __future__ import annotations

from pathlib import Path

import pytest

from skillos.layers.skill_runtime.harness import CodexCliHarness, HarnessTestCase, HarnessWorkspace
from skillos.layers.skill_runtime.harness import codex_cli
from skillos.models.skill_model import Skill, SkillEvaluation, SkillImplementation, SkillInterface


def _skill() -> Skill:
    return Skill(
        name="codex_harness_skill",
        description="Return an ok flag through the Codex CLI harness.",
        interface=SkillInterface(
            input_schema={"type": "object", "properties": {}},
            output_schema={"type": "object", "properties": {"ok": {"type": "boolean"}}},
        ),
        implementation=SkillImplementation(code="output['ok'] = True"),
        evaluation=SkillEvaluation(
            verifier_specs=[{"type": "json_equals", "path": "output.ok", "value": True}]
        ),
    )


@pytest.mark.asyncio
async def test_codex_cli_harness_returns_unavailable_when_missing(tmp_path, monkeypatch):
    monkeypatch.setattr(codex_cli.shutil, "which", lambda command: None)
    workspace = HarnessWorkspace("loop-codex-missing", root=tmp_path)
    skill = _skill()
    test_case = HarnessTestCase(
        test_id="codex-missing",
        name="codex missing",
        goal="return ok",
        verifier_specs=skill.evaluation.verifier_specs,
    )

    result = await CodexCliHarness().run_skill(skill, test_case, workspace, attempt=1)

    assert result.status == "harness_unavailable"
    assert result.verifier_passed is False
    assert "not available" in result.failure_reason


@pytest.mark.asyncio
async def test_codex_cli_harness_parses_output_json_from_mocked_process(tmp_path, monkeypatch):
    monkeypatch.setattr(codex_cli.shutil, "which", lambda command: "C:/fake/codex.exe")

    async def fake_run(args, stdin, *, cwd, timeout_s):  # noqa: ANN001
        Path(cwd, "output.json").write_text('{"ok": true}', encoding="utf-8")
        return {"returncode": 0, "stdout": "done", "stderr": ""}

    monkeypatch.setattr(codex_cli, "_run_subprocess", fake_run)
    workspace = HarnessWorkspace("loop-codex-pass", root=tmp_path)
    skill = _skill()
    test_case = HarnessTestCase(
        test_id="codex-pass",
        name="codex pass",
        goal="return ok",
        verifier_specs=skill.evaluation.verifier_specs,
    )

    result = await CodexCliHarness().run_skill(skill, test_case, workspace, attempt=1)

    assert result.status == "passed"
    assert result.verifier_passed is True
    assert result.stdout == "done"
    assert (tmp_path / "loop-codex-pass" / "attempt-001" / "SKILL.md").exists()
    assert (tmp_path / "loop-codex-pass" / "attempt-001" / "prompt.md").exists()