tests/test_cli.py

"""CLI surface tests — keyless paths only."""
import pytest

from autocodabench.cli.main import main, _make_progress_renderer

from conftest import DEMO_SLUG


# --- create progress renderer (default vs --debug) -------------------------

_EVENTS = [
    {"kind": "phase", "index": 2, "total": 3, "title": "Building the bundle",
     "detail": "writing files; linting; zipping"},
    {"kind": "tool_use", "name": "mcp__autocodabench__autocodabench_init_bundle",
     "input": {"slug": "create"}},
    {"kind": "tool_use", "name": "mcp__autocodabench__autocodabench_log_event",
     "input": {"kind": "progress", "message": "Wrote the scoring program."}},
    {"kind": "tool_result", "is_error": True,
     "preview": "TypeError: unexpected keyword argument 'multi_class'"},
    {"kind": "tool_result", "is_error": True,
     "preview": "Cancelled: parallel tool call Bash(...) errored"},
    {"kind": "tool_use", "name": "mcp__autocodabench__autocodabench_log_event",
     "input": {"kind": "deviation", "message": "Removed multi_class; acc 0.92."}},
    {"kind": "text", "text": "Let me wire the scoring program next."},
    {"kind": "phase_done", "phase": "build", "ok": True, "num_turns": 23},
]


def _render(events, *, debug):
    import io
    import contextlib
    buf = io.StringIO()
    r = _make_progress_renderer(debug=debug)
    with contextlib.redirect_stdout(buf):
        for e in events:
            r(e)
    return buf.getvalue()


def test_default_renderer_is_user_oriented():
    out = _render(_EVENTS, debug=False)
    # User-facing milestone + deviation messages are shown…
    assert "Wrote the scoring program." in out
    assert "Removed multi_class; acc 0.92." in out
    # …tool calls are rendered as friendly actions (not raw tool ids)…
    assert "Init bundle create" in out
    assert "init_bundle" not in out
    # …the agent's narration is shown (it is the user-friendly story)…
    assert "Let me wire the scoring program next." in out
    # …but raw errors and benign parallel cancellations are suppressed.
    assert "TypeError" not in out
    assert "Cancelled" not in out


def test_debug_renderer_shows_full_trace_and_softens_cancellations():
    out = _render(_EVENTS, debug=True)
    assert "Init bundle create" in out             # friendly action…
    assert "init_bundle" in out                    # …with the raw id kept greppable
    assert "TypeError" in out                      # genuine error shown
    assert "Let me wire the scoring program next." in out  # narration shown
    assert "Cancelled" not in out                  # cascade is reworded…
    assert "retried" in out                        # …as a benign retry


# --- markdown-table rendering in narration (CLI-only) ----------------------

_TABLE_TEXT = (
    "Plan ready. Summary:\n\n"
    "| Section | Decision |\n"
    "|---|---|\n"
    "| **Task** | 3-class image classification; **result-submission** |\n"
    "| **Metric** | `scipy.stats.gmean(per_group_accuracies)` |\n\n"
    "Phase 2 will now run."
)


def test_narration_table_is_rendered_as_aligned_box():
    out = _render([{"kind": "text", "text": _TABLE_TEXT}], debug=False)
    # Drawn as a box with aligned borders…
    assert "┌" in out and "┴" in out and "│" in out
    # …header and cell content preserved, emphasis/backtick markers stripped…
    assert "Section" in out and "Decision" in out
    assert "scipy.stats.gmean(per_group_accuracies)" in out
    assert "**Task**" not in out and "`scipy" not in out
    # …and the markdown separator row is not echoed as raw pipes.
    assert "|---|" not in out
    # Surrounding prose is still shown.
    assert "Plan ready. Summary:" in out
    assert "Phase 2 will now run." in out


def test_table_detection_helpers():
    from autocodabench.cli.progress import (
        _has_md_table, _is_table_separator, _table_cells, _strip_md)
    assert _has_md_table(_TABLE_TEXT)
    assert not _has_md_table("just prose | with a stray pipe")
    assert _is_table_separator("|---|:--:|")
    assert not _is_table_separator("| a | b |")
    assert _table_cells("| a | b |") == ["a", "b"]
    assert _table_cells("no pipes here") is None
    assert _strip_md("**bold** and `code`") == "bold and code"


def test_status_glyphs_are_colorized_on_tty():
    from autocodabench.cli.progress import _colorize_glyphs, _GREEN, _YELLOW, _RED
    out = _colorize_glyphs("✓ ⚠ ✗")
    assert _GREEN in out and _YELLOW in out and _RED in out
    # Non-glyph text is untouched.
    assert _colorize_glyphs("plain") == "plain"


def test_provenance_table_renders_with_glyphs():
    from autocodabench.cli.progress import ProgressUI, _GREEN, _RED
    import io, contextlib
    text = (
        "| # | Dimension | Status | Origin |\n"
        "|---|---|---|---|\n"
        "| 1 | Task | ✓ | proposal §2 |\n"
        "| 3 | Metric | ✗ | inferred |\n"
    )
    ui = ProgressUI(debug=False)
    ui.animate = True  # force ANSI so glyph colors are emitted
    buf = io.StringIO()
    with contextlib.redirect_stdout(buf):
        ui._on_text({"text": text})
    out = buf.getvalue()
    assert "┌" in out and "Dimension" in out          # drawn as a box
    assert _GREEN in out and _RED in out               # ✓ green, ✗ red
    assert "|---|" not in out                           # separator not echoed


def test_disp_width_counts_emoji_as_two():
    from autocodabench.cli.progress import _disp_width, _pad
    # ✅/❌ are East-Asian "Wide"; ⚠️ is base glyph + a U+FE0F selector — all 2.
    assert _disp_width("✅") == 2
    assert _disp_width("❌") == 2
    assert _disp_width("⚠️") == 2
    assert _disp_width("✓") == 1 and _disp_width("Status") == 6
    # Emoji-aware ljust pads to the right *display* width.
    assert _disp_width(_pad("✅", 6)) == 6
    assert _disp_width(_pad("⚠️", 6)) == 6


def test_provenance_table_with_emoji_aligns():
    """The provenance table uses colour emoji (✅/⚠️/❌); the CLI box must still
    align — every border/content row renders to the same display width."""
    from autocodabench.cli.progress import ProgressUI, _disp_width
    import io, contextlib
    text = (
        "| # | Dimension | Status | Origin |\n"
        "|---|---|---|---|\n"
        "| 1 | Task | ✅ | proposal §2 |\n"
        "| 2 | Metric | ❌ | inferred |\n"
        "| 3 | Data | ⚠️ | partial |\n"
    )
    ui = ProgressUI(debug=False)
    ui.animate = False  # plain output so widths are measurable
    buf = io.StringIO()
    with contextlib.redirect_stdout(buf):
        ui._on_text({"text": text})
    out = buf.getvalue()
    assert "✅" in out and "❌" in out and "⚠️" in out  # emoji preserved
    rows = [ln for ln in out.splitlines() if ln.strip() and ln.lstrip()[0] in "┌│├└"]
    assert len({_disp_width(r) for r in rows}) == 1  # all rows same display width


def test_checks_list(capsys):
    assert main(["checks", "list"]) == 0
    out = capsys.readouterr().out
    # Grouped by validation type, user-friendly (no internal ids), with the
    # LLM-as-a-judge column header.
    assert "[1. Structural]" in out and "[6. Governance]" in out
    assert "LLM?" in out
    assert "bundle-schema" not in out  # internal id is hidden from users
    assert "Bundle schema" in out      # the friendly title shows (may wrap)


def test_checks_list_json_carries_type_and_llm(capsys):
    assert main(["checks", "list", "--json"]) == 0
    import json as _json
    rows = _json.loads(capsys.readouterr().out)
    r = next(x for x in rows if x["id"] == "bundle-schema")
    assert r["type"] == "1. Structural" and r["llm_judged"] is False
    assert r["how"] and r["citation_url"]


def test_demo_then_validate(tmp_path, capsys):
    out_dir = tmp_path / "demo-out"
    assert main(["demo", "--out", str(out_dir)]) == 0
    out = capsys.readouterr().out
    assert "no LLM, no keys" in out
    assert "Bundle validation — ✅ PASS" in out

    assert main(["validate", str(out_dir / DEMO_SLUG), "--no-execute"]) == 0
    assert "Bundle validation" in capsys.readouterr().out


def test_validate_json_output(demo_bundle, capsys):
    assert main(["validate", str(demo_bundle), "--no-execute", "--json"]) == 0
    out = capsys.readouterr().out
    assert '"ok": true' in out


def test_validate_exit_code_on_gate_failure(demo_bundle, capsys):
    (demo_bundle / "pages" / "terms.md").unlink()
    assert main(["validate", str(demo_bundle), "--no-execute"]) == 1


def test_version(capsys):
    with pytest.raises(SystemExit) as exc:
        main(["--version"])
    assert exc.value.code == 0


def test_create_requires_idea_or_pdf(capsys):
    # Neither an idea nor --pdf → guard returns 2 before any backend/auth touch.
    assert main(["create"]) == 2
    assert "idea" in capsys.readouterr().err.lower()


def test_create_pdf_must_exist(capsys):
    assert main(["create", "--pdf", "/no/such/proposal.pdf"]) == 2
    assert "not a file" in capsys.readouterr().err.lower()


def test_plan_requires_idea_or_pdf(capsys):
    assert main(["plan"]) == 2
    assert "idea" in capsys.readouterr().err.lower()


def test_plan_pdf_must_exist(capsys):
    assert main(["plan", "--pdf", "/no/such/proposal.pdf"]) == 2
    assert "not a file" in capsys.readouterr().err.lower()


# --- docker preflight banner ----------------------------------------------

import autocodabench.runner as _runner
from autocodabench.cli.main import _bundle_declared_image, _print_docker_preflight


def _fake_preflight(**over):
    base = {
        "host_arch": "arm64", "host_os": "Darwin",
        "docker": {"cli_installed": True, "daemon_running": True,
                   "os": "linux", "arch": "arm64", "server_version": "27.0"},
        "image": "codalab/codalab-legacy:py312",
        "image_present_locally": True, "image_arch": "arm64",
        "image_available_arches": ["arm64"], "image_multi_arch": False,
        "image_source": "local image", "image_error": None,
        "runs_natively": True, "emulated": False, "ready": True,
    }
    base.update(over)
    return base


def test_preflight_banner_native(monkeypatch, capsys):
    monkeypatch.setattr(_runner, "docker_preflight", lambda image: _fake_preflight())
    _print_docker_preflight("codalab/codalab-legacy:py312", required=False)
    out = capsys.readouterr().out
    assert "Docker runtime" in out
    assert "runs natively" in out
    assert "arm64 (Darwin)" in out


def test_preflight_banner_emulated_warns(monkeypatch, capsys):
    monkeypatch.delenv("AUTOCODABENCH_ALLOW_EMULATION", raising=False)
    monkeypatch.setattr(_runner, "docker_preflight", lambda image: _fake_preflight(
        image="codalab/codalab-legacy:py39", image_arch="amd64",
        image_available_arches=["amd64"], image_present_locally=False,
        image_source="remote manifest", runs_natively=False, emulated=True))
    _print_docker_preflight("codalab/codalab-legacy:py39", required=True)
    out = capsys.readouterr().out
    assert "QEMU emulation" in out
    # When execution is required but the image emulates, the banner flags the slow
    # emulated run and recommends the OVERRIDE env (which wins over a declared
    # image) or --no-execute. (validate then prompts the user interactively.)
    assert "would run under slow emulation" in out
    assert "AUTOCODABENCH_DOCKER_IMAGE_OVERRIDE=codalab/codalab-legacy:py312" in out
    assert "--no-execute" in out


def test_preflight_banner_no_daemon_warns_when_required(monkeypatch, capsys):
    monkeypatch.setattr(_runner, "docker_preflight", lambda image: _fake_preflight(
        docker={"cli_installed": False, "daemon_running": False,
                "os": None, "arch": None, "server_version": None},
        ready=False, runs_natively=None, emulated=None, image_arch=None,
        image_available_arches=[], image_error="Docker daemon not running"))
    _print_docker_preflight("any/img:1", required=True)
    cap = capsys.readouterr()
    assert "Docker is not installed" in cap.out
    assert "WARNING:" in cap.err  # loud only when the run requires Docker


def test_bundle_declared_image_from_dir(demo_bundle):
    # The shipped demo declares a codalab image in competition.yaml.
    img = _bundle_declared_image(demo_bundle)
    assert img and "codalab" in img


def test_bundle_declared_image_from_zip(tmp_path):
    import zipfile
    z = tmp_path / "b.zip"
    with zipfile.ZipFile(z, "w") as zf:
        zf.writestr("competition.yaml", "title: t\ndocker_image: myorg/img:9\n")
    assert _bundle_declared_image(z) == "myorg/img:9"


def test_bundle_declared_image_none_when_absent(tmp_path):
    (tmp_path / "competition.yaml").write_text("title: t\n")
    assert _bundle_declared_image(tmp_path) is None
# ---------------------------------------------------------------------------
# build argument guards (keyless: rejected before any live-auth probe
# or backend call, so these run without credentials)
# ---------------------------------------------------------------------------

def test_build_requires_a_plan_source(capsys):
    assert main(["build", "--yes"]) == 2
    assert "plan file" in capsys.readouterr().err


def test_build_rejects_both_sources(tmp_path, capsys):
    plan = tmp_path / "plan.md"
    plan.write_text("# plan", encoding="utf-8")
    code = main(["build", str(plan), "--run-dir", str(tmp_path), "--yes"])
    assert code == 2
    assert "not both" in capsys.readouterr().err


def test_build_missing_run_dir(tmp_path, capsys):
    missing = tmp_path / "nope"
    assert main(["build", "--run-dir", str(missing), "--yes"]) == 2
    assert "run dir not found" in capsys.readouterr().err


def test_build_run_dir_without_plan(tmp_path, capsys):
    (tmp_path / "specs").mkdir()
    assert main(["build", "--run-dir", str(tmp_path), "--yes"]) == 2
    assert "implementation_plan.md" in capsys.readouterr().err


def test_build_missing_plan_file(tmp_path, capsys):
    assert main(["build", str(tmp_path / "absent.md"), "--yes"]) == 2
    assert "plan file not found" in capsys.readouterr().err