"""CLI surface tests — keyless paths only.""" import pytest from autocodabench.cli.main import main, _make_progress_renderer from conftest import DEMO_SLUG # --- create progress renderer (default vs --debug) ------------------------- _EVENTS = [ {"kind": "phase", "index": 2, "total": 3, "title": "Building the bundle", "detail": "writing files; linting; zipping"}, {"kind": "tool_use", "name": "mcp__autocodabench__autocodabench_init_bundle", "input": {"slug": "create"}}, {"kind": "tool_use", "name": "mcp__autocodabench__autocodabench_log_event", "input": {"kind": "progress", "message": "Wrote the scoring program."}}, {"kind": "tool_result", "is_error": True, "preview": "TypeError: unexpected keyword argument 'multi_class'"}, {"kind": "tool_result", "is_error": True, "preview": "Cancelled: parallel tool call Bash(...) errored"}, {"kind": "tool_use", "name": "mcp__autocodabench__autocodabench_log_event", "input": {"kind": "deviation", "message": "Removed multi_class; acc 0.92."}}, {"kind": "text", "text": "Let me wire the scoring program next."}, {"kind": "phase_done", "phase": "build", "ok": True, "num_turns": 23}, ] def _render(events, *, debug): import io import contextlib buf = io.StringIO() r = _make_progress_renderer(debug=debug) with contextlib.redirect_stdout(buf): for e in events: r(e) return buf.getvalue() def test_default_renderer_is_user_oriented(): out = _render(_EVENTS, debug=False) # User-facing milestone + deviation messages are shown… assert "Wrote the scoring program." in out assert "Removed multi_class; acc 0.92." in out # …tool calls are rendered as friendly actions (not raw tool ids)… assert "Init bundle create" in out assert "init_bundle" not in out # …the agent's narration is shown (it is the user-friendly story)… assert "Let me wire the scoring program next." in out # …but raw errors and benign parallel cancellations are suppressed. assert "TypeError" not in out assert "Cancelled" not in out def test_debug_renderer_shows_full_trace_and_softens_cancellations(): out = _render(_EVENTS, debug=True) assert "Init bundle create" in out # friendly action… assert "init_bundle" in out # …with the raw id kept greppable assert "TypeError" in out # genuine error shown assert "Let me wire the scoring program next." in out # narration shown assert "Cancelled" not in out # cascade is reworded… assert "retried" in out # …as a benign retry # --- markdown-table rendering in narration (CLI-only) ---------------------- _TABLE_TEXT = ( "Plan ready. Summary:\n\n" "| Section | Decision |\n" "|---|---|\n" "| **Task** | 3-class image classification; **result-submission** |\n" "| **Metric** | `scipy.stats.gmean(per_group_accuracies)` |\n\n" "Phase 2 will now run." ) def test_narration_table_is_rendered_as_aligned_box(): out = _render([{"kind": "text", "text": _TABLE_TEXT}], debug=False) # Drawn as a box with aligned borders… assert "┌" in out and "┴" in out and "│" in out # …header and cell content preserved, emphasis/backtick markers stripped… assert "Section" in out and "Decision" in out assert "scipy.stats.gmean(per_group_accuracies)" in out assert "**Task**" not in out and "`scipy" not in out # …and the markdown separator row is not echoed as raw pipes. assert "|---|" not in out # Surrounding prose is still shown. assert "Plan ready. Summary:" in out assert "Phase 2 will now run." in out def test_table_detection_helpers(): from autocodabench.cli.progress import ( _has_md_table, _is_table_separator, _table_cells, _strip_md) assert _has_md_table(_TABLE_TEXT) assert not _has_md_table("just prose | with a stray pipe") assert _is_table_separator("|---|:--:|") assert not _is_table_separator("| a | b |") assert _table_cells("| a | b |") == ["a", "b"] assert _table_cells("no pipes here") is None assert _strip_md("**bold** and `code`") == "bold and code" def test_status_glyphs_are_colorized_on_tty(): from autocodabench.cli.progress import _colorize_glyphs, _GREEN, _YELLOW, _RED out = _colorize_glyphs("✓ ⚠ ✗") assert _GREEN in out and _YELLOW in out and _RED in out # Non-glyph text is untouched. assert _colorize_glyphs("plain") == "plain" def test_provenance_table_renders_with_glyphs(): from autocodabench.cli.progress import ProgressUI, _GREEN, _RED import io, contextlib text = ( "| # | Dimension | Status | Origin |\n" "|---|---|---|---|\n" "| 1 | Task | ✓ | proposal §2 |\n" "| 3 | Metric | ✗ | inferred |\n" ) ui = ProgressUI(debug=False) ui.animate = True # force ANSI so glyph colors are emitted buf = io.StringIO() with contextlib.redirect_stdout(buf): ui._on_text({"text": text}) out = buf.getvalue() assert "┌" in out and "Dimension" in out # drawn as a box assert _GREEN in out and _RED in out # ✓ green, ✗ red assert "|---|" not in out # separator not echoed def test_disp_width_counts_emoji_as_two(): from autocodabench.cli.progress import _disp_width, _pad # ✅/❌ are East-Asian "Wide"; ⚠️ is base glyph + a U+FE0F selector — all 2. assert _disp_width("✅") == 2 assert _disp_width("❌") == 2 assert _disp_width("⚠️") == 2 assert _disp_width("✓") == 1 and _disp_width("Status") == 6 # Emoji-aware ljust pads to the right *display* width. assert _disp_width(_pad("✅", 6)) == 6 assert _disp_width(_pad("⚠️", 6)) == 6 def test_provenance_table_with_emoji_aligns(): """The provenance table uses colour emoji (✅/⚠️/❌); the CLI box must still align — every border/content row renders to the same display width.""" from autocodabench.cli.progress import ProgressUI, _disp_width import io, contextlib text = ( "| # | Dimension | Status | Origin |\n" "|---|---|---|---|\n" "| 1 | Task | ✅ | proposal §2 |\n" "| 2 | Metric | ❌ | inferred |\n" "| 3 | Data | ⚠️ | partial |\n" ) ui = ProgressUI(debug=False) ui.animate = False # plain output so widths are measurable buf = io.StringIO() with contextlib.redirect_stdout(buf): ui._on_text({"text": text}) out = buf.getvalue() assert "✅" in out and "❌" in out and "⚠️" in out # emoji preserved rows = [ln for ln in out.splitlines() if ln.strip() and ln.lstrip()[0] in "┌│├└"] assert len({_disp_width(r) for r in rows}) == 1 # all rows same display width def test_checks_list(capsys): assert main(["checks", "list"]) == 0 out = capsys.readouterr().out # Grouped by validation type, user-friendly (no internal ids), with the # LLM-as-a-judge column header. assert "[1. Structural]" in out and "[6. Governance]" in out assert "LLM?" in out assert "bundle-schema" not in out # internal id is hidden from users assert "Bundle schema" in out # the friendly title shows (may wrap) def test_checks_list_json_carries_type_and_llm(capsys): assert main(["checks", "list", "--json"]) == 0 import json as _json rows = _json.loads(capsys.readouterr().out) r = next(x for x in rows if x["id"] == "bundle-schema") assert r["type"] == "1. Structural" and r["llm_judged"] is False assert r["how"] and r["citation_url"] def test_demo_then_validate(tmp_path, capsys): out_dir = tmp_path / "demo-out" assert main(["demo", "--out", str(out_dir)]) == 0 out = capsys.readouterr().out assert "no LLM, no keys" in out assert "Bundle validation — ✅ PASS" in out assert main(["validate", str(out_dir / DEMO_SLUG), "--no-execute"]) == 0 assert "Bundle validation" in capsys.readouterr().out def test_validate_json_output(demo_bundle, capsys): assert main(["validate", str(demo_bundle), "--no-execute", "--json"]) == 0 out = capsys.readouterr().out assert '"ok": true' in out def test_validate_exit_code_on_gate_failure(demo_bundle, capsys): (demo_bundle / "pages" / "terms.md").unlink() assert main(["validate", str(demo_bundle), "--no-execute"]) == 1 def test_version(capsys): with pytest.raises(SystemExit) as exc: main(["--version"]) assert exc.value.code == 0 def test_create_requires_idea_or_pdf(capsys): # Neither an idea nor --pdf → guard returns 2 before any backend/auth touch. assert main(["create"]) == 2 assert "idea" in capsys.readouterr().err.lower() def test_create_pdf_must_exist(capsys): assert main(["create", "--pdf", "/no/such/proposal.pdf"]) == 2 assert "not a file" in capsys.readouterr().err.lower() def test_plan_requires_idea_or_pdf(capsys): assert main(["plan"]) == 2 assert "idea" in capsys.readouterr().err.lower() def test_plan_pdf_must_exist(capsys): assert main(["plan", "--pdf", "/no/such/proposal.pdf"]) == 2 assert "not a file" in capsys.readouterr().err.lower() # --- docker preflight banner ---------------------------------------------- import autocodabench.runner as _runner from autocodabench.cli.main import _bundle_declared_image, _print_docker_preflight def _fake_preflight(**over): base = { "host_arch": "arm64", "host_os": "Darwin", "docker": {"cli_installed": True, "daemon_running": True, "os": "linux", "arch": "arm64", "server_version": "27.0"}, "image": "codalab/codalab-legacy:py312", "image_present_locally": True, "image_arch": "arm64", "image_available_arches": ["arm64"], "image_multi_arch": False, "image_source": "local image", "image_error": None, "runs_natively": True, "emulated": False, "ready": True, } base.update(over) return base def test_preflight_banner_native(monkeypatch, capsys): monkeypatch.setattr(_runner, "docker_preflight", lambda image: _fake_preflight()) _print_docker_preflight("codalab/codalab-legacy:py312", required=False) out = capsys.readouterr().out assert "Docker runtime" in out assert "runs natively" in out assert "arm64 (Darwin)" in out def test_preflight_banner_emulated_warns(monkeypatch, capsys): monkeypatch.delenv("AUTOCODABENCH_ALLOW_EMULATION", raising=False) monkeypatch.setattr(_runner, "docker_preflight", lambda image: _fake_preflight( image="codalab/codalab-legacy:py39", image_arch="amd64", image_available_arches=["amd64"], image_present_locally=False, image_source="remote manifest", runs_natively=False, emulated=True)) _print_docker_preflight("codalab/codalab-legacy:py39", required=True) out = capsys.readouterr().out assert "QEMU emulation" in out # When execution is required but the image emulates, the banner flags the slow # emulated run and recommends the OVERRIDE env (which wins over a declared # image) or --no-execute. (validate then prompts the user interactively.) assert "would run under slow emulation" in out assert "AUTOCODABENCH_DOCKER_IMAGE_OVERRIDE=codalab/codalab-legacy:py312" in out assert "--no-execute" in out def test_preflight_banner_no_daemon_warns_when_required(monkeypatch, capsys): monkeypatch.setattr(_runner, "docker_preflight", lambda image: _fake_preflight( docker={"cli_installed": False, "daemon_running": False, "os": None, "arch": None, "server_version": None}, ready=False, runs_natively=None, emulated=None, image_arch=None, image_available_arches=[], image_error="Docker daemon not running")) _print_docker_preflight("any/img:1", required=True) cap = capsys.readouterr() assert "Docker is not installed" in cap.out assert "WARNING:" in cap.err # loud only when the run requires Docker def test_bundle_declared_image_from_dir(demo_bundle): # The shipped demo declares a codalab image in competition.yaml. img = _bundle_declared_image(demo_bundle) assert img and "codalab" in img def test_bundle_declared_image_from_zip(tmp_path): import zipfile z = tmp_path / "b.zip" with zipfile.ZipFile(z, "w") as zf: zf.writestr("competition.yaml", "title: t\ndocker_image: myorg/img:9\n") assert _bundle_declared_image(z) == "myorg/img:9" def test_bundle_declared_image_none_when_absent(tmp_path): (tmp_path / "competition.yaml").write_text("title: t\n") assert _bundle_declared_image(tmp_path) is None # --------------------------------------------------------------------------- # build argument guards (keyless: rejected before any live-auth probe # or backend call, so these run without credentials) # --------------------------------------------------------------------------- def test_build_requires_a_plan_source(capsys): assert main(["build", "--yes"]) == 2 assert "plan file" in capsys.readouterr().err def test_build_rejects_both_sources(tmp_path, capsys): plan = tmp_path / "plan.md" plan.write_text("# plan", encoding="utf-8") code = main(["build", str(plan), "--run-dir", str(tmp_path), "--yes"]) assert code == 2 assert "not both" in capsys.readouterr().err def test_build_missing_run_dir(tmp_path, capsys): missing = tmp_path / "nope" assert main(["build", "--run-dir", str(missing), "--yes"]) == 2 assert "run dir not found" in capsys.readouterr().err def test_build_run_dir_without_plan(tmp_path, capsys): (tmp_path / "specs").mkdir() assert main(["build", "--run-dir", str(tmp_path), "--yes"]) == 2 assert "implementation_plan.md" in capsys.readouterr().err def test_build_missing_plan_file(tmp_path, capsys): assert main(["build", str(tmp_path / "absent.md"), "--yes"]) == 2 assert "plan file not found" in capsys.readouterr().err