"""Tests for the CLI entrypoint (``multitool ask ""``). These tests never make real network calls — the ``_run_agent`` seam is monkeypatched in every test that exercises the orchestrator path. The argparse-error tests (missing question, unknown subcommand) don't even reach ``_run_agent``, so they need no mock. """ import pytest from multitool import cli from multitool.orchestrator import AgentResult def _fake_result( answer: str = "42", steps_taken: int = 1, tool_calls: list[dict] | None = None, error: str | None = None, trace_path: str = "traces/abc123.json", ) -> AgentResult: """Construct a real AgentResult with sensible defaults — using the actual dataclass (not a MagicMock) keeps tests honest about the shape of what the orchestrator returns.""" return AgentResult( answer=answer, steps_taken=steps_taken, tool_calls=tool_calls if tool_calls is not None else [], error=error, trace_path=trace_path, ) class TestCliArgparseErrors: """Tests for cases where argparse rejects input before we touch the agent. These should always exit with code 2 (argparse's convention).""" def test_ask_missing_question_exits_with_error(self): """`multitool ask` with no question is an argparse usage error.""" with pytest.raises(SystemExit) as exc_info: cli.main(["ask"]) assert exc_info.value.code == 2 def test_unknown_subcommand_shows_help(self, capsys): """`multitool bogus` is an argparse usage error → exit 2, usage on stderr.""" with pytest.raises(SystemExit) as exc_info: cli.main(["bogus"]) assert exc_info.value.code == 2 captured = capsys.readouterr() # argparse prints usage to stderr on parse errors assert "usage" in captured.err.lower() or "invalid choice" in captured.err.lower() class TestCliRuntimeErrors: """Tests for cases where the parser succeeds but execution fails — these should exit with code 1 (NOT 2) and print a friendly message.""" def test_ask_missing_api_key_exits_with_friendly_error( self, monkeypatch, capsys ): """Missing GROQ_API_KEY → friendly stderr message + exit 1, NOT a raw KeyError traceback.""" monkeypatch.delenv("GROQ_API_KEY", raising=False) rc = cli.main(["ask", "what is 2+2?"]) assert rc == 1 captured = capsys.readouterr() # Stderr should mention the missing var by name and be actionable. assert "GROQ_API_KEY" in captured.err # Verify it's a FRIENDLY message — no traceback markers should leak. assert "Traceback" not in captured.err assert "KeyError" not in captured.err def test_orchestrator_exception_prints_friendly_error( self, monkeypatch, capsys ): """If _run_agent raises (e.g. network failure), the user sees a clean stderr message + exit 1 — not a traceback (unless --verbose).""" def boom(*args, **kwargs): raise ConnectionError("groq is down") monkeypatch.setattr(cli, "_run_agent", boom) rc = cli.main(["ask", "anything"]) assert rc == 1 captured = capsys.readouterr() assert "ConnectionError" in captured.err assert "groq is down" in captured.err assert "Traceback" not in captured.err def test_verbose_flag_propagates_orchestrator_exception( self, monkeypatch ): """With --verbose, the original exception must be re-raised so the user sees the full traceback for debugging.""" def boom(*args, **kwargs): raise ConnectionError("groq is down") monkeypatch.setattr(cli, "_run_agent", boom) with pytest.raises(ConnectionError, match="groq is down"): cli.main(["ask", "anything", "--verbose"]) class TestCliHappyPath: """Tests covering the success path: orchestrator returns a final answer; CLI prints it to stdout and exits 0.""" def test_ask_calls_orchestrator_and_prints_answer( self, monkeypatch, capsys ): """Stdout should contain ONLY the answer (no decoration).""" monkeypatch.setattr( cli, "_run_agent", lambda **kwargs: _fake_result(answer="42"), ) rc = cli.main(["ask", "what is the meaning of life?"]) assert rc == 0 captured = capsys.readouterr() assert "42" in captured.out # Bare stdout — no [trace: ...] decoration unless --verbose assert "trace" not in captured.out.lower() assert "steps" not in captured.out.lower() def test_ask_forwards_args_to_run_agent(self, monkeypatch): """CLI flags should reach _run_agent unmangled — covers wiring of --provider, --model, --trace.""" captured_kwargs: dict = {} def capture(**kwargs): captured_kwargs.update(kwargs) return _fake_result(answer="ok") monkeypatch.setattr(cli, "_run_agent", capture) rc = cli.main([ "ask", "hello", "--provider", "gemini", "--model", "gemini-pro", "--trace", "/tmp/run.json", ]) assert rc == 0 assert captured_kwargs["question"] == "hello" assert captured_kwargs["provider"] == "gemini" assert captured_kwargs["model"] == "gemini-pro" assert captured_kwargs["trace_path"] == "/tmp/run.json" class TestCliVerboseTrace: """Tests covering --verbose tool-call logging.""" def test_verbose_flag_prints_tool_calls(self, monkeypatch, capsys): """`-v` should print each tool invocation to STDERR (stderr keeps stdout clean for piping the answer).""" result = _fake_result( answer="5", tool_calls=[ {"name": "calculator", "args": {"expression": "2+3"}, "result": "5"}, ], ) monkeypatch.setattr(cli, "_run_agent", lambda **kwargs: result) rc = cli.main(["ask", "what is 2+3?", "-v"]) assert rc == 0 captured = capsys.readouterr() # Answer on stdout assert "5" in captured.out # Tool call name on stderr assert "calculator" in captured.err assert "2+3" in captured.err def test_verbose_truncates_long_tool_results(self, monkeypatch, capsys): """A 5000-char Wikipedia summary shouldn't flood the terminal.""" long_result = "x" * 5000 result = _fake_result( answer="ok", tool_calls=[ {"name": "wikipedia", "args": {"topic": "Python"}, "result": long_result}, ], ) monkeypatch.setattr(cli, "_run_agent", lambda **kwargs: result) cli.main(["ask", "tell me about python", "-v"]) captured = capsys.readouterr() # Truncation marker should appear; full 5000-char string should not. assert "chars]" in captured.err assert long_result not in captured.err class TestResolveTraceDir: """Verify --trace PATH is treated as a directory (created if missing). Regression guard for the foot-gun where ``--trace mytrace`` (no slash, no extension) was passed through ``os.path.dirname("mytrace") == ""`` and silently dropped traces into CWD instead of ``mytrace/``.""" def test_trace_path_is_treated_as_directory_and_created(self, tmp_path): """--trace /path/to/newdir creates newdir and returns it as-is.""" target = tmp_path / "newdir" assert not target.exists() resolved = cli._resolve_trace_dir(str(target)) assert resolved == str(target) assert target.is_dir() def test_bare_name_trace_path_creates_directory_relative_to_cwd( self, tmp_path, monkeypatch ): """--trace mytrace (no slash) must create ``mytrace/`` relative to CWD, NOT silently fall back to CWD itself (the original foot-gun).""" monkeypatch.chdir(tmp_path) resolved = cli._resolve_trace_dir("mytrace") assert resolved == "mytrace" assert (tmp_path / "mytrace").is_dir() # The CWD itself must NOT be the resolved directory. assert resolved != "." def test_no_trace_path_defaults_to_traces_dir(self, tmp_path, monkeypatch): """No --trace flag → default ``traces/`` directory, created if missing.""" monkeypatch.chdir(tmp_path) resolved = cli._resolve_trace_dir(None) assert resolved == "traces" assert (tmp_path / "traces").is_dir() def test_existing_trace_dir_is_idempotent(self, tmp_path): """Re-resolving an existing dir does not error (exist_ok semantics).""" target = tmp_path / "already_here" target.mkdir() resolved = cli._resolve_trace_dir(str(target)) assert resolved == str(target) assert target.is_dir() class TestCliMaxStepsReached: """Tests for the max-steps-reached failure path.""" def test_max_steps_prints_warning_to_stderr_and_exits_nonzero( self, monkeypatch, capsys ): """When orchestrator returns max_steps_reached, CLI prints warning to stderr and exits 1 (no partial answer is surfaced — the orchestrator hard-codes answer=None in this branch, per orchestrator.py:127-133).""" result = _fake_result( answer=None, steps_taken=10, error="max_steps_reached", trace_path="traces/abc.json", ) monkeypatch.setattr(cli, "_run_agent", lambda **kwargs: result) rc = cli.main(["ask", "hard question"]) assert rc == 1 captured = capsys.readouterr() assert captured.out == "" # no partial answer surfaced assert "max steps" in captured.err.lower() # warning on stderr assert "traces/abc.json" in captured.err # trace path included for debug