tests/hermes_cli/test_custom_provider_model_switch.py

"""Tests that `hermes model` always shows the model selection menu for custom
providers, even when a model is already saved.

Regression test for the bug where _model_flow_named_custom() returned
immediately when provider_info had a saved ``model`` field, making it
impossible to switch models on multi-model endpoints.
"""

import os
from unittest.mock import patch, MagicMock, call

import pytest


@pytest.fixture
def config_home(tmp_path, monkeypatch):
    """Isolated HERMES_HOME with a minimal config."""
    home = tmp_path / "hermes"
    home.mkdir()
    config_yaml = home / "config.yaml"
    config_yaml.write_text("model: old-model\ncustom_providers: []\n")
    env_file = home / ".env"
    env_file.write_text("")
    monkeypatch.setenv("HERMES_HOME", str(home))
    monkeypatch.delenv("HERMES_MODEL", raising=False)
    monkeypatch.delenv("LLM_MODEL", raising=False)
    monkeypatch.delenv("HERMES_INFERENCE_PROVIDER", raising=False)
    monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
    monkeypatch.delenv("OPENAI_API_KEY", raising=False)
    return home


class TestCustomProviderModelSwitch:
    """Ensure _model_flow_named_custom always probes and shows menu."""

    def test_saved_model_still_probes_endpoint(self, config_home):
        """When a model is already saved, the function must still call
        fetch_api_models to probe the endpoint — not skip with early return."""
        from hermes_cli.main import _model_flow_named_custom

        provider_info = {
            "name": "My vLLM",
            "base_url": "https://vllm.example.com/v1",
            "api_key": "sk-test",
            "model": "model-A",  # already saved
        }

        with patch("hermes_cli.models.fetch_api_models", return_value=["model-A", "model-B"]) as mock_fetch, \
             patch.dict("sys.modules", {"simple_term_menu": None}), \
             patch("builtins.input", return_value="2"), \
             patch("builtins.print"):
            _model_flow_named_custom({}, provider_info)

        # fetch_api_models MUST be called even though model was saved
        mock_fetch.assert_called_once_with("sk-test", "https://vllm.example.com/v1", timeout=8.0)

    def test_can_switch_to_different_model(self, config_home):
        """User selects a different model than the saved one."""
        import yaml
        from hermes_cli.main import _model_flow_named_custom

        provider_info = {
            "name": "My vLLM",
            "base_url": "https://vllm.example.com/v1",
            "api_key": "sk-test",
            "model": "model-A",
        }

        with patch("hermes_cli.models.fetch_api_models", return_value=["model-A", "model-B"]), \
             patch.dict("sys.modules", {"simple_term_menu": None}), \
             patch("builtins.input", return_value="2"), \
             patch("builtins.print"):
            _model_flow_named_custom({}, provider_info)

        config = yaml.safe_load((config_home / "config.yaml").read_text()) or {}
        model = config.get("model")
        assert isinstance(model, dict)
        assert model["default"] == "model-B"

    def test_probe_failure_falls_back_to_saved(self, config_home):
        """When endpoint probe fails and user presses Enter, saved model is used."""
        import yaml
        from hermes_cli.main import _model_flow_named_custom

        provider_info = {
            "name": "My vLLM",
            "base_url": "https://vllm.example.com/v1",
            "api_key": "sk-test",
            "model": "model-A",
        }

        # fetch returns empty list (probe failed), user presses Enter (empty input)
        with patch("hermes_cli.models.fetch_api_models", return_value=[]), \
             patch("builtins.input", return_value=""), \
             patch("builtins.print"):
            _model_flow_named_custom({}, provider_info)

        config = yaml.safe_load((config_home / "config.yaml").read_text()) or {}
        model = config.get("model")
        assert isinstance(model, dict)
        assert model["default"] == "model-A"

    def test_no_saved_model_still_works(self, config_home):
        """First-time flow (no saved model) still works as before."""
        import yaml
        from hermes_cli.main import _model_flow_named_custom

        provider_info = {
            "name": "My vLLM",
            "base_url": "https://vllm.example.com/v1",
            "api_key": "sk-test",
            # no "model" key
        }

        with patch("hermes_cli.models.fetch_api_models", return_value=["model-X"]), \
             patch.dict("sys.modules", {"simple_term_menu": None}), \
             patch("builtins.input", return_value="1"), \
             patch("builtins.print"):
            _model_flow_named_custom({}, provider_info)

        config = yaml.safe_load((config_home / "config.yaml").read_text()) or {}
        model = config.get("model")
        assert isinstance(model, dict)
        assert model["default"] == "model-X"

    def test_api_mode_set_from_provider_info(self, config_home):
        """When custom_providers entry has api_mode, it should be applied."""
        import yaml
        from hermes_cli.main import _model_flow_named_custom

        provider_info = {
            "name": "Anthropic Proxy",
            "base_url": "https://proxy.example.com/anthropic",
            "api_key": "***",
            "model": "claude-3",
            "api_mode": "anthropic_messages",
        }

        with patch("hermes_cli.models.fetch_api_models", return_value=["claude-3"]), \
             patch.dict("sys.modules", {"simple_term_menu": None}), \
             patch("builtins.input", return_value="1"), \
             patch("builtins.print"):
            _model_flow_named_custom({}, provider_info)

        config = yaml.safe_load((config_home / "config.yaml").read_text()) or {}
        model = config.get("model")
        assert isinstance(model, dict)
        assert model.get("api_mode") == "anthropic_messages"

    def test_api_mode_cleared_when_not_specified(self, config_home):
        """When custom_providers entry has no api_mode, stale api_mode is removed."""
        import yaml
        from hermes_cli.main import _model_flow_named_custom

        # Pre-seed a stale api_mode in config
        config_path = config_home / "config.yaml"
        config_path.write_text(yaml.dump({"model": {"api_mode": "anthropic_messages"}}))

        provider_info = {
            "name": "My vLLM",
            "base_url": "https://vllm.example.com/v1",
            "api_key": "***",
            "model": "llama-3",
        }

        with patch("hermes_cli.models.fetch_api_models", return_value=["llama-3"]), \
             patch.dict("sys.modules", {"simple_term_menu": None}), \
             patch("builtins.input", return_value="1"), \
             patch("builtins.print"):
            _model_flow_named_custom({}, provider_info)

        config = yaml.safe_load((config_home / "config.yaml").read_text()) or {}
        model = config.get("model")
        assert isinstance(model, dict)
        assert "api_mode" not in model, "Stale api_mode should be removed"