"""Tests for agents/gap_analyzer.py — LLM-based gap detection.""" from __future__ import annotations from unittest.mock import patch, MagicMock from agents.gap_analyzer import ( DetectedGap, GeneratedQuestionCard, GapAnalysisResult, ) class TestModels: def test_detected_gap_creation(self): gap = DetectedGap( category="bolt_pattern", description="Bolt circle diameter not specified", severity="blocking", ) assert gap.category == "bolt_pattern" assert gap.severity == "blocking" def test_generated_question_card_defaults(self): card = GeneratedQuestionCard( category="material", question="What material?", responsible_agent="engineering", suggestions=["Aluminum 6061"], allow_custom=True, ) assert card.agent_name == "" assert card.agent_color == "" def test_gap_analysis_result_no_gaps(self): result = GapAnalysisResult(has_gaps=False, gaps=[], question_cards=[]) assert not result.has_gaps assert result.gaps == [] assert result.question_cards == [] def test_gap_analysis_result_with_gaps(self): result = GapAnalysisResult( has_gaps=True, gaps=[DetectedGap(category="material", description="missing", severity="blocking")], question_cards=[GeneratedQuestionCard( category="material", question="What material?", responsible_agent="engineering", suggestions=[], allow_custom=True, )], ) assert result.has_gaps assert len(result.gaps) == 1 assert len(result.question_cards) == 1 def test_severity_literal_validation(self): """Severity must be one of the allowed values.""" import pytest with pytest.raises(Exception): DetectedGap(category="x", description="y", severity="invalid") def test_question_card_severity_default(self): card = GeneratedQuestionCard( category="material", question="What material?", responsible_agent="engineering", ) assert card.severity == "recommended" def test_question_card_severity_blocking(self): card = GeneratedQuestionCard( category="material", question="What material?", responsible_agent="engineering", severity="blocking", ) assert card.severity == "blocking" def test_question_card_severity_validation(self): import pytest with pytest.raises(Exception): GeneratedQuestionCard( category="x", question="q", responsible_agent="design", severity="invalid", ) from agents.agent_flow import AgentResponse from agents.design_state import DesignState from agents.gap_analyzer import analyze_gaps class TestAnalyzeGaps: def _mock_llm_response(self, result: GapAnalysisResult): """Create a mock LLM that returns a structured result.""" mock_llm_instance = MagicMock() mock_llm_instance.call.return_value = result return mock_llm_instance @patch("agents.gap_analyzer._build_llm") def test_returns_gaps_from_llm(self, mock_build_llm): expected = GapAnalysisResult( has_gaps=True, gaps=[DetectedGap(category="material", description="No material specified", severity="blocking")], question_cards=[GeneratedQuestionCard( category="material", question="What material should the bracket be made from?", responsible_agent="engineering", suggestions=["Aluminum 6061", "Steel 304"], allow_custom=True, )], ) mock_build_llm.return_value = self._mock_llm_response(expected) responses = [AgentResponse.from_agent("design", "I suggest an L-bracket.")] result = analyze_gaps(responses, DesignState(), user_message="design a bracket") assert result.has_gaps assert result.gaps[0].category == "material" assert result.question_cards[0].question == "What material should the bracket be made from?" @patch("agents.gap_analyzer._build_llm") def test_enriches_agent_metadata(self, mock_build_llm): expected = GapAnalysisResult( has_gaps=True, gaps=[DetectedGap(category="machining", description="missing", severity="recommended")], question_cards=[GeneratedQuestionCard( category="machining", question="What machining approach?", responsible_agent="cnc", suggestions=[], allow_custom=True, )], ) mock_build_llm.return_value = self._mock_llm_response(expected) result = analyze_gaps( [AgentResponse.from_agent("design", "A bracket design.")], DesignState(), ) card = result.question_cards[0] assert card.agent_name == "CNC Agent" assert card.agent_color == "#00e676" @patch("agents.gap_analyzer._build_llm") def test_returns_empty_on_no_gaps(self, mock_build_llm): expected = GapAnalysisResult(has_gaps=False, gaps=[], question_cards=[]) mock_build_llm.return_value = self._mock_llm_response(expected) result = analyze_gaps( [AgentResponse.from_agent("design", "Everything looks good.")], DesignState(material="aluminum"), ) assert not result.has_gaps assert result.question_cards == [] @patch("agents.gap_analyzer._build_llm") def test_fallback_on_llm_failure(self, mock_build_llm): mock_llm = MagicMock() mock_llm.call.side_effect = RuntimeError("API timeout") mock_build_llm.return_value = mock_llm result = analyze_gaps( [AgentResponse.from_agent("cad", "NOT READY: need dimensions")], DesignState(), ) assert not result.has_gaps assert result.gaps == [] assert result.question_cards == [] @patch("agents.gap_analyzer._build_llm") def test_prompt_includes_state_and_responses(self, mock_build_llm): mock_llm = MagicMock() mock_llm.call.return_value = GapAnalysisResult(has_gaps=False, gaps=[], question_cards=[]) mock_build_llm.return_value = mock_llm state = DesignState(material="aluminum 6061", part_name="bracket") responses = [AgentResponse.from_agent("engineering", "Looks reasonable.")] analyze_gaps(responses, state, user_message="make it 50mm wide") call_args = mock_llm.call.call_args # First positional arg is the messages list messages = call_args[0][0] prompt_text = str(messages) assert "aluminum 6061" in prompt_text assert "bracket" in prompt_text assert "engineering" in prompt_text assert "Looks reasonable." in prompt_text assert "make it 50mm wide" in prompt_text