File size: 7,162 Bytes
81e3673 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 | """
Test AdvisorService guidance generation.
Tests cover:
- Guidance generation
- Variant performance analysis
- LLM integration
- Error handling
- Caching
"""
import pytest
from unittest.mock import AsyncMock, MagicMock
from core.auto_dev.advisor_service import AdvisorService
class TestAdvisorServiceGuidanceGeneration:
"""Test generate_guidance() creates human-readable advice."""
@pytest.mark.asyncio
async def test_generate_guidance_calls_llm(self, mock_auto_dev_llm, auto_dev_db_session):
"""Test generate_guidance() calls LLM."""
service = AdvisorService(db=auto_dev_db_session, llm_service=mock_auto_dev_llm)
result = await service.generate_guidance(tenant_id="tenant-001")
assert "status" in result
assert result["status"] == "success"
assert "message" in result
@pytest.mark.asyncio
async def test_includes_variant_metadata(self, mock_auto_dev_llm, auto_dev_db_session, sample_workflow_variant):
"""Test includes variant metadata."""
service = AdvisorService(db=auto_dev_db_session, llm_service=mock_auto_dev_llm)
result = await service.generate_guidance(tenant_id=sample_workflow_variant.tenant_id)
assert "data_summary" in result
@pytest.mark.asyncio
async def test_includes_fitness_signals(self, mock_auto_dev_llm, auto_dev_db_session, sample_workflow_variant):
"""Test includes fitness signals."""
service = AdvisorService(db=auto_dev_db_session, llm_service=mock_auto_dev_llm)
result = await service.generate_guidance(tenant_id=sample_workflow_variant.tenant_id)
summary = result["data_summary"]
assert "top_fitness_score" in summary
@pytest.mark.asyncio
async def test_returns_human_readable_text(self, mock_auto_dev_llm, auto_dev_db_session):
"""Test returns human-readable text."""
service = AdvisorService(db=auto_dev_db_session, llm_service=mock_auto_dev_llm)
result = await service.generate_guidance(tenant_id="tenant-001")
assert isinstance(result["message"], str)
assert len(result["message"]) > 0
@pytest.mark.asyncio
async def test_handles_llm_errors(self, auto_dev_db_session):
"""Test handles LLM errors."""
# Mock LLM that fails
mock_llm = MagicMock()
mock_llm.generate_completion = AsyncMock(side_effect=Exception("LLM error"))
service = AdvisorService(db=auto_dev_db_session, llm_service=mock_llm)
result = await service.generate_guidance(tenant_id="tenant-001")
# Should fall back to heuristic guidance
assert result["status"] == "success"
assert "message" in result
class TestAdvisorServiceVariantAnalysis:
"""Test analyze_variant_performance() provides detailed analysis."""
@pytest.mark.asyncio
async def test_compares_variants(self, mock_auto_dev_llm, auto_dev_db_session, sample_workflow_variant):
"""Test compares variants."""
service = AdvisorService(db=auto_dev_db_session, llm_service=mock_auto_dev_llm)
result = await service.generate_guidance(tenant_id=sample_workflow_variant.tenant_id)
assert "data_summary" in result
@pytest.mark.asyncio
async def test_identifies_strengths_weaknesses(self, mock_auto_dev_llm, auto_dev_db_session, sample_workflow_variant):
"""Test identifies strengths/weaknesses."""
service = AdvisorService(db=auto_dev_db_session, llm_service=mock_auto_dev_llm)
result = await service.generate_guidance(tenant_id=sample_workflow_variant.tenant_id)
# Message should contain guidance (mock returns 36 chars, heuristic returns more)
assert len(result["message"]) > 10
class TestAdvisorServiceLLMIntegration:
"""Test uses LLMService for guidance."""
@pytest.mark.asyncio
async def test_uses_llm_service(self, mock_auto_dev_llm, auto_dev_db_session, sample_workflow_variant):
"""Test uses LLMService."""
service = AdvisorService(db=auto_dev_db_session, llm_service=mock_auto_dev_llm)
await service.generate_guidance(tenant_id=sample_workflow_variant.tenant_id)
# Should have called LLM
assert mock_auto_dev_llm.generate_completion.called
@pytest.mark.asyncio
async def test_passes_context_correctly(self, mock_auto_dev_llm, auto_dev_db_session, sample_workflow_variant):
"""Test passes context correctly."""
service = AdvisorService(db=auto_dev_db_session, llm_service=mock_auto_dev_llm)
await service.generate_guidance(tenant_id=sample_workflow_variant.tenant_id)
# Check call arguments
call_args = mock_auto_dev_llm.generate_completion.call_args
assert "system_prompt" in call_args[1]
assert "user_prompt" in call_args[1]
class TestAdvisorServiceErrorHandling:
"""Test handles missing data gracefully."""
@pytest.mark.asyncio
async def test_handles_missing_variant_data(self, auto_dev_db_session):
"""Test handles missing variant data."""
service = AdvisorService(db=auto_dev_db_session, llm_service=None)
result = await service.generate_guidance(tenant_id="nonexistent-tenant")
# Should still return success
assert result["status"] == "success"
@pytest.mark.asyncio
async def test_returns_safe_default_guidance(self, auto_dev_db_session):
"""Test returns safe default guidance."""
service = AdvisorService(db=auto_dev_db_session, llm_service=None)
result = await service.generate_guidance(tenant_id="tenant-001")
assert "message" in result
assert len(result["message"]) > 0
class TestAdvisorServiceReadinessScore:
"""Test readiness score calculation."""
@pytest.mark.asyncio
async def test_calculates_readiness_score(self, mock_auto_dev_llm, auto_dev_db_session, sample_workflow_variant):
"""Test calculates readiness score."""
service = AdvisorService(db=auto_dev_db_session, llm_service=mock_auto_dev_llm)
result = await service.generate_guidance(tenant_id=sample_workflow_variant.tenant_id)
assert "readiness_score" in result
assert 0 <= result["readiness_score"] <= 100
@pytest.mark.asyncio
async def test_readiness_increases_with_mutations(self, mock_auto_dev_llm, auto_dev_db_session, sample_workflow_variant):
"""Test readiness increases with passed mutations."""
from core.auto_dev.models import ToolMutation
# Create some mutations
for i in range(5):
mutation = ToolMutation(
tenant_id=sample_workflow_variant.tenant_id,
tool_name=f"tool_{i}",
mutated_code="def test(): pass",
sandbox_status="passed" if i < 3 else "failed",
)
auto_dev_db_session.add(mutation)
auto_dev_db_session.commit()
service = AdvisorService(db=auto_dev_db_session, llm_service=mock_auto_dev_llm)
result = await service.generate_guidance(tenant_id=sample_workflow_variant.tenant_id)
# Readiness should be > 0
assert result["readiness_score"] > 0
|