File size: 8,728 Bytes
81e3673 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 | """
Test EvolutionEngine monitoring and optimization.
Tests cover:
- Monitoring variant performance
- Detecting trigger conditions
- Triggering background optimization
- Pruning low-fitness variants
- Publishing evolution events
"""
import pytest
from unittest.mock import AsyncMock, MagicMock
from core.auto_dev.evolution_engine import EvolutionEngine
from core.auto_dev.event_hooks import SkillExecutionEvent
class TestEvolutionEngineMonitoring:
"""Test EvolutionEngine monitors variant performance."""
def test_monitors_workflow_variants(self, auto_dev_db_session, sample_workflow_variant):
"""Test monitors WorkflowVariant performance."""
engine = EvolutionEngine(db=auto_dev_db_session)
# Should initialize without errors
assert engine.db == auto_dev_db_session
class TestEvolutionEngineTriggerDetection:
"""Test detects trigger conditions (failure rate, latency)."""
@pytest.mark.asyncio
async def test_triggers_on_high_latency(self, auto_dev_db_session, sample_skill_execution_event, monkeypatch):
"""Test triggers on high latency (>5s)."""
engine = EvolutionEngine(db=auto_dev_db_session)
# Mock capability gate to return True (bypass AUTONOMOUS check)
monkeypatch.setattr(engine, "_should_optimize", lambda agent_id, tenant_id: True)
# Mock _get_skill_code to return test code
monkeypatch.setattr(engine, "_get_skill_code", lambda skill_id, tenant_id: "def test_skill():\n pass")
# Mock AlphaEvolverEngine
mock_evolver = MagicMock()
mock_mutation = MagicMock()
mock_mutation.id = "mutation-001"
mock_evolver.generate_tool_mutation = AsyncMock(return_value=mock_mutation)
mock_evolver.sandbox_execute_mutation = AsyncMock(return_value={"success": True})
import sys
original_module = sys.modules.get("core.auto_dev.alpha_evolver_engine")
class MockAlphaEvolverEngine:
def __init__(self, db):
pass
sys.modules["core.auto_dev.alpha_evolver_engine"] = MockAlphaEvolverEngine
sys.modules["core.auto_dev.alpha_evolver_engine"].AlphaEvolverEngine = lambda db: mock_evolver
try:
# Create high-latency event
high_latency_event = SkillExecutionEvent(
execution_id="exec-001",
agent_id="agent-001",
tenant_id="tenant-001",
skill_id="skill-001",
skill_name="test_skill",
execution_seconds=10.0, # High latency > 5s threshold
token_usage=1000,
success=True,
)
await engine.process_execution(high_latency_event)
# Should trigger optimization
mock_evolver.generate_tool_mutation.assert_called_once()
finally:
if original_module:
sys.modules["core.auto_dev.alpha_evolver_engine"] = original_module
else:
sys.modules.pop("core.auto_dev.alpha_evolver_engine", None)
@pytest.mark.asyncio
async def test_triggers_on_high_token_usage(self, auto_dev_db_session, monkeypatch):
"""Test triggers on high token usage."""
engine = EvolutionEngine(db=auto_dev_db_session)
# Mock capability gate to return True (bypass AUTONOMOUS check)
monkeypatch.setattr(engine, "_should_optimize", lambda agent_id, tenant_id: True)
# Mock _get_skill_code to return test code
monkeypatch.setattr(engine, "_get_skill_code", lambda skill_id, tenant_id: "def test_skill():\n pass")
mock_evolver = MagicMock()
mock_mutation = MagicMock()
mock_mutation.id = "mutation-001"
mock_evolver.generate_tool_mutation = AsyncMock(return_value=mock_mutation)
mock_evolver.sandbox_execute_mutation = AsyncMock(return_value={"success": True})
import sys
original_module = sys.modules.get("core.auto_dev.alpha_evolver_engine")
class MockAlphaEvolverEngine:
def __init__(self, db):
pass
sys.modules["core.auto_dev.alpha_evolver_engine"] = MockAlphaEvolverEngine
sys.modules["core.auto_dev.alpha_evolver_engine"].AlphaEvolverEngine = lambda db: mock_evolver
try:
high_token_event = SkillExecutionEvent(
execution_id="exec-002",
agent_id="agent-001",
tenant_id="tenant-001",
skill_id="skill-001",
skill_name="test_skill",
execution_seconds=2.0,
token_usage=10000, # High token usage > 5000 threshold
success=True,
)
await engine.process_execution(high_token_event)
mock_evolver.generate_tool_mutation.assert_called_once()
finally:
if original_module:
sys.modules["core.auto_dev.alpha_evolver_engine"] = original_module
else:
sys.modules.pop("core.auto_dev.alpha_evolver_engine", None)
class TestEvolutionEngineBackgroundOptimization:
"""Test triggers background optimization."""
@pytest.mark.asyncio
async def test_spawns_alpha_evolver(self, auto_dev_db_session, monkeypatch):
"""Test spawns AlphaEvolverEngine."""
engine = EvolutionEngine(db=auto_dev_db_session)
# Mock capability gate to return True (bypass AUTONOMOUS check)
monkeypatch.setattr(engine, "_should_optimize", lambda agent_id, tenant_id: True)
# Mock _get_skill_code to return test code
monkeypatch.setattr(engine, "_get_skill_code", lambda skill_id, tenant_id: "def test_skill():\n pass")
mock_evolver = MagicMock()
mock_mutation = MagicMock()
mock_mutation.id = "mutation-001"
mock_evolver.generate_tool_mutation = AsyncMock(return_value=mock_mutation)
mock_evolver.sandbox_execute_mutation = AsyncMock(return_value={"success": True})
import sys
original_module = sys.modules.get("core.auto_dev.alpha_evolver_engine")
class MockAlphaEvolverEngine:
def __init__(self, db):
pass
sys.modules["core.auto_dev.alpha_evolver_engine"] = MockAlphaEvolverEngine
sys.modules["core.auto_dev.alpha_evolver_engine"].AlphaEvolverEngine = lambda db: mock_evolver
try:
event = SkillExecutionEvent(
execution_id="exec-003",
agent_id="agent-001",
tenant_id="tenant-001",
skill_id="skill-001",
skill_name="test_skill",
execution_seconds=8.0,
token_usage=6000,
success=False, # Execution failure
)
await engine.process_execution(event)
# Should have spawned evolver
mock_evolver.generate_tool_mutation.assert_called_once()
finally:
if original_module:
sys.modules["core.auto_dev.alpha_evolver_engine"] = original_module
else:
sys.modules.pop("core.auto_dev.alpha_evolver_engine", None)
class TestEvolutionEngineVariantPruning:
"""Test prunes low-fitness variants."""
def test_prunes_low_fitness_variants(self, auto_dev_db_session, sample_tenant_id):
"""Test prunes low-fitness variants."""
from core.auto_dev.models import WorkflowVariant
# Create variants with different fitness scores
low_fitness = WorkflowVariant(
tenant_id=sample_tenant_id,
workflow_definition={},
fitness_score=0.1,
evaluation_status="evaluated",
)
high_fitness = WorkflowVariant(
tenant_id=sample_tenant_id,
workflow_definition={},
fitness_score=0.9,
evaluation_status="evaluated",
)
auto_dev_db_session.add(low_fitness)
auto_dev_db_session.add(high_fitness)
auto_dev_db_session.commit()
# Query should return high fitness first
from core.auto_dev.fitness_service import FitnessService
service = FitnessService(auto_dev_db_session)
top_variants = service.get_top_variants(sample_tenant_id, limit=10)
assert len(top_variants) >= 1
assert top_variants[0].fitness_score >= 0.9
class TestEvolutionEngineEventPublishing:
"""Test publishes evolution events."""
def test_register_on_event_bus(self, auto_dev_db_session):
"""Test registers on event bus."""
engine = EvolutionEngine(db=auto_dev_db_session)
# Should register without error
engine.register()
assert engine.db == auto_dev_db_session
|