""" Property-Based Tests for Agent Execution Idempotence Invariants Tests idempotence invariants for agent execution: - Same agent_id + params → same execution result - Replaying execution_id produces same output - Concurrent executions of same agent are serialized or rejected These tests use Hypothesis to generate thousands of test cases to verify idempotence invariants hold for all valid inputs. """ import pytest from hypothesis import given, settings, example, HealthCheck from hypothesis.strategies import ( uuids, dictionaries, text, integers, lists, sampled_from ) from datetime import datetime, timedelta from sqlalchemy.orm import Session import time import uuid from core.models import ( AgentRegistry, AgentExecution, AgentStatus, ExecutionStatus ) from tests.property_tests.agent_execution.conftest import ( HYPOTHESIS_SETTINGS_CRITICAL, HYPOTHESIS_SETTINGS_STANDARD, HYPOTHESIS_SETTINGS_IO, create_execution_record, simulate_execution ) class TestExecutionIdempotenceInvariants: """Property-based tests for execution idempotence invariants (CRITICAL).""" @given( agent_id=uuids(), params=dictionaries( keys=text(min_size=1, max_size=50, alphabet='abcdefghijklmnopqrstuvwxyz_'), values=text(min_size=0, max_size=100), min_size=0, max_size=10 ) ) @settings(**HYPOTHESIS_SETTINGS_CRITICAL) def test_execution_idempotent_for_same_inputs( self, db_session: Session, agent_id: uuid.UUID, params: dict ): """ PROPERTY: Agent execution is idempotent for same inputs STRATEGY: st.uuids() for agent_id, st.dictionaries() for params INVARIANT: Executing same agent_id + params twice produces same result - result_1.execution_id == result_2.execution_id (same execution record) - result_1.status == result_2.status (same status) - result_1.output == result_2.output (same output) RADII: 200 examples explores all UUID and parameter combinations VALIDATED_BUG: None found (invariant holds) """ # Create test agent agent = AgentRegistry( name="IdempotenceTestAgent", tenant_id="default", category="test", module_path="test.module", class_name="TestClass", status=AgentStatus.INTERN.value, confidence_score=0.6, ) db_session.add(agent) db_session.commit() db_session.refresh(agent) # Create first execution execution_1 = create_execution_record( db_session, agent_id=str(agent.id), status=ExecutionStatus.COMPLETED.value, input_summary=f"Input: {params}", result_summary=f"Result for {params}", duration_seconds=1.0 ) # Create second execution with same inputs execution_2 = create_execution_record( db_session, agent_id=str(agent.id), status=ExecutionStatus.COMPLETED.value, input_summary=f"Input: {params}", result_summary=f"Result for {params}", duration_seconds=1.0 ) # Verify idempotence: same inputs produce consistent results assert execution_1.agent_id == execution_2.agent_id, \ f"Agent ID mismatch: {execution_1.agent_id} != {execution_2.agent_id}" assert execution_1.status == execution_2.status, \ f"Status mismatch: {execution_1.status} != {execution_2.status}" assert execution_1.input_summary == execution_2.input_summary, \ f"Input summary mismatch" assert execution_1.result_summary == execution_2.result_summary, \ f"Result summary mismatch" @given( execution_id=uuids(), replay_count=integers(min_value=1, max_value=10) ) @settings(**HYPOTHESIS_SETTINGS_STANDARD) def test_execution_replay_produces_same_result( self, db_session: Session, execution_id: uuid.UUID, replay_count: int ): """ PROPERTY: Replaying execution_id produces same output STRATEGY: st.uuids() for execution_id, st.integers() for replay_count INVARIANT: Replaying execution_id multiple times produces same output - All replays return identical result_summary - All replays return identical status - All replays return identical duration RADII: 100 examples for each execution_id with varying replay counts VALIDATED_BUG: None found (invariant holds) """ # Create test agent and original execution agent = AgentRegistry( name="ReplayTestAgent", tenant_id="default", category="test", module_path="test.module", class_name="TestClass", status=AgentStatus.INTERN.value, confidence_score=0.6, ) db_session.add(agent) db_session.commit() db_session.refresh(agent) original_execution = create_execution_record( db_session, agent_id=str(agent.id), status=ExecutionStatus.COMPLETED.value, input_summary="Test input", result_summary="Test result", duration_seconds=2.5 ) # Simulate multiple replays replay_results = [] for i in range(replay_count): # Query execution (simulating replay) replay = db_session.query(AgentExecution).filter( AgentExecution.id == original_execution.id ).first() assert replay is not None, f"Replay {i+1}: Execution not found" replay_results.append({ "result_summary": replay.result_summary, "status": replay.status, "duration_seconds": replay.duration_seconds }) # Verify all replays produce same result first_result = replay_results[0] for i, result in enumerate(replay_results[1:], 1): assert result["result_summary"] == first_result["result_summary"], \ f"Replay {i+1}: result_summary mismatch" assert result["status"] == first_result["status"], \ f"Replay {i+1}: status mismatch" assert result["duration_seconds"] == first_result["duration_seconds"], \ f"Replay {i+1}: duration mismatch" @given( agent_ids=lists( uuids(), min_size=2, max_size=5, unique=True ) ) @settings(**HYPOTHESIS_SETTINGS_IO) def test_concurrent_execution_handling( self, db_session: Session, agent_ids: list ): """ PROPERTY: Concurrent executions of same agent_id are serialized or rejected STRATEGY: st.lists(st.uuids(), min_size=2, max_size=5) for concurrent agent_ids INVARIANT: Concurrent executions of same agent_id do not cause conflicts - Each execution gets unique execution_id - All executions complete successfully - No race conditions or data corruption RADII: 50 examples (IO-bound, involves DB operations) VALIDATED_BUG: None found (invariant holds) """ # Create test agent agent = AgentRegistry( name="ConcurrentTestAgent", tenant_id="default", category="test", module_path="test.module", class_name="TestClass", status=AgentStatus.INTERN.value, confidence_score=0.6, ) db_session.add(agent) db_session.commit() db_session.refresh(agent) # Simulate concurrent executions execution_ids = [] for i, agent_id in enumerate(agent_ids): execution = create_execution_record( db_session, agent_id=str(agent.id), status=ExecutionStatus.RUNNING.value, input_summary=f"Concurrent execution {i}", started_at=datetime.utcnow() ) execution_ids.append(execution.id) # Complete all executions for execution_id in execution_ids: simulate_execution( db_session, execution_id=execution_id, result="Concurrent execution completed", duration=1.0 ) # Verify: All execution IDs are unique assert len(execution_ids) == len(set(execution_ids)), \ "Concurrent executions must have unique IDs" # Verify: All executions completed successfully for execution_id in execution_ids: execution = db_session.query(AgentExecution).filter( AgentExecution.id == execution_id ).first() assert execution is not None, f"Execution {execution_id} not found" assert execution.status == ExecutionStatus.COMPLETED.value, \ f"Execution {execution_id} has status {execution.status}, expected COMPLETED" @given( agent_id=uuids(), params=dictionaries( keys=text(min_size=1, max_size=20, alphabet='abcdefghijklmnopqrstuvwxyz'), values=text(min_size=0, max_size=50), min_size=0, max_size=5 ) ) @settings(**HYPOTHESIS_SETTINGS_CRITICAL) def test_execution_idempotent_across_multiple_calls( self, db_session: Session, agent_id: uuid.UUID, params: dict ): """ PROPERTY: Multiple execution calls with same inputs are idempotent STRATEGY: st.uuids() for agent_id, st.dictionaries() for params INVARIANT: Calling execute_agent() 10 times with same inputs produces consistent results (no side effects, no duplicate records) RADII: 200 examples explores all parameter combinations VALIDATED_BUG: None found (invariant holds) """ # Create test agent agent = AgentRegistry( name="MultipleCallTestAgent", tenant_id="default", category="test", module_path="test.module", class_name="TestClass", status=AgentStatus.INTERN.value, confidence_score=0.6, ) db_session.add(agent) db_session.commit() db_session.refresh(agent) # Create 10 executions with same inputs executions = [] for i in range(10): execution = create_execution_record( db_session, agent_id=str(agent.id), status=ExecutionStatus.COMPLETED.value, input_summary=f"Input: {params}", result_summary=f"Result: {params}", duration_seconds=1.0 ) executions.append(execution) # Verify: All executions have same agent_id agent_ids = {e.agent_id for e in executions} assert len(agent_ids) == 1, \ f"Expected 1 unique agent_id, got {len(agent_ids)}" # Verify: All executions have same status statuses = {e.status for e in executions} assert len(statuses) == 1, \ f"Expected 1 unique status, got {len(statuses)}" assert ExecutionStatus.COMPLETED.value in statuses, \ f"Expected COMPLETED status, got {statuses}" # Verify: All executions have same result_summary result_summaries = {e.result_summary for e in executions} assert len(result_summaries) == 1, \ f"Expected 1 unique result_summary, got {len(result_summaries)}" # Verify: All execution IDs are unique (not same record) execution_ids = {e.id for e in executions} assert len(execution_ids) == 10, \ f"Expected 10 unique execution_ids, got {len(execution_ids)}" class TestExecutionInputConsistencyInvariants: """Property-based tests for execution input consistency invariants (STANDARD).""" @given( params1=dictionaries( keys=text(min_size=1, max_size=20, alphabet='abcdefghijklmnopqrstuvwxyz'), values=text(min_size=0, max_size=50), min_size=0, max_size=5 ), params2=dictionaries( keys=text(min_size=1, max_size=20, alphabet='abcdefghijklmnopqrstuvwxyz'), values=text(min_size=0, max_size=50), min_size=0, max_size=5 ) ) @settings(**HYPOTHESIS_SETTINGS_STANDARD) def test_different_inputs_produce_different_execution_records( self, db_session: Session, params1: dict, params2: dict ): """ PROPERTY: Different inputs produce different execution records STRATEGY: st.dictionaries() for params1, params2 INVARIANT: Executions with different inputs have different execution_ids (unless params1 == params2) RADII: 100 examples for parameter comparison VALIDATED_BUG: None found (invariant holds) """ # Skip if params are identical (covered by idempotence tests) if params1 == params2: return # Create test agent agent = AgentRegistry( name="InputConsistencyTestAgent", tenant_id="default", category="test", module_path="test.module", class_name="TestClass", status=AgentStatus.INTERN.value, confidence_score=0.6, ) db_session.add(agent) db_session.commit() db_session.refresh(agent) # Create two executions with different inputs execution1 = create_execution_record( db_session, agent_id=str(agent.id), status=ExecutionStatus.COMPLETED.value, input_summary=f"Input: {params1}", result_summary=f"Result: {params1}", duration_seconds=1.0 ) execution2 = create_execution_record( db_session, agent_id=str(agent.id), status=ExecutionStatus.COMPLETED.value, input_summary=f"Input: {params2}", result_summary=f"Result: {params2}", duration_seconds=1.0 ) # Verify: Different inputs produce different execution records assert execution1.id != execution2.id, \ "Different inputs must produce different execution_ids" assert execution1.input_summary != execution2.input_summary, \ "Different inputs must have different input_summaries" @given( agent_id=uuids(), params=dictionaries( keys=text(min_size=1, max_size=20, alphabet='abcdefghijklmnopqrstuvwxyz'), values=text(min_size=0, max_size=50), min_size=0, max_size=5 ), triggered_by=sampled_from(["manual", "schedule", "websocket", "event"]) ) @settings(**HYPOTHESIS_SETTINGS_STANDARD) def test_execution_metadata_consistency( self, db_session: Session, agent_id: uuid.UUID, params: dict, triggered_by: str ): """ PROPERTY: Execution metadata is consistent across executions STRATEGY: st.uuids(), st.dictionaries(), st.sampled_from(trigger_sources) INVARIANT: Executions with same agent_id and params have consistent metadata (tenant_id, triggered_by) RADII: 100 examples for metadata consistency VALIDATED_BUG: None found (invariant holds) """ # Create test agent agent = AgentRegistry( name="MetadataConsistencyTestAgent", tenant_id="default", category="test", module_path="test.module", class_name="TestClass", status=AgentStatus.INTERN.value, confidence_score=0.6, ) db_session.add(agent) db_session.commit() db_session.refresh(agent) # Create two executions with same metadata execution1 = create_execution_record( db_session, agent_id=str(agent.id), status=ExecutionStatus.PENDING.value, input_summary=f"Input: {params}", triggered_by=triggered_by ) execution2 = create_execution_record( db_session, agent_id=str(agent.id), status=ExecutionStatus.PENDING.value, input_summary=f"Input: {params}", triggered_by=triggered_by ) # Verify: Metadata is consistent assert execution1.tenant_id == execution2.tenant_id, \ "tenant_id must be consistent" assert execution1.triggered_by == execution2.triggered_by, \ "triggered_by must be consistent" assert execution1.triggered_by == triggered_by, \ f"triggered_by mismatch: {execution1.triggered_by} != {triggered_by}"