""" End-to-end integration tests for ArcisVLM cloud pipeline. Tests the full API contract, agent routing, alert CRUD, WebSocket connectivity, and orchestrator task classification — all without a GPU. Model inference is mocked; the goal is to verify the API layer and agent routing. """ import pytest import sys import os import base64 import io import json from unittest.mock import patch, MagicMock # Ensure project root is on path sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) # Set env to skip real model loading os.environ["ARCISVLM_CONFIG"] = "configs/scale_1.3b.yaml" def _make_test_image_base64() -> str: """Generate a small random test image as base64-encoded JPEG.""" # Create a minimal valid JPEG without needing PIL # Use a 2x2 red pixel BMP converted concept — but simpler: just use raw bytes # Actually, let's create a tiny valid JPEG # Minimal JPEG: SOI + APP0 + minimal frame # Easier approach: create with PIL if available, otherwise use a known minimal JPEG try: from PIL import Image img = Image.new("RGB", (4, 4), color=(128, 64, 32)) buf = io.BytesIO() img.save(buf, format="JPEG") return base64.b64encode(buf.getvalue()).decode("utf-8") except ImportError: # Minimal valid JPEG bytes (1x1 pixel white) minimal_jpeg = bytes([ 0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, 0x4A, 0x46, 0x49, 0x46, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0xFF, 0xDB, 0x00, 0x43, 0x00, 0x08, 0x06, 0x06, 0x07, 0x06, 0x05, 0x08, 0x07, 0x07, 0x07, 0x09, 0x09, 0x08, 0x0A, 0x0C, 0x14, 0x0D, 0x0C, 0x0B, 0x0B, 0x0C, 0x19, 0x12, 0x13, 0x0F, 0x14, 0x1D, 0x1A, 0x1F, 0x1E, 0x1D, 0x1A, 0x1C, 0x1C, 0x20, 0x24, 0x2E, 0x27, 0x20, 0x22, 0x2C, 0x23, 0x1C, 0x1C, 0x28, 0x37, 0x29, 0x2C, 0x30, 0x31, 0x34, 0x34, 0x34, 0x1F, 0x27, 0x39, 0x3D, 0x38, 0x32, 0x3C, 0x2E, 0x33, 0x34, 0x32, 0xFF, 0xC0, 0x00, 0x0B, 0x08, 0x00, 0x01, 0x00, 0x01, 0x01, 0x01, 0x11, 0x00, 0xFF, 0xC4, 0x00, 0x1F, 0x00, 0x00, 0x01, 0x05, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0xFF, 0xC4, 0x00, 0xB5, 0x10, 0x00, 0x02, 0x01, 0x03, 0x03, 0x02, 0x04, 0x03, 0x05, 0x05, 0x04, 0x04, 0x00, 0x00, 0x01, 0x7D, 0x01, 0x02, 0x03, 0x00, 0x04, 0x11, 0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xA1, 0x08, 0x23, 0x42, 0xB1, 0xC1, 0x15, 0x52, 0xD1, 0xF0, 0x24, 0x33, 0x62, 0x72, 0x82, 0x09, 0x0A, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFF, 0xDA, 0x00, 0x08, 0x01, 0x01, 0x00, 0x00, 0x3F, 0x00, 0x7B, 0x94, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xD9, ]) return base64.b64encode(minimal_jpeg).decode("utf-8") def _mock_result(answer="mocked answer", confidence=0.85, expert_used="vqa", **kwargs): """Create a mock Result object.""" from agents.base import Result return Result( answer=answer, confidence=confidence, expert_used=expert_used, task_id=kwargs.get("task_id", "test-task-001"), metadata=kwargs.get("metadata", {}), ) # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @pytest.fixture(scope="module") def client(): """Create a TestClient with mocked model manager for integration tests.""" from fastapi.testclient import TestClient from api.main import app with TestClient(app) as c: yield c @pytest.fixture(scope="module") def test_image_b64(): """A small test image as base64.""" return _make_test_image_base64() # --------------------------------------------------------------------------- # 1. Health endpoint returns model info # --------------------------------------------------------------------------- class TestHealthIntegration: def test_health_returns_200_with_model_info(self, client): resp = client.get("/health") assert resp.status_code == 200 data = resp.json() assert data["model"] == "arcisvlm-1.6b" assert data["version"] == "1.0.0" assert "status" in data assert "model_loaded" in data assert "agents_ready" in data def test_health_status_is_valid(self, client): resp = client.get("/health") data = resp.json() assert data["status"] in ("ok", "starting") # --------------------------------------------------------------------------- # 2–5. POST /api/v1/query with various task_types (mocked inference) # --------------------------------------------------------------------------- class TestInferenceIntegration: """Tests POST /api/v1/query with different task_types, mocking the submit_task.""" def test_vqa_query_with_image(self, client, test_image_b64): """POST /api/v1/query with a test image gets a VQA answer.""" mock_result = _mock_result( answer="This is a red building", expert_used="vqa", confidence=0.92, ) with patch("api.routes.inference.get_model_manager") as mock_mgr: manager = MagicMock() manager.camera_manager = None manager.submit_task.return_value = mock_result mock_mgr.return_value = manager resp = client.post("/api/v1/query", json={ "question": "What is in this image?", "image_base64": test_image_b64, "task_type": "vqa", }) assert resp.status_code == 200 data = resp.json() assert data["answer"] == "This is a red building" assert data["expert_used"] == "vqa" assert data["confidence"] == 0.92 assert "processing_time_ms" in data def test_detect_task_type(self, client, test_image_b64): """POST /api/v1/query with task_type=detect gets detection response.""" mock_result = _mock_result( answer="Detected 3 objects: person, car, dog", expert_used="detect", confidence=0.88, metadata={"objects": ["person", "car", "dog"], "count": 3}, ) with patch("api.routes.inference.get_model_manager") as mock_mgr: manager = MagicMock() manager.camera_manager = None manager.submit_task.return_value = mock_result mock_mgr.return_value = manager resp = client.post("/api/v1/query", json={ "question": "Detect objects in the scene", "image_base64": test_image_b64, "task_type": "detect", }) assert resp.status_code == 200 data = resp.json() assert "detect" in data["answer"].lower() or "object" in data["answer"].lower() assert data["expert_used"] == "detect" def test_count_task_type(self, client, test_image_b64): """POST /api/v1/query with task_type=count gets counting response.""" mock_result = _mock_result( answer="There are 5 people in the image", expert_used="count", confidence=0.90, metadata={"count": 5}, ) with patch("api.routes.inference.get_model_manager") as mock_mgr: manager = MagicMock() manager.camera_manager = None manager.submit_task.return_value = mock_result mock_mgr.return_value = manager resp = client.post("/api/v1/query", json={ "question": "How many people are there?", "image_base64": test_image_b64, "task_type": "count", }) assert resp.status_code == 200 data = resp.json() assert data["expert_used"] == "count" assert data["confidence"] > 0 def test_caption_task_type(self, client, test_image_b64): """POST /api/v1/query with task_type=caption gets caption response.""" mock_result = _mock_result( answer="A busy city street with cars and pedestrians under a cloudy sky", expert_used="caption", confidence=0.87, ) with patch("api.routes.inference.get_model_manager") as mock_mgr: manager = MagicMock() manager.camera_manager = None manager.submit_task.return_value = mock_result mock_mgr.return_value = manager resp = client.post("/api/v1/query", json={ "question": "Describe this scene", "image_base64": test_image_b64, "task_type": "caption", }) assert resp.status_code == 200 data = resp.json() assert data["expert_used"] == "caption" assert len(data["answer"]) > 0 def test_query_without_image_returns_400(self, client): """POST /api/v1/query with no image source returns 400.""" resp = client.post("/api/v1/query", json={ "question": "What is this?", }) assert resp.status_code == 400 # --------------------------------------------------------------------------- # 6. Alert rule CRUD: create → list → delete # --------------------------------------------------------------------------- class TestAlertCRUDIntegration: """Full CRUD cycle for alert rules.""" def test_alert_rule_crud_lifecycle(self, client): """Create an alert rule, verify it in the list, then delete it.""" # Patch get_model_manager for alerts routes to have an orchestrator from agents.mother import MotherOrchestrator orchestrator = MotherOrchestrator(max_agents_per_type=2) with patch("api.routes.alerts.get_model_manager") as mock_mgr: manager = MagicMock() manager.orchestrator = orchestrator manager.alert_history = [] mock_mgr.return_value = manager # CREATE rule_payload = { "rule_id": "integ-test-fire-alert", "condition_type": "presence", "target_object": "fire", "action": "webhook", } resp = client.post("/api/v1/alerts/rules", json=rule_payload) assert resp.status_code == 200 data = resp.json() assert data["rule_id"] == "integ-test-fire-alert" assert data["condition_type"] == "presence" assert data["target_object"] == "fire" assert data["enabled"] is True # LIST — verify the rule appears resp = client.get("/api/v1/alerts/rules") assert resp.status_code == 200 rules = resp.json() assert isinstance(rules, list) rule_ids = [r["rule_id"] for r in rules] assert "integ-test-fire-alert" in rule_ids # DELETE resp = client.delete("/api/v1/alerts/rules/integ-test-fire-alert") assert resp.status_code == 200 assert resp.json()["deleted"] == "integ-test-fire-alert" # Verify deletion — rule should be gone resp = client.get("/api/v1/alerts/rules") assert resp.status_code == 200 rules_after = resp.json() rule_ids_after = [r["rule_id"] for r in rules_after] assert "integ-test-fire-alert" not in rule_ids_after def test_delete_nonexistent_rule_returns_404(self, client): """Deleting a rule that doesn't exist returns 404.""" from agents.mother import MotherOrchestrator orchestrator = MotherOrchestrator(max_agents_per_type=2) with patch("api.routes.alerts.get_model_manager") as mock_mgr: manager = MagicMock() manager.orchestrator = orchestrator mock_mgr.return_value = manager resp = client.delete("/api/v1/alerts/rules/nonexistent-rule") assert resp.status_code == 404 def test_alert_history_returns_list(self, client): """GET /api/v1/alerts/history returns a list.""" with patch("api.routes.alerts.get_model_manager") as mock_mgr: manager = MagicMock() manager.alert_history = [] mock_mgr.return_value = manager resp = client.get("/api/v1/alerts/history") assert resp.status_code == 200 assert isinstance(resp.json(), list) # --------------------------------------------------------------------------- # 7. Agent status endpoint returns all 8 agent types # --------------------------------------------------------------------------- class TestAgentStatusIntegration: def test_agent_types_returns_all_eight(self, client): """GET /api/v1/agents/types returns all 8 expert types.""" resp = client.get("/api/v1/agents/types") assert resp.status_code == 200 data = resp.json() expected_types = {"vqa", "detect", "alert", "caption", "track", "count", "ocr", "reason"} actual_types = set(data["expert_types"]) assert expected_types == actual_types def test_agents_status_endpoint(self, client): """GET /api/v1/agents/status returns proper structure.""" resp = client.get("/api/v1/agents/status") assert resp.status_code == 200 data = resp.json() assert "agents" in data assert "total_agents" in data assert isinstance(data["agents"], dict) assert isinstance(data["total_agents"], int) # --------------------------------------------------------------------------- # 8. Streams endpoint returns camera list # --------------------------------------------------------------------------- class TestStreamsIntegration: def test_streams_status_returns_list(self, client): """GET /api/v1/streams/status returns a list (empty when no cameras).""" resp = client.get("/api/v1/streams/status") assert resp.status_code == 200 data = resp.json() assert isinstance(data, list) def test_start_stream_requires_fields(self, client): """POST /api/v1/streams/start with missing fields returns 422.""" resp = client.post("/api/v1/streams/start", json={}) assert resp.status_code == 422 # --------------------------------------------------------------------------- # 9. WebSocket /ws/chat connects and can send/receive # --------------------------------------------------------------------------- class TestWebSocketIntegration: def test_ws_chat_connect_and_send(self, client): """WebSocket /ws/chat connects and responds to a question.""" mock_result = _mock_result( answer="The scene shows a parking lot", expert_used="vqa", confidence=0.80, ) with patch("api.deps.get_model_manager") as mock_mgr: manager = MagicMock() manager.camera_manager = None manager.submit_task.return_value = mock_result manager.model = None mock_mgr.return_value = manager with client.websocket_connect("/ws/chat") as ws: ws.send_text(json.dumps({ "question": "What is happening here?", "task_type": "vqa", })) resp = ws.receive_json() assert resp["type"] == "answer" assert resp["answer"] == "The scene shows a parking lot" assert resp["expert_used"] == "vqa" def test_ws_chat_missing_question(self, client): """WebSocket /ws/chat returns error when question is missing.""" with client.websocket_connect("/ws/chat") as ws: ws.send_text(json.dumps({"task_type": "vqa"})) resp = ws.receive_json() assert "error" in resp # --------------------------------------------------------------------------- # 10. Agent routing: MotherOrchestrator classify_task # --------------------------------------------------------------------------- class TestAgentRoutingIntegration: """Verify MotherOrchestrator routes queries to the correct agent type.""" def test_tracking_query_routes_to_track(self): """'track the person' routes to TrackingAgent.""" from agents.mother import classify_task from agents.base import EXPERT_TRACK result = classify_task("track the person moving across the room") assert result == EXPERT_TRACK def test_caption_query_routes_to_caption(self): """'describe this scene' routes to CaptionAgent.""" from agents.mother import classify_task from agents.base import EXPERT_CAPTION result = classify_task("describe this scene in detail") assert result == EXPERT_CAPTION def test_counting_query_routes_to_count(self): """'how many cars' routes to CountingAgent.""" from agents.mother import classify_task from agents.base import EXPERT_COUNT result = classify_task("how many cars are in the parking lot") assert result == EXPERT_COUNT def test_detection_query_routes_to_detect(self): """'detect objects' routes to DetectAgent.""" from agents.mother import classify_task from agents.base import EXPERT_DETECT result = classify_task("detect all objects in the frame") assert result == EXPERT_DETECT def test_alert_query_routes_to_alert(self): """'alert for intruder' routes to AlertAgent.""" from agents.mother import classify_task from agents.base import EXPERT_ALERT result = classify_task("alert me if there is an intruder") assert result == EXPERT_ALERT def test_ocr_query_routes_to_ocr(self): """'read the text' routes to OCRAgent.""" from agents.mother import classify_task from agents.base import EXPERT_OCR result = classify_task("read the text on the sign") assert result == EXPERT_OCR def test_reasoning_query_routes_to_reason(self): """'explain why' routes to ReasoningAgent.""" from agents.mother import classify_task from agents.base import EXPERT_REASON result = classify_task("explain why the car stopped") assert result == EXPERT_REASON def test_vqa_fallback(self): """Generic question falls back to VQA.""" from agents.mother import classify_task from agents.base import EXPERT_VQA result = classify_task("what color is the building") assert result == EXPERT_VQA def test_orchestrator_classify_method(self): """MotherOrchestrator.classify() uses the task query for routing.""" from agents.mother import MotherOrchestrator from agents.base import Task, EXPERT_TRACK mother = MotherOrchestrator(max_agents_per_type=2) task = Task( type="track", payload={"image_ref": "/tmp/test.jpg", "query": "track the person"}, source="test", expert_hint="vqa", # hint says vqa but query says track ) # classify should use the query text, not the hint expert = mother.classify(task) assert expert == EXPERT_TRACK mother.shutdown()