itay
Deploy modular architecture with custom LFS models
cf5ed6c
Raw
History Blame Contribute Delete
32.6 kB
"""Tests for the FastAPI HTTP layer.
Covers:
- ``_extract_reasoning()`` with mock L1 and L2 findings
- ``GET /rules`` returns the expected 6 rules
- ``GET /health`` returns correct structure
- Authentication enforcement on ``POST /analyze``
"""
from __future__ import annotations
import os
from unittest.mock import AsyncMock, patch
import pytest
from app.engines.contracts import AnalysisResult, AnalyzerLayer, Verdict
from app.engines.manager import FinalVerdict
from app.routers.analyze import _derive_severity, _extract_reasoning
from app.schemas import FindingDetail
# ---- _extract_reasoning unit tests ----------------------------------------
class TestExtractReasoningL1:
"""Verify L1 trigger extraction from AnalysisResult metadata."""
def test_extracts_all_triggers(self):
"""All L1 triggers are converted to FindingDetail objects."""
l1_result = AnalysisResult(
layer=AnalyzerLayer.HEURISTICS,
score=45.0,
confidence=1.0,
findings=["CSS display:none hiding detected"],
metadata={
"triggers": [
{
"check_id": "css_display_none",
"severity": "high",
"score_contribution": 35.0,
"description": "CSS display:none hiding detected",
"evidence": "<span style='display:none'>hidden</span>",
},
{
"check_id": "url_mismatch",
"severity": "medium",
"score_contribution": 20.0,
"description": "Display text does not match href",
"evidence": "<a href='evil.com'>bank.com</a>",
},
],
},
)
verdict = FinalVerdict(
verdict=Verdict.SUSPICIOUS,
fused_score=45.0,
layer_results=[l1_result],
)
details = _extract_reasoning(verdict)
assert len(details) == 2
assert details[0] == FindingDetail(
check_id="css_display_none",
severity="high",
description="CSS display:none hiding detected",
)
assert details[1] == FindingDetail(
check_id="url_mismatch",
severity="medium",
description="Display text does not match href",
)
def test_empty_triggers(self):
"""No findings when L1 has no triggers."""
l1_result = AnalysisResult(
layer=AnalyzerLayer.HEURISTICS,
score=0.0,
confidence=1.0,
metadata={"triggers": []},
)
verdict = FinalVerdict(
verdict=Verdict.CLEAN,
fused_score=0.0,
layer_results=[l1_result],
)
assert _extract_reasoning(verdict) == []
class TestExtractReasoningL2:
"""Verify L2 semantic finding extraction with severity derivation.
Extraction now reads ``per_model`` directly instead of parsing
``findings`` strings. Only models with ``predicted_label == 1``
are emitted.
"""
def test_high_severity_above_08(self):
"""Probability >= 0.8 yields severity 'high'."""
l2_result = AnalysisResult(
layer=AnalyzerLayer.SEMANTIC,
score=90.0,
confidence=0.9,
metadata={
"per_model": {
"prompt_injection": {
"probability": 0.95,
"confidence": 0.9,
"predicted_label": 1,
"threshold": 0.5,
"model_id": "test-model",
"error": None,
},
},
},
)
verdict = FinalVerdict(
verdict=Verdict.MALICIOUS,
fused_score=90.0,
layer_results=[l2_result],
)
details = _extract_reasoning(verdict)
assert len(details) == 1
assert details[0].check_id == "prompt_injection"
assert details[0].severity == "high"
assert "0.950" in details[0].description
def test_medium_severity_between_05_08(self):
"""Probability in [0.5, 0.8) yields severity 'medium'."""
l2_result = AnalysisResult(
layer=AnalyzerLayer.SEMANTIC,
score=65.0,
confidence=0.5,
metadata={
"per_model": {
"malicious_intent": {
"probability": 0.65,
"confidence": 0.5,
"predicted_label": 1,
"threshold": 0.5,
"model_id": "test-model",
"error": None,
},
},
},
)
verdict = FinalVerdict(
verdict=Verdict.SUSPICIOUS,
fused_score=65.0,
layer_results=[l2_result],
)
details = _extract_reasoning(verdict)
assert len(details) == 1
assert details[0].severity == "medium"
def test_low_severity_below_05(self):
"""Probability < 0.5 yields severity 'low'."""
l2_result = AnalysisResult(
layer=AnalyzerLayer.SEMANTIC,
score=30.0,
confidence=0.2,
metadata={
"per_model": {
"prompt_injection": {
"probability": 0.3,
"confidence": 0.2,
"predicted_label": 1,
"threshold": 0.5,
"model_id": "test-model",
"error": None,
},
},
},
)
verdict = FinalVerdict(
verdict=Verdict.SUSPICIOUS,
fused_score=30.0,
layer_results=[l2_result],
)
details = _extract_reasoning(verdict)
assert len(details) == 1
assert details[0].severity == "low"
def test_predicted_label_0_skipped(self):
"""Models with predicted_label == 0 are not emitted."""
l2_result = AnalysisResult(
layer=AnalyzerLayer.SEMANTIC,
score=10.0,
confidence=0.8,
metadata={
"per_model": {
"prompt_injection": {
"probability": 0.1,
"confidence": 0.8,
"predicted_label": 0,
"threshold": 0.5,
"model_id": "test-model",
"error": None,
},
},
},
)
verdict = FinalVerdict(
verdict=Verdict.CLEAN,
fused_score=10.0,
layer_results=[l2_result],
)
assert _extract_reasoning(verdict) == []
def test_severity_derivation_function(self):
"""_derive_severity maps probabilities correctly."""
assert _derive_severity(0.95) == "high"
assert _derive_severity(0.80) == "high"
assert _derive_severity(0.79) == "medium"
assert _derive_severity(0.50) == "medium"
assert _derive_severity(0.49) == "low"
assert _derive_severity(0.0) == "low"
class TestExtractReasoningBothLayers:
"""Verify combined L1 + L2 extraction."""
def test_both_layers_combined(self):
"""Findings from both layers are collected in order."""
l1_result = AnalysisResult(
layer=AnalyzerLayer.HEURISTICS,
score=35.0,
confidence=1.0,
findings=["CSS hiding detected"],
metadata={
"triggers": [
{
"check_id": "css_display_none",
"severity": "high",
"score_contribution": 35.0,
"description": "CSS hiding detected",
"evidence": "...",
},
],
},
)
l2_result = AnalysisResult(
layer=AnalyzerLayer.SEMANTIC,
score=85.0,
confidence=0.8,
metadata={
"per_model": {
"prompt_injection": {
"probability": 0.85,
"confidence": 0.8,
"predicted_label": 1,
"threshold": 0.5,
"model_id": "test-model",
"error": None,
},
},
},
)
verdict = FinalVerdict(
verdict=Verdict.MALICIOUS,
fused_score=68.0,
layer_results=[l1_result, l2_result],
)
details = _extract_reasoning(verdict)
assert len(details) == 2
assert details[0].check_id == "css_display_none"
assert details[1].check_id == "prompt_injection"
def test_no_layer_results(self):
"""Empty layer_results yields empty reasoning."""
verdict = FinalVerdict(
verdict=Verdict.CLEAN,
fused_score=0.0,
layer_results=[],
)
assert _extract_reasoning(verdict) == []
# ---- Router-level tests via TestClient ------------------------------------
@pytest.fixture
def _set_api_key(monkeypatch):
"""Set API_KEY env var for test isolation."""
monkeypatch.setenv("API_KEY", "test-key-123")
@pytest.fixture
def client(_set_api_key, monkeypatch):
"""Create a FastAPI TestClient with mocked lifespan.
Patches model loading/shutdown so no real ML models are
instantiated during tests.
"""
from fastapi.testclient import TestClient
import app.engines.semantic.orchestrator as sem_orch
# Prevent real model loading/shutdown in lifespan.
monkeypatch.setattr(sem_orch, "load_models", lambda: None)
monkeypatch.setattr(sem_orch, "shutdown_models", lambda: None)
import app.main as main_mod
# app.main imports lifecycle functions directly, so patch those names too.
monkeypatch.setattr(main_mod, "load_models", lambda: None)
monkeypatch.setattr(main_mod, "shutdown_models", lambda: None)
monkeypatch.setattr(main_mod, "load_custom_models", lambda: None)
monkeypatch.setattr(main_mod, "shutdown_custom_models", lambda: None)
app = main_mod.app
with TestClient(app, raise_server_exceptions=False) as tc:
yield tc
class TestHealthEndpoint:
"""GET /health returns correct structure."""
def test_health_returns_status(self, client):
"""Health endpoint returns valid JSON with status field."""
resp = client.get("/health")
assert resp.status_code == 200
data = resp.json()
assert "status" in data
assert "models_loaded" in data
assert isinstance(data["models_loaded"], bool)
class TestRulesEndpoint:
"""GET /rules returns the full rule catalogue."""
def test_returns_six_rules(self, client):
"""Rules endpoint returns exactly 6 rules."""
resp = client.get("/rules")
assert resp.status_code == 200
rules = resp.json()
assert len(rules) == 6
def test_rule_ids(self, client):
"""All expected rule IDs are present."""
resp = client.get("/rules")
rule_ids = {r["id"] for r in resp.json()}
expected = {
"hidden_content",
"url_mismatch",
"structural_anomaly",
"attachment_surface",
"prompt_injection",
"phishing",
}
assert rule_ids == expected
def test_prompt_injection_spans_both_layers(self, client):
"""The prompt_injection rule has layer='both'."""
resp = client.get("/rules")
pi_rule = next(r for r in resp.json() if r["id"] == "prompt_injection")
assert pi_rule["layer"] == "both"
class TestAnalyzeAuth:
"""POST /analyze enforces API key authentication."""
def test_missing_api_key_returns_401(self, client):
"""Missing X-API-Key header returns 401 (uniform error surface)."""
resp = client.post(
"/analyze",
json={"message_id": "test"},
)
assert resp.status_code == 401
def test_wrong_api_key_returns_401(self, client):
"""Invalid API key returns 401."""
resp = client.post(
"/analyze",
json={"message_id": "test"},
headers={"X-API-Key": "wrong-key"},
)
assert resp.status_code == 401
def test_valid_key_passes_auth(self, client):
"""Valid API key does not trigger 401.
The request may still fail (e.g. 500 from unregistered analyzers),
but auth itself should pass.
"""
resp = client.post(
"/analyze",
json={
"message_id": "test",
"body_html": "",
"body_text": "",
},
headers={"X-API-Key": "test-key-123"},
)
# Should NOT be 401 — auth passed.
assert resp.status_code != 401
class TestAnalyzeEndpoint:
"""POST /analyze returns correct response structure."""
def test_full_analysis_response(self, client):
"""A mocked analysis returns all expected response fields."""
mock_verdict = FinalVerdict(
verdict=Verdict.SUSPICIOUS,
fused_score=45.0,
layer_results=[
AnalysisResult(
layer=AnalyzerLayer.HEURISTICS,
score=45.0,
confidence=1.0,
findings=["URL mismatch detected"],
metadata={
"triggers": [
{
"check_id": "url_mismatch",
"severity": "medium",
"score_contribution": 20.0,
"description": "URL mismatch detected",
"evidence": "<a>...</a>",
},
],
},
),
],
fusion_metadata={"winner": "heuristics"},
)
mock_manager = AsyncMock()
mock_manager.run.return_value = mock_verdict
client.app.state.manager = mock_manager
resp = client.post(
"/analyze",
json={
"message_id": "msg-123",
"subject": "Test",
"sender": "test@example.com",
"body_html": "<p>Hello</p>",
"body_text": "Hello",
},
headers={"X-API-Key": "test-key-123"},
)
assert resp.status_code == 200
data = resp.json()
assert data["message_id"] == "msg-123"
assert data["final_score"] == 45.0
assert data["verdict"] == "suspicious"
assert data["winning_layer"] == "heuristics"
assert data["l1_score"] == 45.0
assert data["was_short_circuited"] is False
assert len(data["reasoning"]) == 1
assert data["reasoning"][0]["check_id"] == "url_mismatch"
def test_request_body_too_large_html(self, client):
"""body_html exceeding 100k chars is rejected with 422."""
resp = client.post(
"/analyze",
json={
"message_id": "test",
"body_html": "x" * 100_001,
},
headers={"X-API-Key": "test-key-123"},
)
assert resp.status_code == 422
def test_extra_fields_rejected(self, client):
"""Extra fields in request body are rejected (extra='forbid')."""
resp = client.post(
"/analyze",
json={
"message_id": "test",
"unexpected_field": "value",
},
headers={"X-API-Key": "test-key-123"},
)
assert resp.status_code == 422
def test_message_id_too_long_rejected(self, client):
"""message_id exceeding 255 chars is rejected with 422."""
resp = client.post(
"/analyze",
json={"message_id": "x" * 256},
headers={"X-API-Key": "test-key-123"},
)
assert resp.status_code == 422
class TestGranularAnalyzeRouting:
"""E2E API tests for granular configuration routing semantics."""
def test_explicit_default_granular_config_is_echoed(self, client):
"""Presence of l1_config must produce analysis_config, even if default.
The architecture requires a presence-based echo so clients can
distinguish "legacy request" from "granular request with default
choices." Value-based legacy detection hides that distinction.
"""
resp = client.post(
"/analyze",
json={
"message_id": "explicit-default-l1",
"body_html": "<p>Hello team</p>",
"body_text": "Hello team",
"enable_l1": True,
"enable_l2": False,
"l1_config": {
"hidden_content": True,
"url_analysis": True,
"prompt_injection": True,
"structural_anomaly": True,
"attachment_surface": True,
},
},
headers={"X-API-Key": "test-key-123"},
)
assert resp.status_code == 200
data = resp.json()
assert data["analysis_config"] is not None
assert data["analysis_config"]["mode"] == "granular"
assert data["analysis_config"]["l1"]["hidden_content"] is True
def test_all_l1_checks_disabled_l2_enabled_returns_200(self, client):
"""All disabled L1 checks should skip L1 and still run L2."""
resp = client.post(
"/analyze",
json={
"message_id": "no-l1-checks",
"body_html": "<p>Verify your account immediately</p>",
"body_text": "Verify your account immediately",
"enable_l1": True,
"enable_l2": True,
"l1_config": {
"hidden_content": False,
"url_analysis": False,
"prompt_injection": False,
"structural_anomaly": False,
"attachment_surface": False,
},
},
headers={"X-API-Key": "test-key-123"},
)
assert resp.status_code == 200
data = resp.json()
assert data["l1_score"] is None
assert data["analysis_config"]["l1"]["hidden_content"] is False
def test_all_l2_engines_disabled_l1_enabled_returns_200(self, client):
"""All disabled L2 engines should skip L2 and still run L1."""
resp = client.post(
"/analyze",
json={
"message_id": "no-l2-engines",
"body_html": "<p>Hello team</p>",
"body_text": "Hello team",
"enable_l1": True,
"enable_l2": True,
"l2_config": {
"malicious_intent": False,
"prompt_injection": False,
"model_source": "ots",
},
},
headers={"X-API-Key": "test-key-123"},
)
assert resp.status_code == 200
data = resp.json()
assert data["l2_score"] is None
assert data["analysis_config"]["l2"]["malicious_intent"] is False
def test_both_layers_effectively_disabled_returns_400(self, client):
"""Granular config cannot bypass empty-scan validation."""
resp = client.post(
"/analyze",
json={
"message_id": "effectively-empty",
"body_html": "<p>Hello team</p>",
"body_text": "Hello team",
"enable_l1": True,
"enable_l2": True,
"l1_config": {
"hidden_content": False,
"url_analysis": False,
"prompt_injection": False,
"structural_anomaly": False,
"attachment_surface": False,
},
"l2_config": {
"malicious_intent": False,
"prompt_injection": False,
"model_source": "ots",
},
},
headers={"X-API-Key": "test-key-123"},
)
assert resp.status_code == 400
assert "At least one evaluation layer" in resp.json()["detail"]
def test_custom_l2_unavailable_returns_clean_503(self, client, monkeypatch):
"""Missing custom model files should produce a clean HTTP 503."""
monkeypatch.setattr(
"app.routers.analyze.custom_models_loaded",
lambda: False,
)
resp = client.post(
"/analyze",
json={
"message_id": "custom-unavailable",
"body_html": "<p>Verify your account</p>",
"body_text": "Verify your account",
"enable_l1": False,
"enable_l2": True,
"l2_config": {
"malicious_intent": True,
"prompt_injection": True,
"model_source": "custom",
},
},
headers={"X-API-Key": "test-key-123"},
)
assert resp.status_code == 503
assert "Custom models not available" in resp.json()["detail"]
def test_custom_single_engine_succeeds_when_other_custom_model_missing(
self, client, monkeypatch,
):
"""Only the requested custom engine should need to be available.
The current all-or-nothing custom_models_loaded() gate rejects this
scenario before constructing the analyzer. The fixed implementation
should permit malicious_intent-only custom analysis when the prompt
injection model directory is absent.
"""
class FakeCustomL2SemanticAnalyzer:
def __init__(self, l2_config=None) -> None:
self._config = l2_config
@property
def layer(self) -> str:
return AnalyzerLayer.SEMANTIC
async def analyze(self, payload):
return AnalysisResult(
layer=AnalyzerLayer.SEMANTIC,
score=42.0,
confidence=0.8,
metadata={
"per_model": {
"malicious_intent": {
"probability": 0.42,
"confidence": 0.8,
"predicted_label": 0,
"threshold": 0.5,
"model_id": "custom:malicious-only",
"error": None,
},
},
"model_type": "custom",
},
)
monkeypatch.setattr(
"app.routers.analyze.custom_models_loaded",
lambda: False,
)
monkeypatch.setattr(
"app.routers.analyze.CustomL2SemanticAnalyzer",
FakeCustomL2SemanticAnalyzer,
)
resp = client.post(
"/analyze",
json={
"message_id": "custom-mi-only",
"body_html": "<p>Verify your account</p>",
"body_text": "Verify your account",
"enable_l1": False,
"enable_l2": True,
"l2_config": {
"malicious_intent": True,
"prompt_injection": False,
"model_source": "custom",
},
},
headers={"X-API-Key": "test-key-123"},
)
assert resp.status_code == 200
data = resp.json()
assert data["l2_score"] == 42.0
assert data["analysis_config"]["l2"]["prompt_injection"] is False
class TestDefensiveL1Extraction:
"""Verify L1 extraction handles malformed trigger dicts gracefully."""
def test_missing_keys_use_defaults(self):
"""Trigger dicts with missing keys produce degraded findings."""
l1_result = AnalysisResult(
layer=AnalyzerLayer.HEURISTICS,
score=10.0,
confidence=1.0,
metadata={
"triggers": [
{"check_id": "css_display_none"},
],
},
)
verdict = FinalVerdict(
verdict=Verdict.CLEAN,
fused_score=10.0,
layer_results=[l1_result],
)
details = _extract_reasoning(verdict)
assert len(details) == 1
assert details[0].check_id == "css_display_none"
assert details[0].severity == "medium"
assert details[0].description == "No description"
def test_completely_empty_trigger_dict(self):
"""An empty trigger dict produces a finding with all defaults."""
l1_result = AnalysisResult(
layer=AnalyzerLayer.HEURISTICS,
score=5.0,
confidence=1.0,
metadata={"triggers": [{}]},
)
verdict = FinalVerdict(
verdict=Verdict.CLEAN,
fused_score=5.0,
layer_results=[l1_result],
)
details = _extract_reasoning(verdict)
assert len(details) == 1
assert details[0].check_id == "unknown"
def test_non_dict_trigger_skipped(self):
"""Non-dict entries in triggers list are silently skipped."""
l1_result = AnalysisResult(
layer=AnalyzerLayer.HEURISTICS,
score=5.0,
confidence=1.0,
metadata={"triggers": ["not_a_dict", 42, None]},
)
verdict = FinalVerdict(
verdict=Verdict.CLEAN,
fused_score=5.0,
layer_results=[l1_result],
)
assert _extract_reasoning(verdict) == []
class TestLayerToggleValidation:
"""POST /analyze enforces that at least one layer is enabled."""
def test_both_layers_disabled_returns_400(self, client):
"""Disabling both L1 and L2 returns 400 with descriptive detail."""
resp = client.post(
"/analyze",
json={
"message_id": "test",
"body_html": "",
"body_text": "",
"enable_l1": False,
"enable_l2": False,
},
headers={"X-API-Key": "test-key-123"},
)
assert resp.status_code == 400
assert "At least one evaluation layer" in resp.json()["detail"]
def test_only_l1_enabled_passes_validation(self, client):
"""enable_l1=True, enable_l2=False is accepted (not 400)."""
mock_verdict = FinalVerdict(
verdict=Verdict.CLEAN,
fused_score=5.0,
layer_results=[
AnalysisResult(
layer=AnalyzerLayer.HEURISTICS,
score=5.0,
confidence=1.0,
metadata={"triggers": []},
),
],
fusion_metadata={"winner": "heuristics"},
)
mock_manager = AsyncMock()
mock_manager.run.return_value = mock_verdict
client.app.state.manager = mock_manager
resp = client.post(
"/analyze",
json={
"message_id": "test",
"body_html": "",
"body_text": "",
"enable_l1": True,
"enable_l2": False,
},
headers={"X-API-Key": "test-key-123"},
)
assert resp.status_code == 200
def test_only_l2_enabled_passes_validation(self, client):
"""enable_l1=False, enable_l2=True is accepted (not 400)."""
mock_verdict = FinalVerdict(
verdict=Verdict.CLEAN,
fused_score=0.0,
layer_results=[
AnalysisResult(
layer=AnalyzerLayer.SEMANTIC,
score=0.0,
confidence=0.0,
metadata={"per_model": {}},
),
],
fusion_metadata={"winner": None},
)
mock_manager = AsyncMock()
mock_manager.run.return_value = mock_verdict
client.app.state.manager = mock_manager
resp = client.post(
"/analyze",
json={
"message_id": "test",
"body_html": "",
"body_text": "",
"enable_l1": False,
"enable_l2": True,
},
headers={"X-API-Key": "test-key-123"},
)
assert resp.status_code == 200
def test_defaults_enable_both_layers(self, client):
"""Omitting enable_l1/enable_l2 defaults both to True (not 400)."""
mock_verdict = FinalVerdict(
verdict=Verdict.CLEAN,
fused_score=0.0,
layer_results=[],
fusion_metadata={"winner": None},
)
mock_manager = AsyncMock()
mock_manager.run.return_value = mock_verdict
client.app.state.manager = mock_manager
resp = client.post(
"/analyze",
json={"message_id": "test"},
headers={"X-API-Key": "test-key-123"},
)
assert resp.status_code != 400
def test_layers_evaluated_in_response(self, client):
"""Response includes layers_evaluated metadata reflecting the flags."""
mock_verdict = FinalVerdict(
verdict=Verdict.CLEAN,
fused_score=5.0,
layer_results=[
AnalysisResult(
layer=AnalyzerLayer.HEURISTICS,
score=5.0,
confidence=1.0,
metadata={"triggers": []},
),
],
fusion_metadata={"winner": "heuristics"},
)
mock_manager = AsyncMock()
mock_manager.run.return_value = mock_verdict
client.app.state.manager = mock_manager
resp = client.post(
"/analyze",
json={
"message_id": "test",
"body_html": "",
"body_text": "",
"enable_l1": True,
"enable_l2": False,
},
headers={"X-API-Key": "test-key-123"},
)
assert resp.status_code == 200
data = resp.json()
assert data["layers_evaluated"] == {"l1": True, "l2": False}
def test_skip_layers_forwarded_to_manager(self, client):
"""Manager.run() receives the correct skip_layers argument."""
mock_verdict = FinalVerdict(
verdict=Verdict.CLEAN,
fused_score=0.0,
layer_results=[],
fusion_metadata={"winner": None},
)
mock_manager = AsyncMock()
mock_manager.run.return_value = mock_verdict
client.app.state.manager = mock_manager
client.post(
"/analyze",
json={
"message_id": "test",
"enable_l1": True,
"enable_l2": False,
},
headers={"X-API-Key": "test-key-123"},
)
mock_manager.run.assert_called_once()
call_kwargs = mock_manager.run.call_args
assert call_kwargs.kwargs["skip_layers"] == frozenset({"semantic"})
class TestBodySizeMiddleware:
"""Verify the body-size-limit middleware."""
def test_oversized_content_length_returns_413(self, client):
"""Content-Length exceeding 1 MB is rejected with 413."""
resp = client.post(
"/analyze",
content=b"x",
headers={
"X-API-Key": "test-key-123",
"Content-Type": "application/json",
"Content-Length": "2000000",
},
)
assert resp.status_code == 413
def test_normal_content_length_passes(self, client):
"""Content-Length under 1 MB is allowed through."""
resp = client.post(
"/analyze",
json={"message_id": "test"},
headers={"X-API-Key": "test-key-123"},
)
# Should not be 413 — body size is fine.
assert resp.status_code != 413