"""Tests for RAG upload sanitisation (regex path; no live OpenAI).""" from __future__ import annotations import pytest from app.services.document_sanitiser import ( RedactionStrategy, _deterministic_redact, regex_sanitise_text, sanitise_text_for_rag_sync, should_sanitise_for_rag, ) def test_should_skip_kb_tenant(monkeypatch: pytest.MonkeyPatch) -> None: from app.config import settings monkeypatch.setattr(settings, "enable_rag_upload_sanitisation", True) monkeypatch.setattr(settings, "rag_sanitisation_skip_kb_tenant", True) monkeypatch.setattr(settings, "knowledge_base_tenant_id", "__rics_kb__") assert should_sanitise_for_rag("tenant-a") is True assert should_sanitise_for_rag("__rics_kb__") is False def test_regex_removes_postcode_and_email() -> None: raw = ( "The roof covering is slate. Client: jane.doe@example.com. " "Property at 12 High Street, London SW1A 1AA. Price £450,000." ) out = regex_sanitise_text(raw) assert "jane.doe@example.com" not in out assert "SW1A" not in out assert "450,000" not in out assert "slate" in out.lower() def test_sanitise_sync_without_openai_key(monkeypatch: pytest.MonkeyPatch) -> None: from app.config import settings monkeypatch.setattr(settings, "enable_rag_upload_sanitisation", True) monkeypatch.setattr(settings, "openai_api_key", "") monkeypatch.setattr(settings, "rag_sanitisation_fail_closed", False) text = "Defect noted at 10 Church Road, Bristol BS1 4ST." out = sanitise_text_for_rag_sync(text, tenant_id="firm-1") assert "BS1" not in out assert "defect" in out.lower() def test_regex_preserves_proptech_vocabulary() -> None: raw = ( "Timber purlins to the left rear elevation; lead flashing and uPVC gutters. " "Certified under FENSA and Building Regulations." ) out = regex_sanitise_text(raw) assert "timber" in out.lower() assert "fensa" in out.lower() assert "building regulations" in out.lower() def test_system_prompt_includes_descriptive_intent_and_antipatterns() -> None: from app.services.document_sanitiser import RAG_UPLOAD_SANITISATION_SYSTEM_PROMPT assert "DESCRIPTIVE INTENT" in RAG_UPLOAD_SANITISATION_SYSTEM_PROMPT assert "ANTI-PATTERNS" in RAG_UPLOAD_SANITISATION_SYSTEM_PROMPT assert "timber purlins" in RAG_UPLOAD_SANITISATION_SYSTEM_PROMPT assert "downstream deterministic pass" in RAG_UPLOAD_SANITISATION_SYSTEM_PROMPT.lower() assert "" not in RAG_UPLOAD_SANITISATION_SYSTEM_PROMPT def test_deterministic_redacts_regex_pii_and_preserves_vocabulary() -> None: raw = ( "Timber purlins to the left rear elevation; email surveyor@firm.co.uk, " "call 07911 123456, property postcode SW1A 1AA." ) out = _deterministic_redact(raw) assert "surveyor@firm.co.uk" not in out assert "07911 123456" not in out assert "SW1A 1AA" not in out assert "[REDACTED]" in out # Survey vocabulary survives the deterministic pass. for term in ("timber", "purlins", "left rear elevation"): assert term in out.lower() def test_deterministic_db_context_exact_wipe() -> None: raw = "Prepared for Jonathan Pemberton at the Maple Lodge instruction." out = _deterministic_redact( raw, db_context={"client": "Jonathan Pemberton", "property": "Maple Lodge"} ) assert "Jonathan Pemberton" not in out assert "Maple Lodge" not in out assert out.lower().count("[redacted]") >= 2 def test_deterministic_db_context_none_is_graceful() -> None: raw = "Roof covering is natural slate in satisfactory condition." out = _deterministic_redact(raw, db_context=None) assert out == raw # no regex PII, no context → unchanged def test_deterministic_empty_input_raises() -> None: with pytest.raises(ValueError): _deterministic_redact(" ") def test_deterministic_strategy_runs_without_network(monkeypatch: pytest.MonkeyPatch) -> None: """DETERMINISTIC_CODE must not touch the LLM even with a key set + policy off.""" from app.config import settings from app.services import document_sanitiser monkeypatch.setattr(settings, "enable_rag_upload_sanitisation", False) monkeypatch.setattr(settings, "openai_api_key", "sk-should-not-be-used") def _boom(*args, **kwargs): # pragma: no cover - must never run raise AssertionError("LLM path invoked on deterministic strategy") monkeypatch.setattr(document_sanitiser, "_llm_sanitise_chunk_sync", _boom) out = sanitise_text_for_rag_sync( "Contact a@b.com about 12 Acacia Road SW1A 1AA.", tenant_id="firm-1", strategy=RedactionStrategy.DETERMINISTIC_CODE, db_context={"agent": "a@b.com"}, ) assert "a@b.com" not in out assert "SW1A 1AA" not in out def test_default_strategy_is_ai_hybrid_and_policy_gate_preserved( monkeypatch: pytest.MonkeyPatch, ) -> None: """Existing callers (no strategy arg) keep AI_HYBRID behaviour: policy off → passthrough.""" from app.config import settings monkeypatch.setattr(settings, "enable_rag_upload_sanitisation", False) text = "Email keep@me.com — policy is off so AI path returns text unchanged." out = sanitise_text_for_rag_sync(text, tenant_id="firm-1") assert out == text.strip()