app/tests/test_document_sanitiser.py

"""Tests for RAG upload sanitisation (regex path; no live OpenAI)."""

from __future__ import annotations

import pytest

from app.services.document_sanitiser import (
    RedactionStrategy,
    _deterministic_redact,
    regex_sanitise_text,
    sanitise_text_for_rag_sync,
    should_sanitise_for_rag,
)


def test_should_skip_kb_tenant(monkeypatch: pytest.MonkeyPatch) -> None:
    from app.config import settings

    monkeypatch.setattr(settings, "enable_rag_upload_sanitisation", True)
    monkeypatch.setattr(settings, "rag_sanitisation_skip_kb_tenant", True)
    monkeypatch.setattr(settings, "knowledge_base_tenant_id", "__rics_kb__")
    assert should_sanitise_for_rag("tenant-a") is True
    assert should_sanitise_for_rag("__rics_kb__") is False


def test_regex_removes_postcode_and_email() -> None:
    raw = (
        "The roof covering is slate. Client: jane.doe@example.com. "
        "Property at 12 High Street, London SW1A 1AA. Price £450,000."
    )
    out = regex_sanitise_text(raw)
    assert "jane.doe@example.com" not in out
    assert "SW1A" not in out
    assert "450,000" not in out
    assert "slate" in out.lower()


def test_sanitise_sync_without_openai_key(monkeypatch: pytest.MonkeyPatch) -> None:
    from app.config import settings

    monkeypatch.setattr(settings, "enable_rag_upload_sanitisation", True)
    monkeypatch.setattr(settings, "openai_api_key", "")
    monkeypatch.setattr(settings, "rag_sanitisation_fail_closed", False)
    text = "Defect noted at 10 Church Road, Bristol BS1 4ST."
    out = sanitise_text_for_rag_sync(text, tenant_id="firm-1")
    assert "BS1" not in out
    assert "defect" in out.lower()


def test_regex_preserves_proptech_vocabulary() -> None:
    raw = (
        "Timber purlins to the left rear elevation; lead flashing and uPVC gutters. "
        "Certified under FENSA and Building Regulations."
    )
    out = regex_sanitise_text(raw)
    assert "timber" in out.lower()
    assert "fensa" in out.lower()
    assert "building regulations" in out.lower()


def test_system_prompt_includes_descriptive_intent_and_antipatterns() -> None:
    from app.services.document_sanitiser import RAG_UPLOAD_SANITISATION_SYSTEM_PROMPT

    assert "DESCRIPTIVE INTENT" in RAG_UPLOAD_SANITISATION_SYSTEM_PROMPT
    assert "ANTI-PATTERNS" in RAG_UPLOAD_SANITISATION_SYSTEM_PROMPT
    assert "timber purlins" in RAG_UPLOAD_SANITISATION_SYSTEM_PROMPT
    assert "downstream deterministic pass" in RAG_UPLOAD_SANITISATION_SYSTEM_PROMPT.lower()
    assert "<input_document>" not in RAG_UPLOAD_SANITISATION_SYSTEM_PROMPT


def test_deterministic_redacts_regex_pii_and_preserves_vocabulary() -> None:
    raw = (
        "Timber purlins to the left rear elevation; email surveyor@firm.co.uk, "
        "call 07911 123456, property postcode SW1A 1AA."
    )
    out = _deterministic_redact(raw)
    assert "surveyor@firm.co.uk" not in out
    assert "07911 123456" not in out
    assert "SW1A 1AA" not in out
    assert "[REDACTED]" in out
    # Survey vocabulary survives the deterministic pass.
    for term in ("timber", "purlins", "left rear elevation"):
        assert term in out.lower()


def test_deterministic_db_context_exact_wipe() -> None:
    raw = "Prepared for Jonathan Pemberton at the Maple Lodge instruction."
    out = _deterministic_redact(
        raw, db_context={"client": "Jonathan Pemberton", "property": "Maple Lodge"}
    )
    assert "Jonathan Pemberton" not in out
    assert "Maple Lodge" not in out
    assert out.lower().count("[redacted]") >= 2


def test_deterministic_db_context_none_is_graceful() -> None:
    raw = "Roof covering is natural slate in satisfactory condition."
    out = _deterministic_redact(raw, db_context=None)
    assert out == raw  # no regex PII, no context → unchanged


def test_deterministic_empty_input_raises() -> None:
    with pytest.raises(ValueError):
        _deterministic_redact("   ")


def test_deterministic_strategy_runs_without_network(monkeypatch: pytest.MonkeyPatch) -> None:
    """DETERMINISTIC_CODE must not touch the LLM even with a key set + policy off."""
    from app.config import settings
    from app.services import document_sanitiser

    monkeypatch.setattr(settings, "enable_rag_upload_sanitisation", False)
    monkeypatch.setattr(settings, "openai_api_key", "sk-should-not-be-used")

    def _boom(*args, **kwargs):  # pragma: no cover - must never run
        raise AssertionError("LLM path invoked on deterministic strategy")

    monkeypatch.setattr(document_sanitiser, "_llm_sanitise_chunk_sync", _boom)

    out = sanitise_text_for_rag_sync(
        "Contact a@b.com about 12 Acacia Road SW1A 1AA.",
        tenant_id="firm-1",
        strategy=RedactionStrategy.DETERMINISTIC_CODE,
        db_context={"agent": "a@b.com"},
    )
    assert "a@b.com" not in out
    assert "SW1A 1AA" not in out


def test_default_strategy_is_ai_hybrid_and_policy_gate_preserved(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Existing callers (no strategy arg) keep AI_HYBRID behaviour: policy off → passthrough."""
    from app.config import settings

    monkeypatch.setattr(settings, "enable_rag_upload_sanitisation", False)
    text = "Email keep@me.com — policy is off so AI path returns text unchanged."
    out = sanitise_text_for_rag_sync(text, tenant_id="firm-1")
    assert out == text.strip()