tests/observability/test_export_failure.py

"""A failing exporter or store must never break or block product code.

Global constraint from the plan preamble: "Telemetry export, local persistence,
and SigNoz failures must never fail a product request."
"""

from __future__ import annotations

import pytest

from app.observability import bootstrap
from app.observability.local_store import LocalObservationStore
from app.observability.operation import observe_operation


class _FailingExporter:
    def export(self, observation):  # noqa: ARG002 - signature must match real exporter
        raise RuntimeError("exporter boom")


class _FailingStore:
    def record(self, observation):  # noqa: ARG002
        raise OSError("disk full")

    def latest(self):
        raise OSError("disk full")


@pytest.fixture(autouse=True)
def _reset_failure_counter():
    bootstrap.reset_suppressed_failures()
    yield
    bootstrap.shutdown_observability(timeout_seconds=2.0)


# --- Brief Step 2 example ---------------------------------------------------


def test_export_failure_does_not_escape_product_operation():
    exporter = _FailingExporter()
    with observe_operation("rag.request", exporter=exporter):
        result = "product-result"
    assert result == "product-result"


def test_export_failure_is_counted_but_suppressed():
    exporter = _FailingExporter()
    with observe_operation("rag.request", exporter=exporter):
        pass
    assert bootstrap.get_suppressed_failure_count() >= 1


def test_store_failure_does_not_escape_product_operation():
    store = _FailingStore()
    with observe_operation("rag.request", store=store):
        result = 42
    assert result == 42
    assert bootstrap.get_suppressed_failure_count() >= 1


def test_store_and_exporter_failure_together_still_returns_result(tmp_path):
    store = _FailingStore()
    exporter = _FailingExporter()
    with observe_operation("rag.request", store=store, exporter=exporter) as op:
        op.set("raw_candidate_count", 3)
        value = ["ok"]
    assert value == ["ok"]


def test_exporter_failure_does_not_suppress_a_real_exception():
    exporter = _FailingExporter()
    with pytest.raises(ValueError, match="product bug"):
        with observe_operation("rag.request", exporter=exporter):
            raise ValueError("product bug")


def test_working_store_still_records_when_exporter_fails(tmp_path):
    store = LocalObservationStore(tmp_path)
    exporter = _FailingExporter()
    with observe_operation("rag.request", store=store, exporter=exporter) as op:
        op.set("raw_candidate_count", 8)
    # Store write must still succeed even though the exporter blew up.
    assert store.latest()["attributes"]["raw_candidate_count"] == 8