benchmarks/test_accessibility_timing.py

"""Timing benchmarks for the accessibility test suite.

Measures collection and total execution wall-clock time for
``tests/test_accessibility_components.py``. Each measurement runs
pytest as a subprocess — these are coarse regression detectors,
not micro-benchmarks.

Uses manual ``time.perf_counter()`` timing (not pytest-benchmark's
``benchmark`` fixture) because the subprocess per run is too expensive
for the multiple rounds pytest-benchmark would need (default min_rounds=5).
"""

from __future__ import annotations

import re
import subprocess
import time
from pathlib import Path

import pytest

pytestmark = pytest.mark.benchmark(group="accessibility_timing")

PROJECT_ROOT = Path(__file__).resolve().parent.parent

# Base pytest args shared by both collection and execution runs.
_PYTEST_ARGS = [
    "tests/test_accessibility_components.py",
    "-q",
    "--tb=short",
    "-p", "no:cacheprovider",
]

# ── Helpers ───────────────────────────────────────────────────────────────


def _parse_pytest_summary(stdout: str) -> dict[str, int]:
    """Parse the summary line from ``pytest -q`` output.

    Handles both execution format (

        147 passed, 2 warnings in 18.61s

    ) and ``--collect-only`` format (

        147 tests collected in 16.53s
    ).

    Returns ``{"passed": …, "failed": …}``.  If no summary line is
    found, ``failed`` is set to ``-1`` to distinguish from a clean
    zero-failure run.
    """
    for line in stdout.splitlines():
        # Execution format: "147 passed, 2 failed, 5 warnings in 18.61s"
        m = re.search(r"(\d+)\s+passed", line)
        if m:
            passed = int(m.group(1))
            failed = 0
            fm = re.search(r"(\d+)\s+failed", line)
            if fm:
                failed = int(fm.group(1))
            return {"passed": passed, "failed": failed}
        # Collect-only format: "147 tests collected in 16.53s"
        m = re.search(r"(\d+)\s+tests?\s+collected", line)
        if m:
            count = int(m.group(1))
            return {"passed": count, "failed": 0}
    return {"passed": 0, "failed": -1}


# ── Measurement tests ─────────────────────────────────────────────────────


class TestAccessibilityCollectionTime:
    """Measure pytest collection (import) time for the accessibility tests.

    Collection exercises the full ``shopstack.ui`` import chain end-to-end:
    every screen module, component, renderer, and service module must be
    imported and registered.  A regression here typically means a new
    circular import, an unexpectedly heavy transitive dependency, or a
    change in the test discovery pattern.
    """

    def test_collection_time(self) -> None:
        """Collect-only wall-clock time should be <30s."""
        start = time.perf_counter()
        result = subprocess.run(
            ["uv", "run", "pytest", *_PYTEST_ARGS, "--collect-only"],
            capture_output=True,
            text=True,
            cwd=str(PROJECT_ROOT),
        )
        elapsed = time.perf_counter() - start

        assert result.returncode == 0, (
            f"pytest --collect-only failed:\n{result.stderr[:500]}"
        )
        assert elapsed < 30.0, (
            f"Collection took {elapsed:.2f}s — expected <30s. "
            "This may indicate a new circular import or heavy transitive dependency."
        )

        summary = _parse_pytest_summary(result.stdout)
        assert summary["passed"] >= 140, (
            f"Expected ≥140 tests, got {summary['passed']}. "
            f"Output preview:\n{result.stdout[:500]}"
        )
        assert summary["failed"] not in (-1,), (
            f"Could not parse test count from --collect-only output. "
            f"Full stdout:\n{result.stdout}"
        )


class TestAccessibilityTotalTime:
    """Measure total (collection + execution) wall-clock time for all 147+
    accessibility tests.

    This is the primary regression detector: any change that slows down
    the full test run (new imports, slower renderers, larger test data)
    will cause this benchmark to fail.
    """

    def test_total_time(self) -> None:
        """Full test-suite wall-clock time should be <45s."""
        start = time.perf_counter()
        result = subprocess.run(
            ["uv", "run", "pytest", *_PYTEST_ARGS],
            capture_output=True,
            text=True,
            cwd=str(PROJECT_ROOT),
        )
        elapsed = time.perf_counter() - start

        # Parse summary *before* the assertion so we can report both
        # timing and pass/fail in a single error message.
        summary = _parse_pytest_summary(result.stdout)

        assert result.returncode == 0, (
            f"pytest run failed (exit {result.returncode}):\n"
            f"{result.stderr[:500]}"
        )
        assert elapsed < 45.0, (
            f"Total test time {elapsed:.2f}s exceeds 45s threshold. "
            "This may indicate a performance regression in the import chain "
            "or test execution."
        )
        assert summary["passed"] >= 140, (
            f"Expected ≥140 passed, got {summary['passed']}. "
            f"Output preview:\n{result.stdout[:500]}"
        )
        assert summary["failed"] == 0, (
            f"Expected 0 failures, got {summary['failed']}. "
            "All accessibility tests must pass before timing is meaningful."
        )