tests/test_reliability.py

"""Test-retest reliability: does the same footage give the same answer twice?

Validity ("is the number right?") needs labelled ground truth, which we do not
have. RELIABILITY ("is the number stable?") needs none at all, and it is the
cheaper question — an unreliable metric cannot be valid, so this rules things
out before any labelling effort is spent.

Two failure modes are checked, and the second is the one that motivated the
file:

1. DETERMINISM — the same poses analysed twice give identical output. Cheap,
   and it protects against accidental state or ordering dependence.

2. TRIM STABILITY — the same footage analysed at different SAMPLING RATES
   gives the same answer. This matters because the trackers sample a fixed
   frame budget rather than a fixed rate, so effective fps depends on clip
   length: a user who trims a 40s clip to 20s gets a different frame rate,
   and any metric whose thresholds are expressed per FRAME rather than per
   SECOND then means something different. If a fighter's guard percentage
   changes because they trimmed their video, the number is not a measurement.

The tolerances below are deliberately loose. They are a regression guard, not
a specification — the point is that a future change cannot quietly make a
Tier-A read rate-dependent without this failing.
"""
import numpy as np
import pytest

import pose_features as pf
from test_trackers import fighter


FPS = 30.0


def moving_clip(n=900, seed=0):
    """A synthetic clip with real structure: the fighters close and separate,
    and one of them drops the rear hand once inside.

    Deterministic jitter stands in for keypoint noise, so the metrics are
    exercised against something that is not perfectly clean — a clip with zero
    noise would make every stability check pass trivially.
    """
    rng = np.random.default_rng(seed)
    poses = np.empty((n, 2, 17, 2))
    for i in range(n):
        inside = (i // 45) % 2 == 0
        a = fighter(400.0)
        b = fighter(470.0 if inside else 650.0)
        a[pf.L_WR] = (370.0, 115.0)          # lead hand stays up
        if not inside:
            a[pf.R_WR] = (430.0, 115.0)      # rear hand only up at range
        poses[i, 0] = a + rng.normal(0, 1.2, a.shape)
        poses[i, 1] = b + rng.normal(0, 1.2, b.shape)
    return poses


def analyse(poses, fps):
    feats, _ = pf._features_from_poses(poses, fps, ("A", "B"))
    assert feats, "the synthetic clip should analyze"
    return feats


# Reads the research note puts in Tier A: their margin over keypoint noise is
# wide enough that a change of sampling rate should not move them much.
STABLE_PCT_KEYS = ["hands_up_pct", "pct_in_range_with_hands_down"]


def test_the_same_poses_analysed_twice_give_the_same_answer():
    poses = moving_clip()
    assert analyse(poses, FPS)["A"] == analyse(poses, FPS)["A"]


@pytest.mark.parametrize("stride", [2, 3])
def test_tier_a_percentages_survive_a_change_of_sampling_rate(stride):
    """Decimating the clip is what trimming it does, via the frame budget."""
    poses = moving_clip()
    full = analyse(poses, FPS)["A"]
    thin = analyse(poses[::stride], FPS / stride)["A"]
    for key in STABLE_PCT_KEYS:
        a, b = full.get(key), thin.get(key)
        assert a is not None and b is not None, f"{key} vanished at stride {stride}"
        assert abs(a - b) <= 12.0, (
            f"{key} moved {abs(a - b):.1f} points between "
            f"{FPS:.0f} and {FPS / stride:.0f} fps — a user trimming their "
            f"clip would see a different number")


@pytest.mark.parametrize("stride", [2, 3])
def test_the_pressure_label_survives_a_change_of_sampling_rate(stride):
    """The label is net-decided, and net telescopes, so it should not care."""
    poses = moving_clip()
    full = analyse(poses, FPS)["A"]["pressure"]
    thin = analyse(poses[::stride], FPS / stride)["A"]["pressure"]
    assert full.split(" (")[0] == thin.split(" (")[0], (
        f"pressure label flipped between {FPS:.0f} and {FPS / stride:.0f} fps: "
        f"{full!r} vs {thin!r}")


def test_reliability_report(capsys):
    """Not an assertion — a printed per-metric report.

    Run with `-s` to read it. This is the thing to look at before promoting a
    metric out of "coaching observation": a read whose value swings here is
    not one to put a number on.
    """
    poses = moving_clip()
    rows = []
    base = analyse(poses, FPS)["A"]
    for stride in (2, 3, 4):
        thin = analyse(poses[::stride], FPS / stride)["A"]
        for key, val in sorted(base.items()):
            if not isinstance(val, (int, float)) or isinstance(val, bool):
                continue
            other = thin.get(key)
            if isinstance(other, (int, float)):
                rows.append((key, FPS / stride, val, other, abs(val - other)))

    with capsys.disabled():
        print(f"\n{'metric':34} {'fps':>5} {'full':>8} {'thin':>8} {'delta':>8}")
        for key, fps, a, b, d in rows:
            print(f"{key:34} {fps:5.1f} {a:8.2f} {b:8.2f} {d:8.2f}")