"""Test-retest reliability: does the same footage give the same answer twice? Validity ("is the number right?") needs labelled ground truth, which we do not have. RELIABILITY ("is the number stable?") needs none at all, and it is the cheaper question — an unreliable metric cannot be valid, so this rules things out before any labelling effort is spent. Two failure modes are checked, and the second is the one that motivated the file: 1. DETERMINISM — the same poses analysed twice give identical output. Cheap, and it protects against accidental state or ordering dependence. 2. TRIM STABILITY — the same footage analysed at different SAMPLING RATES gives the same answer. This matters because the trackers sample a fixed frame budget rather than a fixed rate, so effective fps depends on clip length: a user who trims a 40s clip to 20s gets a different frame rate, and any metric whose thresholds are expressed per FRAME rather than per SECOND then means something different. If a fighter's guard percentage changes because they trimmed their video, the number is not a measurement. The tolerances below are deliberately loose. They are a regression guard, not a specification — the point is that a future change cannot quietly make a Tier-A read rate-dependent without this failing. """ import numpy as np import pytest import pose_features as pf from test_trackers import fighter FPS = 30.0 def moving_clip(n=900, seed=0): """A synthetic clip with real structure: the fighters close and separate, and one of them drops the rear hand once inside. Deterministic jitter stands in for keypoint noise, so the metrics are exercised against something that is not perfectly clean — a clip with zero noise would make every stability check pass trivially. """ rng = np.random.default_rng(seed) poses = np.empty((n, 2, 17, 2)) for i in range(n): inside = (i // 45) % 2 == 0 a = fighter(400.0) b = fighter(470.0 if inside else 650.0) a[pf.L_WR] = (370.0, 115.0) # lead hand stays up if not inside: a[pf.R_WR] = (430.0, 115.0) # rear hand only up at range poses[i, 0] = a + rng.normal(0, 1.2, a.shape) poses[i, 1] = b + rng.normal(0, 1.2, b.shape) return poses def analyse(poses, fps): feats, _ = pf._features_from_poses(poses, fps, ("A", "B")) assert feats, "the synthetic clip should analyze" return feats # Reads the research note puts in Tier A: their margin over keypoint noise is # wide enough that a change of sampling rate should not move them much. STABLE_PCT_KEYS = ["hands_up_pct", "pct_in_range_with_hands_down"] def test_the_same_poses_analysed_twice_give_the_same_answer(): poses = moving_clip() assert analyse(poses, FPS)["A"] == analyse(poses, FPS)["A"] @pytest.mark.parametrize("stride", [2, 3]) def test_tier_a_percentages_survive_a_change_of_sampling_rate(stride): """Decimating the clip is what trimming it does, via the frame budget.""" poses = moving_clip() full = analyse(poses, FPS)["A"] thin = analyse(poses[::stride], FPS / stride)["A"] for key in STABLE_PCT_KEYS: a, b = full.get(key), thin.get(key) assert a is not None and b is not None, f"{key} vanished at stride {stride}" assert abs(a - b) <= 12.0, ( f"{key} moved {abs(a - b):.1f} points between " f"{FPS:.0f} and {FPS / stride:.0f} fps — a user trimming their " f"clip would see a different number") @pytest.mark.parametrize("stride", [2, 3]) def test_the_pressure_label_survives_a_change_of_sampling_rate(stride): """The label is net-decided, and net telescopes, so it should not care.""" poses = moving_clip() full = analyse(poses, FPS)["A"]["pressure"] thin = analyse(poses[::stride], FPS / stride)["A"]["pressure"] assert full.split(" (")[0] == thin.split(" (")[0], ( f"pressure label flipped between {FPS:.0f} and {FPS / stride:.0f} fps: " f"{full!r} vs {thin!r}") def test_reliability_report(capsys): """Not an assertion — a printed per-metric report. Run with `-s` to read it. This is the thing to look at before promoting a metric out of "coaching observation": a read whose value swings here is not one to put a number on. """ poses = moving_clip() rows = [] base = analyse(poses, FPS)["A"] for stride in (2, 3, 4): thin = analyse(poses[::stride], FPS / stride)["A"] for key, val in sorted(base.items()): if not isinstance(val, (int, float)) or isinstance(val, bool): continue other = thin.get(key) if isinstance(other, (int, float)): rows.append((key, FPS / stride, val, other, abs(val - other))) with capsys.disabled(): print(f"\n{'metric':34} {'fps':>5} {'full':>8} {'thin':>8} {'delta':>8}") for key, fps, a, b, d in rows: print(f"{key:34} {fps:5.1f} {a:8.2f} {b:8.2f} {d:8.2f}")