#!/usr/bin/env python3 """ Test script for validating timeout oval detection. This script tests the calibrated timeout detection system against known ground truth: 1. Calibrates at the opening kickoff time (117s) when all 6 timeouts are visible 2. Tests detection at each of the 6 known timeout timestamps 3. Tests that no false changes are detected at normal play timestamps Ground Truth Timeouts (OSU vs Tenn 12.21.24): - 4:25 (265s) - HOME timeout - 1:07:30 (4050s) - AWAY timeout - 1:09:40 (4180s) - AWAY timeout - 1:14:07 (4447s) - HOME timeout - 1:16:06 (4566s) - HOME timeout - 1:44:54 (6294s) - AWAY timeout (after halftime reset) Usage: python scripts/test_timeout_oval_detection.py """ import sys from pathlib import Path from typing import Optional, Tuple # Add project root to path project_root = Path(__file__).parent.parent sys.path.insert(0, str(project_root / "src")) import cv2 import logging from detection.models import TimeoutReading from detection.timeout_calibrator import calibrate_timeout_ovals, visualize_calibration from detection.timeouts import CalibratedTimeoutDetector from detection.models import CalibratedTimeoutRegion logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") logger = logging.getLogger(__name__) # Video path VIDEO_PATH = project_root / "full_videos" / "OSU vs Tenn 12.21.24.mkv" # Timeout region config (from saved config) HOME_TIMEOUT_REGION = (1228, 973, 32, 48) # x, y, width, height AWAY_TIMEOUT_REGION = (660, 973, 31, 47) # Calibration timestamp - use a time when scorebug is visible with all timeouts # (117s is during kickoff action with no scorebug, 150s has scorebug visible) CALIBRATION_TIMESTAMP = 150.0 # Ground truth timeouts # Note: Some timeouts near the end of the first half (1:14:07, 1:16:06) may have detection issues # because the "after" frame search can cross into halftime/second half where timeouts reset GROUND_TRUTH_TIMEOUTS = [ {"timestamp": 265, "team": "home", "label": "4:25", "before_home": 3, "before_away": 3, "after_home": 2, "after_away": 3}, {"timestamp": 4050, "team": "away", "label": "1:07:30", "before_home": 2, "before_away": 3, "after_home": 2, "after_away": 2}, {"timestamp": 4180, "team": "away", "label": "1:09:40", "before_home": 2, "before_away": 2, "after_home": 2, "after_away": 1}, {"timestamp": 4447, "team": "home", "label": "1:14:07", "before_home": 2, "before_away": 1, "after_home": 1, "after_away": 1, "near_halftime": True}, {"timestamp": 4566, "team": "home", "label": "1:16:06", "before_home": 1, "before_away": 1, "after_home": 0, "after_away": 1, "near_halftime": True}, {"timestamp": 6294, "team": "away", "label": "1:44:54", "before_home": 3, "before_away": 3, "after_home": 3, "after_away": 2}, # After halftime reset ] # Normal plays (should NOT detect timeout change) # Note: Some timestamps may have no scorebug visible (replays, commercials) # The test should skip these or handle them appropriately NORMAL_PLAYS = [ {"timestamp": 160, "label": "2:40 (early first half, scorebug visible)"}, {"timestamp": 350, "label": "5:50 (first quarter, before first timeout)"}, {"timestamp": 500, "label": "8:20 (first quarter)"}, {"timestamp": 3100, "label": "51:40 (early third quarter)"}, {"timestamp": 5800, "label": "1:36:40 (late third quarter)"}, ] def get_frame_at_timestamp(video: cv2.VideoCapture, timestamp: float): """Get a frame at a specific timestamp.""" fps = video.get(cv2.CAP_PROP_FPS) frame_num = int(timestamp * fps) video.set(cv2.CAP_PROP_POS_FRAMES, frame_num) ret, frame = video.read() if not ret: raise RuntimeError(f"Failed to read frame at timestamp {timestamp}s") return frame def calibrate_detector(video: cv2.VideoCapture) -> CalibratedTimeoutDetector: """Calibrate the timeout detector at the opening kickoff.""" logger.info("=" * 70) logger.info("CALIBRATION PHASE") logger.info("=" * 70) logger.info("Calibrating at timestamp %.1fs (opening kickoff)", CALIBRATION_TIMESTAMP) frame = get_frame_at_timestamp(video, CALIBRATION_TIMESTAMP) # Calibrate home region home_calibrated = calibrate_timeout_ovals(frame, HOME_TIMEOUT_REGION, "home", CALIBRATION_TIMESTAMP) if home_calibrated is None or len(home_calibrated.ovals) == 0: logger.error("Failed to calibrate HOME timeout region") return None # Calibrate away region away_calibrated = calibrate_timeout_ovals(frame, AWAY_TIMEOUT_REGION, "away", CALIBRATION_TIMESTAMP) if away_calibrated is None or len(away_calibrated.ovals) == 0: logger.error("Failed to calibrate AWAY timeout region") return None logger.info("HOME region: %d ovals found", len(home_calibrated.ovals)) for i, oval in enumerate(home_calibrated.ovals): logger.info(" Oval %d: pos=(%d,%d) size=%dx%d brightness=%.1f", i + 1, oval.x, oval.y, oval.width, oval.height, oval.baseline_brightness) logger.info("AWAY region: %d ovals found", len(away_calibrated.ovals)) for i, oval in enumerate(away_calibrated.ovals): logger.info(" Oval %d: pos=(%d,%d) size=%dx%d brightness=%.1f", i + 1, oval.x, oval.y, oval.width, oval.height, oval.baseline_brightness) # Create detector with calibrated regions detector = CalibratedTimeoutDetector(home_region=home_calibrated, away_region=away_calibrated) # Save visualization vis_frame = visualize_calibration(frame, home_calibrated) vis_frame = visualize_calibration(vis_frame, away_calibrated) output_path = project_root / "output" / "timeout_calibration_visualization.png" cv2.imwrite(str(output_path), vis_frame) logger.info("Saved calibration visualization to: %s", output_path) return detector def find_valid_reading( video: cv2.VideoCapture, detector: CalibratedTimeoutDetector, start_ts: float, direction: str = "forward", max_search: float = 60.0, step: float = 1.0, expected_home: Optional[int] = None, expected_away: Optional[int] = None, ) -> Tuple[Optional[TimeoutReading], Optional[float]]: """ Search for a valid scorebug reading near a timestamp. Args: video: Video capture object detector: Timeout detector start_ts: Starting timestamp direction: "forward" or "backward" max_search: Maximum seconds to search step: Step size in seconds expected_home: If provided, only accept readings matching this home count expected_away: If provided, only accept readings matching this away count Returns: Tuple of (reading, timestamp) or (None, None) if not found """ searched = 0.0 while searched <= max_search: if direction == "forward": ts = start_ts + searched else: ts = start_ts - searched if ts < 0: searched += step continue frame = get_frame_at_timestamp(video, ts) reading = detector.read_timeouts(frame) if reading.confidence >= 0.5: # If we have expected values, verify we match (to avoid crossing halftime) if expected_home is not None and reading.home_timeouts != expected_home: searched += step continue if expected_away is not None and reading.away_timeouts != expected_away: searched += step continue return reading, ts searched += step return None, None def test_ground_truth_timeouts(video: cv2.VideoCapture, detector: CalibratedTimeoutDetector) -> Tuple[int, int]: """Test detection at ground truth timeout timestamps.""" logger.info("") logger.info("=" * 70) logger.info("GROUND TRUTH TIMEOUTS (expect change)") logger.info("=" * 70) passed = 0 failed = 0 for gt in GROUND_TRUTH_TIMEOUTS: timestamp = gt["timestamp"] expected_team = gt["team"] label = gt["label"] expected_before_home = gt["before_home"] expected_before_away = gt["before_away"] expected_after_home = gt["after_home"] expected_after_away = gt["after_away"] # Find valid BEFORE reading - search backward from timeout, match expected counts reading_before, before_ts = find_valid_reading( video, detector, timestamp - 5, direction="backward", max_search=30.0, step=1.0, expected_home=expected_before_home, expected_away=expected_before_away, ) if reading_before is None: # Try without expected counts constraint reading_before, before_ts = find_valid_reading(video, detector, timestamp - 5, direction="backward", max_search=30.0, step=1.0) # Find valid AFTER reading - search forward from timeout, match expected counts reading_after, after_ts = find_valid_reading( video, detector, timestamp + 5, direction="forward", max_search=60.0, step=1.0, expected_home=expected_after_home, expected_away=expected_after_away, ) if reading_after is None: # Try without expected counts constraint reading_after, after_ts = find_valid_reading(video, detector, timestamp + 5, direction="forward", max_search=60.0, step=1.0) # Check if we got valid readings if reading_before is None or reading_after is None: logger.info( " ⚠ %s (%ds) %s: Could not find valid scorebug frames", label, timestamp, expected_team.upper(), ) failed += 1 continue # Calculate changes home_change = reading_before.home_timeouts - reading_after.home_timeouts away_change = reading_before.away_timeouts - reading_after.away_timeouts # Determine detected team detected_team = None if home_change == 1 and away_change == 0: detected_team = "home" elif away_change == 1 and home_change == 0: detected_team = "away" # Check if correct if detected_team == expected_team: status = "✓" passed += 1 else: status = "✗" failed += 1 logger.info( " %s %s (%ds) %s: before(H=%d,A=%d @%.0fs) after(H=%d,A=%d @%.0fs) -> detected=%s (expected=%s)", status, label, timestamp, expected_team.upper(), reading_before.home_timeouts, reading_before.away_timeouts, before_ts, reading_after.home_timeouts, reading_after.away_timeouts, after_ts, detected_team or "NONE", expected_team, ) return passed, failed def test_normal_plays(video: cv2.VideoCapture, detector: CalibratedTimeoutDetector) -> Tuple[int, int]: """Test that no false changes are detected at normal play timestamps.""" logger.info("") logger.info("=" * 70) logger.info("NORMAL PLAYS (expect NO change)") logger.info("=" * 70) passed = 0 failed = 0 for play in NORMAL_PLAYS: timestamp = play["timestamp"] label = play["label"] # Find valid START reading near the timestamp reading_start, start_ts = find_valid_reading(video, detector, timestamp, direction="forward", max_search=30.0, step=1.0) if reading_start is None: reading_start, start_ts = find_valid_reading(video, detector, timestamp, direction="backward", max_search=30.0, step=1.0) if reading_start is None: logger.info(" ⚠ %s: Could not find valid scorebug near start - SKIPPED", label) passed += 1 # Not a failure, just no scorebug continue # Find valid END reading - must have SAME timeout counts (no timeout during normal play) # Search forward from start, requiring same timeout counts reading_end, end_ts = find_valid_reading( video, detector, start_ts + 5, direction="forward", max_search=30.0, step=1.0, expected_home=reading_start.home_timeouts, expected_away=reading_start.away_timeouts, ) if reading_end is None: # Could not find a frame with matching counts - this means scorebug coverage ended # (e.g., crossed halftime, commercial break, etc.) # This is NOT a false positive, just incomplete coverage logger.info(" ⚠ %s: Could not find end frame with matching counts (H=%d, A=%d) - SKIPPED", label, reading_start.home_timeouts, reading_start.away_timeouts) passed += 1 # Not a failure, just no continuous scorebug coverage continue # Calculate changes home_change = reading_start.home_timeouts - reading_end.home_timeouts away_change = reading_start.away_timeouts - reading_end.away_timeouts # Check if no change (correct behavior) if home_change == 0 and away_change == 0: status = "✓" passed += 1 logger.info(" %s %s: No change detected (HOME=%d, AWAY=%d @%.0fs-%.0fs)", status, label, reading_start.home_timeouts, reading_start.away_timeouts, start_ts, end_ts) else: status = "✗" failed += 1 logger.info( " %s %s: FALSE CHANGE detected! before(H=%d,A=%d @%.0fs) after(H=%d,A=%d @%.0fs)", status, label, reading_start.home_timeouts, reading_start.away_timeouts, start_ts, reading_end.home_timeouts, reading_end.away_timeouts, end_ts, ) return passed, failed def main(): """Run the timeout detection test.""" logger.info("TIMEOUT DETECTION TEST") logger.info("Video: %s", VIDEO_PATH) if not VIDEO_PATH.exists(): logger.error("Video file not found: %s", VIDEO_PATH) return 1 video = cv2.VideoCapture(str(VIDEO_PATH)) if not video.isOpened(): logger.error("Failed to open video: %s", VIDEO_PATH) return 1 try: # Calibrate detector = calibrate_detector(video) if detector is None: logger.error("Calibration failed") return 1 # Test ground truth timeouts timeout_passed, timeout_failed = test_ground_truth_timeouts(video, detector) # Test normal plays normal_passed, normal_failed = test_normal_plays(video, detector) # Summary logger.info("") logger.info("=" * 70) logger.info("SUMMARY") logger.info("=" * 70) logger.info("Ground truth timeouts: %d/%d detected correctly", timeout_passed, timeout_passed + timeout_failed) logger.info("Normal plays: %d/%d correctly ignored", normal_passed, normal_passed + normal_failed) total_passed = timeout_passed + normal_passed total_tests = timeout_passed + timeout_failed + normal_passed + normal_failed logger.info("Overall: %d/%d tests passed", total_passed, total_tests) if total_passed == total_tests: logger.info("ALL TESTS PASSED!") return 0 else: logger.warning("SOME TESTS FAILED") return 1 finally: video.release() if __name__ == "__main__": sys.exit(main())