import time import numpy as np import cv2 from typing import List, Tuple, Sequence, Any from numpy import ndarray FOOTBALL_KEYPOINTS: list[tuple[int, int]] = [ (0, 0), # 1 (0, 0), # 2 (0, 0), # 3 (0, 0), # 4 (0, 0), # 5 (0, 0), # 6 (0, 0), # 7 (0, 0), # 8 (0, 0), # 9 (0, 0), # 10 (0, 0), # 11 (0, 0), # 12 (0, 0), # 13 (0, 0), # 14 (527, 283), # 15 (527, 403), # 16 (0, 0), # 17 (0, 0), # 18 (0, 0), # 19 (0, 0), # 20 (0, 0), # 21 (0, 0), # 22 (0, 0), # 23 (0, 0), # 24 (0, 0), # 25 (0, 0), # 26 (0, 0), # 27 (0, 0), # 28 (0, 0), # 29 (0, 0), # 30 (405, 340), # 31 (645, 340), # 32 ] def convert_keypoints_to_val_format(keypoints): return [tuple(int(x) for x in pair) for pair in keypoints] def validate_with_nearby_keypoints( kp_idx: int, kp: tuple[int, int], valid_indices: list[int], result: list[tuple[int, int]], template_keypoints: list[tuple[int, int]], scale_factor: float = None, ) -> float: """ Validate a keypoint by checking distances to nearby keypoints on the same side. Returns validation score (lower is better), or None if validation not possible. """ template_kp = template_keypoints[kp_idx] # Define which keypoints are on the same side # Left side: 10, 11, 12, 13 (indices 9, 10, 11, 12) # Right side: 18, 19, 20, 21, 22, 23, 24, 25-30 (indices 17-29) left_side_indices = [9, 10, 11, 12] # Keypoints 10-13 right_side_indices = list(range(17, 30)) # Keypoints 18-30 # Determine which side this keypoint should be on if kp_idx in left_side_indices: same_side_indices = left_side_indices elif kp_idx in right_side_indices: same_side_indices = right_side_indices else: return None # Can't validate # Find nearby keypoints on the same side that are detected nearby_kps = [] for nearby_idx in same_side_indices: if nearby_idx != kp_idx and nearby_idx in valid_indices: nearby_kp = result[nearby_idx] nearby_template_kp = template_keypoints[nearby_idx] nearby_kps.append((nearby_idx, nearby_kp, nearby_template_kp)) if len(nearby_kps) == 0: return None # No nearby keypoints to validate with # Calculate distance errors to nearby keypoints distance_errors = [] for nearby_idx, nearby_kp, nearby_template_kp in nearby_kps: # Detected distance detected_dist = np.sqrt((kp[0] - nearby_kp[0])**2 + (kp[1] - nearby_kp[1])**2) # Template distance template_dist = np.sqrt((template_kp[0] - nearby_template_kp[0])**2 + (template_kp[1] - nearby_template_kp[1])**2) if template_dist > 0: # Expected detected distance if scale_factor: expected_dist = template_dist * scale_factor else: expected_dist = template_dist if expected_dist > 0: # Normalized error error = abs(detected_dist - expected_dist) / expected_dist distance_errors.append(error) if len(distance_errors) > 0: return np.mean(distance_errors) return None def remove_duplicate_detections( keypoints: list[tuple[int, int]], frame_width: int = None, frame_height: int = None, ) -> list[tuple[int, int]]: """ Remove duplicate/conflicting keypoint detections using distance-based validation. Uses the principle that if two keypoints are detected very close together, but in the template they should be far apart, one of them is likely wrong. Validates each keypoint by checking if its distances to other keypoints match the expected template distances. Args: keypoints: List of 32 keypoints frame_width: Optional frame width for validation frame_height: Optional frame height for validation Returns: Cleaned list of keypoints with duplicates removed """ if len(keypoints) != 32: if len(keypoints) < 32: keypoints = list(keypoints) + [(0, 0)] * (32 - len(keypoints)) else: keypoints = keypoints[:32] result = list(keypoints) try: from keypoint_evaluation import TEMPLATE_KEYPOINTS template_available = True except ImportError: template_available = False if not template_available: return result # Get all valid detected keypoints valid_indices = [] for i in range(32): if result[i][0] > 0 and result[i][1] > 0: valid_indices.append(i) if len(valid_indices) < 2: return result # Calculate scale factor from detected keypoints to template # Use pairs of keypoints that are far apart in template to estimate scale scale_factor = None if len(valid_indices) >= 2: max_template_dist = 0 max_detected_dist = 0 for i in range(len(valid_indices)): for j in range(i + 1, len(valid_indices)): idx_i = valid_indices[i] idx_j = valid_indices[j] template_i = TEMPLATE_KEYPOINTS[idx_i] template_j = TEMPLATE_KEYPOINTS[idx_j] template_dist = np.sqrt((template_i[0] - template_j[0])**2 + (template_i[1] - template_j[1])**2) kp_i = result[idx_i] kp_j = result[idx_j] detected_dist = np.sqrt((kp_i[0] - kp_j[0])**2 + (kp_i[1] - kp_j[1])**2) if template_dist > max_template_dist and detected_dist > 0: max_template_dist = template_dist max_detected_dist = detected_dist if max_template_dist > 0 and max_detected_dist > 0: scale_factor = max_detected_dist / max_template_dist # For each keypoint, validate it by checking distances to other keypoints keypoint_scores = {} for idx in valid_indices: kp = result[idx] template_kp = TEMPLATE_KEYPOINTS[idx] # Calculate how well this keypoint's distances match template distances distance_errors = [] num_comparisons = 0 for other_idx in valid_indices: if other_idx == idx: continue other_kp = result[other_idx] other_template_kp = TEMPLATE_KEYPOINTS[other_idx] # Calculate detected distance detected_dist = np.sqrt((kp[0] - other_kp[0])**2 + (kp[1] - other_kp[1])**2) # Calculate template distance template_dist = np.sqrt((template_kp[0] - other_template_kp[0])**2 + (template_kp[1] - other_template_kp[1])**2) if template_dist > 50: # Only check keypoints that should be reasonably far apart num_comparisons += 1 # Expected detected distance (scaled from template) if scale_factor: expected_dist = template_dist * scale_factor else: expected_dist = template_dist # Calculate error (normalized) if expected_dist > 0: error = abs(detected_dist - expected_dist) / expected_dist distance_errors.append(error) # Score: lower is better (smaller distance errors) if num_comparisons > 0: avg_error = np.mean(distance_errors) keypoint_scores[idx] = avg_error else: keypoint_scores[idx] = 0.0 # Find pairs of keypoints that are too close but should be far apart conflicts = [] for i in range(len(valid_indices)): for j in range(i + 1, len(valid_indices)): idx_i = valid_indices[i] idx_j = valid_indices[j] kp_i = result[idx_i] kp_j = result[idx_j] # Calculate detected distance detected_dist = np.sqrt((kp_i[0] - kp_j[0])**2 + (kp_i[1] - kp_j[1])**2) # Calculate template distance template_i = TEMPLATE_KEYPOINTS[idx_i] template_j = TEMPLATE_KEYPOINTS[idx_j] template_dist = np.sqrt((template_i[0] - template_j[0])**2 + (template_i[1] - template_j[1])**2) # If template distance is large but detected distance is small, it's a conflict if template_dist > 100 and detected_dist < 30: # Enhanced validation: use nearby keypoints to determine which is correct # For example, if we have 24 and 29, we can check distances to determine if it's 13 or 21 score_i = keypoint_scores.get(idx_i, 1.0) score_j = keypoint_scores.get(idx_j, 1.0) # Try to validate using nearby keypoints on the same side # Keypoint 13 is on left side, keypoint 21 is on right side # If we have right-side keypoints (like 24, 29), check distances nearby_validation_i = validate_with_nearby_keypoints( idx_i, kp_i, valid_indices, result, TEMPLATE_KEYPOINTS, scale_factor ) nearby_validation_j = validate_with_nearby_keypoints( idx_j, kp_j, valid_indices, result, TEMPLATE_KEYPOINTS, scale_factor ) # Prioritize nearby validation: if one has nearby validation and the other doesn't, # prefer the one with nearby validation (it's more reliable) validation_score_i = score_i validation_score_j = score_j if nearby_validation_i is not None and nearby_validation_j is not None: # Both have nearby validation, use those scores validation_score_i = nearby_validation_i validation_score_j = nearby_validation_j elif nearby_validation_i is not None: # Only i has nearby validation, prefer it (give it much better score) validation_score_i = nearby_validation_i validation_score_j = score_j + 1.0 # Penalize j for not having nearby validation elif nearby_validation_j is not None: # Only j has nearby validation, prefer it validation_score_i = score_i + 1.0 # Penalize i for not having nearby validation validation_score_j = nearby_validation_j # If neither has nearby validation, use general distance scores # Remove the one with worse validation score if validation_score_i > validation_score_j: conflicts.append((idx_i, idx_j, validation_score_i, validation_score_j)) else: conflicts.append((idx_j, idx_i, validation_score_j, validation_score_i)) # Remove conflicting keypoints (keep the one with better score) removed_indices = set() for remove_idx, keep_idx, remove_score, keep_score in conflicts: if remove_idx not in removed_indices: print(f"Removing duplicate detection: keypoint {remove_idx+1} at {result[remove_idx]} conflicts with keypoint {keep_idx+1} at {result[keep_idx]} " f"(detected distance: {np.sqrt((result[remove_idx][0] - result[keep_idx][0])**2 + (result[remove_idx][1] - result[keep_idx][1])**2):.1f}, " f"template distance: {np.sqrt((TEMPLATE_KEYPOINTS[remove_idx][0] - TEMPLATE_KEYPOINTS[keep_idx][0])**2 + (TEMPLATE_KEYPOINTS[remove_idx][1] - TEMPLATE_KEYPOINTS[keep_idx][1])**2):.1f}). " f"Keeping keypoint {keep_idx+1} (score: {keep_score:.3f} vs {remove_score:.3f}).") result[remove_idx] = (0, 0) removed_indices.add(remove_idx) return result def calculate_missing_keypoints( keypoints: list[tuple[int, int]], frame_width: int = None, frame_height: int = None, ) -> list[tuple[int, int]]: """ Calculate missing keypoint coordinates for multiple cases: 1. Given keypoints 14, 15, 16 (and possibly 17), and either 31 or 32, calculate the missing center circle point (32 or 31). 2. Given three or four of keypoints 18, 19, 20, 21 and any of 22-30, calculate missing keypoint positions (like 22 or others) to prevent warping failures. Args: keypoints: List of 32 keypoints (some may be (0,0) if missing) frame_width: Optional frame width for validation frame_height: Optional frame height for validation Returns: Updated list of 32 keypoints with calculated missing keypoints filled in """ if len(keypoints) != 32: # Pad or truncate to 32 if len(keypoints) < 32: keypoints = list(keypoints) + [(0, 0)] * (32 - len(keypoints)) else: keypoints = keypoints[:32] result = list(keypoints) # Helper to get keypoint def get_kp(kp_idx): if kp_idx < 0 or kp_idx >= 32: return None x, y = result[kp_idx] if x == 0 and y == 0: return None return (x, y) # Case 1: Find center x-coordinate from center line keypoints (14, 15, 16, or 17) # Keypoints 14, 15, 16, 17 are on the center vertical line (indices 13, 14, 15, 16) center_x = None for center_kp_idx in [13, 14, 15, 16]: # 14, 15, 16, 17 (0-indexed) kp = get_kp(center_kp_idx) if kp: center_x = kp[0] break # If we have center line, calculate missing center circle point if center_x is not None: # Keypoint 31 is at index 30 (left side of center circle) # Keypoint 32 is at index 31 (right side of center circle) kp_31 = get_kp(30) # Keypoint 31 kp_32 = get_kp(31) # Keypoint 32 if kp_31 and not kp_32: # Given 31, calculate 32 by reflecting across center_x # Formula: x_32 = center_x + (center_x - x_31) = 2*center_x - x_31 # y_32 = y_31 (same y-coordinate, both on center horizontal line) dx = center_x - kp_31[0] result[31] = (int(round(center_x + dx)), kp_31[1]) elif kp_32 and not kp_31: # Given 32, calculate 31 by reflecting across center_x # Formula: x_31 = center_x - (x_32 - center_x) = 2*center_x - x_32 # y_31 = y_32 (same y-coordinate, both on center horizontal line) dx = kp_32[0] - center_x result[30] = (int(round(center_x - dx)), kp_32[1]) # Case 1.5: Unified handling of left side keypoints (1-13) # Three parallel vertical lines on left side: # - Line 1-6: keypoints 1, 2, 3, 4, 5, 6 (indices 0-5) # - Line 7-8: keypoints 7, 8 (indices 6-7) # - Line 10-13: keypoints 10, 11, 12, 13 (indices 9-12) # Keypoint 9 (index 8) is between line 1-6 and line 10-13 # Collect all left-side keypoints (1-13, indices 0-12, excluding 9 which is center) left_side_all = [] line_1_6_points = [] # Indices 0-5 line_7_8_points = [] # Indices 6-7 line_10_13_points = [] # Indices 9-12 for idx in range(0, 13): # Keypoints 1-13 (indices 0-12) if idx == 8: # Skip keypoint 9 (index 8) - it's a center point continue kp = get_kp(idx) if kp: left_side_all.append((idx, kp)) if 0 <= idx <= 5: # Line 1-6 line_1_6_points.append((idx, kp)) elif 6 <= idx <= 7: # Line 7-8 line_7_8_points.append((idx, kp)) elif 9 <= idx <= 12: # Line 10-13 line_10_13_points.append((idx, kp)) kp_9 = get_kp(8) # Keypoint 9 if kp_9: left_side_all.append((8, kp_9)) total_left_side_count = len(left_side_all) # If we have 6 or more points, no need to calculate more if total_left_side_count >= 6: pass # Don't calculate more points elif total_left_side_count == 5: # Check if 4 points are on one line and 1 on another line counts_per_line = [ len(line_1_6_points), len(line_7_8_points), len(line_10_13_points) ] if max(counts_per_line) == 4 and sum(counts_per_line) == 4: # 4 points on one line, need to calculate 1 more point on another line # Determine which line has 4 points and calculate on a different line if len(line_1_6_points) == 4: # All 4 on line 1-6, calculate on line 10-13 or 7-8 # Prefer line 10-13 (right edge of left side) if len(line_10_13_points) == 0: # Calculate a point on line 10-13 # Fit line through 1-6 points points_1_6 = np.array([[kp[0], kp[1]] for _, kp in line_1_6_points]) x_coords = points_1_6[:, 0] y_coords = points_1_6[:, 1] A = np.vstack([x_coords, np.ones(len(x_coords))]).T m_1_6, b_1_6 = np.linalg.lstsq(A, y_coords, rcond=None)[0] # Calculate a point on line 10-13 (parallel to 1-6) # Use template y-coordinate for one of 10-13 points template_ys_10_13 = [140, 270, 410, 540] # Template y for 10-13 template_indices_10_13 = [9, 10, 11, 12] # Use median y from 1-6 points to estimate scale median_y = np.median(y_coords) # Calculate x using parallel line geometry # In template: line 10-13 is at x=165, line 1-6 is at x=5 # Ratio: 165/5 = 33 if abs(m_1_6) > 1e-6: x_on_line_1_6 = (median_y - b_1_6) / m_1_6 x_new = int(round(x_on_line_1_6 * 33)) else: x_new = int(round(np.median(x_coords) * 33)) # Find first missing index in 10-13 range for template_y, idx in zip(template_ys_10_13, template_indices_10_13): if result[idx] is None: result[idx] = (x_new, int(round(median_y))) break elif len(line_10_13_points) == 4: # All 4 on line 10-13, calculate on line 1-6 # Similar logic but in reverse points_10_13 = np.array([[kp[0], kp[1]] for _, kp in line_10_13_points]) x_coords = points_10_13[:, 0] y_coords = points_10_13[:, 1] A = np.vstack([x_coords, np.ones(len(x_coords))]).T m_10_13, b_10_13 = np.linalg.lstsq(A, y_coords, rcond=None)[0] # Calculate a point on line 1-6 template_ys_1_6 = [5, 140, 250, 430, 540, 675] # Template y for 1-6 template_indices_1_6 = [0, 1, 2, 3, 4, 5] median_y = np.median(y_coords) # Calculate x using parallel line geometry # Ratio: 5/165 ≈ 0.0303 if abs(m_10_13) > 1e-6: x_on_line_10_13 = (median_y - b_10_13) / m_10_13 x_new = int(round(x_on_line_10_13 * 0.0303)) else: x_new = int(round(np.median(x_coords) * 0.0303)) for template_y, idx in zip(template_ys_1_6, template_indices_1_6): if result[idx] is None: result[idx] = (x_new, int(round(median_y))) break elif total_left_side_count < 5: # Need to calculate missing keypoints to get exactly 5 points # Requirements: # 1. Must have keypoint 9 (if possible) # 2. 4 points shouldn't be all on one line (need distribution) # Template coordinates for reference template_coords_left = { 0: (5, 5), # 1 1: (5, 140), # 2 2: (5, 250), # 3 3: (5, 430), # 4 4: (5, 540), # 5 5: (5, 675), # 6 6: (55, 250), # 7 7: (55, 430), # 8 8: (110, 340), # 9 (what we're calculating) 9: (165, 140), # 10 10: (165, 270), # 11 11: (165, 410), # 12 12: (165, 540), # 13 } # Define line groups (vertical and horizontal lines) # Vertical lines: 1-6, 7-8, 10-13 # Horizontal lines: 2-10, 3-7, 4-8, 5-13 line_groups_left = { '1-6': ([0, 1, 2, 3, 4, 5], 'vertical'), # indices: 1, 2, 3, 4, 5, 6 '7-8': ([6, 7], 'vertical'), # indices: 7, 8 '10-13': ([9, 10, 11, 12], 'vertical'), # indices: 10, 11, 12, 13 '2-10': ([1, 9], 'horizontal'), # indices: 2, 10 '3-7': ([2, 6], 'horizontal'), # indices: 3, 7 '4-8': ([3, 7], 'horizontal'), # indices: 4, 8 '5-13': ([4, 12], 'horizontal'), # indices: 5, 13 } # Collect all available points with their indices all_available_points_left = {} for idx, kp in line_1_6_points: all_available_points_left[idx] = kp for idx, kp in line_7_8_points: all_available_points_left[idx] = kp for idx, kp in line_10_13_points: all_available_points_left[idx] = kp # Step 1: Find the best vertical line and best horizontal line separately best_vertical_line_name_left = None best_vertical_line_points_left = [] max_vertical_points_left = 1 best_horizontal_line_name_left = None best_horizontal_line_points_left = [] max_horizontal_points_left = 1 for line_name, (indices, line_type) in line_groups_left.items(): line_points = [(idx, all_available_points_left[idx]) for idx in indices if idx in all_available_points_left] if line_type == 'vertical' and len(line_points) > max_vertical_points_left: max_vertical_points_left = len(line_points) best_vertical_line_name_left = line_name best_vertical_line_points_left = line_points elif line_type == 'horizontal' and len(line_points) > max_horizontal_points_left: max_horizontal_points_left = len(line_points) best_horizontal_line_name_left = line_name best_horizontal_line_points_left = line_points # Check and calculate missing points on detected lines # For vertical lines if best_vertical_line_name_left is not None: expected_indices = line_groups_left[best_vertical_line_name_left][0] detected_indices = {idx for idx, _ in best_vertical_line_points_left} missing_indices = [idx for idx in expected_indices if idx not in detected_indices] if len(missing_indices) > 0: # Calculate missing points using template ratios template_start = template_coords_left[best_vertical_line_points_left[0][0]] template_end = template_coords_left[best_vertical_line_points_left[-1][0]] frame_start = best_vertical_line_points_left[0][1] frame_end = best_vertical_line_points_left[-1][1] for missing_idx in missing_indices: template_missing = template_coords_left[missing_idx] # Calculate ratio along the line based on y-coordinate (vertical line) template_y_start = template_start[1] template_y_end = template_end[1] template_y_missing = template_missing[1] if abs(template_y_end - template_y_start) > 1e-6: ratio = (template_y_missing - template_y_start) / (template_y_end - template_y_start) else: ratio = 0.5 # Calculate frame coordinates x_new = frame_start[0] + (frame_end[0] - frame_start[0]) * ratio y_new = frame_start[1] + (frame_end[1] - frame_start[1]) * ratio new_point = (int(round(x_new)), int(round(y_new))) # Add to result and update collections result[missing_idx] = new_point best_vertical_line_points_left.append((missing_idx, new_point)) all_available_points_left[missing_idx] = new_point total_left_side_count += 1 max_vertical_points_left = len(best_vertical_line_points_left) # Sort by index to maintain order best_vertical_line_points_left.sort(key=lambda x: x[0]) # Check if we can now form a horizontal line with the newly calculated points for line_name, (indices, line_type) in line_groups_left.items(): if line_type == 'horizontal': line_points = [(idx, all_available_points_left[idx]) for idx in indices if idx in all_available_points_left] if len(line_points) > max_horizontal_points_left: max_horizontal_points_left = len(line_points) best_horizontal_line_name_left = line_name best_horizontal_line_points_left = line_points # For horizontal lines if best_horizontal_line_name_left is not None: expected_indices = line_groups_left[best_horizontal_line_name_left][0] detected_indices = {idx for idx, _ in best_horizontal_line_points_left} missing_indices = [idx for idx in expected_indices if idx not in detected_indices] if len(missing_indices) > 0: # Calculate missing points using template ratios template_start = template_coords_left[best_horizontal_line_points_left[0][0]] template_end = template_coords_left[best_horizontal_line_points_left[-1][0]] frame_start = best_horizontal_line_points_left[0][1] frame_end = best_horizontal_line_points_left[-1][1] for missing_idx in missing_indices: template_missing = template_coords_left[missing_idx] # Calculate ratio along the line based on x-coordinate (horizontal line) template_x_start = template_start[0] template_x_end = template_end[0] template_x_missing = template_missing[0] if abs(template_x_end - template_x_start) > 1e-6: ratio = (template_x_missing - template_x_start) / (template_x_end - template_x_start) else: ratio = 0.5 # Calculate frame coordinates x_new = frame_start[0] + (frame_end[0] - frame_start[0]) * ratio y_new = frame_start[1] + (frame_end[1] - frame_start[1]) * ratio new_point = (int(round(x_new)), int(round(y_new))) # Add to result and update collections result[missing_idx] = new_point best_horizontal_line_points_left.append((missing_idx, new_point)) all_available_points_left[missing_idx] = new_point total_left_side_count += 1 max_horizontal_points_left = len(best_horizontal_line_points_left) # Sort by index to maintain order best_horizontal_line_points_left.sort(key=lambda x: x[0]) # Check if we can now form a vertical line with the newly calculated points for line_name, (indices, line_type) in line_groups_left.items(): if line_type == 'vertical': line_points = [(idx, all_available_points_left[idx]) for idx in indices if idx in all_available_points_left] if len(line_points) > max_vertical_points_left: max_vertical_points_left = len(line_points) best_vertical_line_name_left = line_name best_vertical_line_points_left = line_points # If we only have one direction, try to calculate the other direction line # Similar logic to right side, adapted for left side structure if best_vertical_line_name_left is not None and best_horizontal_line_name_left is None: # We have vertical line but no horizontal line # Find an off-line point (not on the vertical line) off_line_point = None off_line_idx = None vertical_line_indices = line_groups_left[best_vertical_line_name_left][0] for idx, kp in all_available_points_left.items(): if idx not in vertical_line_indices: off_line_point = kp off_line_idx = idx break if off_line_point is not None: # Convert off_line_point to numpy array for arithmetic operations off_line_point = np.array(off_line_point) # Project off_line_point onto vertical line template_off_line = template_coords_left[off_line_idx] template_vertical_start_index = best_vertical_line_points_left[0][0] template_vertical_end_index = best_vertical_line_points_left[-1][0] template_vertical_start = template_coords_left[template_vertical_start_index] template_vertical_end = template_coords_left[template_vertical_end_index] # Project at same y as off_line_point template_y_off = template_off_line[1] template_y_vertical_start = template_vertical_start[1] template_y_vertical_end = template_vertical_end[1] if abs(template_y_vertical_end - template_y_vertical_start) > 1e-6: ratio_proj = (template_y_off - template_y_vertical_start) / (template_y_vertical_end - template_y_vertical_start) else: ratio_proj = 0.5 frame_vertical_start = best_vertical_line_points_left[0][1] frame_vertical_end = best_vertical_line_points_left[-1][1] proj_x = frame_vertical_start[0] + (frame_vertical_end[0] - frame_vertical_start[0]) * ratio_proj proj_y = frame_vertical_start[1] + (frame_vertical_end[1] - frame_vertical_start[1]) * ratio_proj proj_point = np.array([proj_x, proj_y]) # Calculate horizontal line points based on which vertical line we have if best_vertical_line_name_left == '10-13': # Line 10-13: can calculate points on horizontal lines 2-10, 5-13 if off_line_idx == 1: # Point 2 (index 1) is off-line, calculate point 10 (index 9) kp_10 = np.array(best_vertical_line_points_left[0][1]) # 10 point kp_2 = off_line_point + (kp_10 - proj_point) result[1] = tuple(kp_2.astype(int)) total_left_side_count += 1 all_available_points_left[1] = tuple(kp_2.astype(int)) elif off_line_idx == 4: # Point 5 (index 4) is off-line, calculate point 13 (index 12) kp_13 = np.array(best_vertical_line_points_left[-1][1]) # 13 point kp_5 = off_line_point + (kp_13 - proj_point) result[4] = tuple(kp_5.astype(int)) total_left_side_count += 1 all_available_points_left[4] = tuple(kp_5.astype(int)) elif best_vertical_line_name_left == '1-6': # Line 1-6: can calculate points on horizontal lines 2-10, 3-7, 4-8, 5-13 if off_line_idx == 6 or off_line_idx == 7: # Point 7 or 8 is off-line, calculate point 3 or 4 template_off = template_coords_left[off_line_idx] template_3 = template_coords_left[2] # 3 point, index 2 template_4 = template_coords_left[3] # 4 point, index 3 template_7 = template_coords_left[6] # 7 point, index 6 template_8 = template_coords_left[7] # 8 point, index 7 if off_line_idx == 6: # Point 7, calculate point 3 ratio = (template_3[0] - template_7[0]) / (template_7[0] - template_off[0]) if abs(template_7[0] - template_off[0]) > 1e-6 else 0.5 kp_3 = proj_point + (off_line_point - proj_point) * ratio result[2] = tuple(kp_3.astype(int)) total_left_side_count += 1 all_available_points_left[2] = tuple(kp_3.astype(int)) else: # Point 8, calculate point 4 ratio = (template_4[0] - template_8[0]) / (template_8[0] - template_off[0]) if abs(template_8[0] - template_off[0]) > 1e-6 else 0.5 kp_4 = proj_point + (off_line_point - proj_point) * ratio result[3] = tuple(kp_4.astype(int)) total_left_side_count += 1 all_available_points_left[3] = tuple(kp_4.astype(int)) elif off_line_idx == 9 or off_line_idx == 12: # Point 10 or 13 is off-line, calculate point 2 or 5 if off_line_idx == 9: # Point 10, calculate point 2 kp_2 = off_line_point + (np.array(best_vertical_line_points_left[1][1]) - proj_point) result[1] = tuple(kp_2.astype(int)) total_left_side_count += 1 all_available_points_left[1] = tuple(kp_2.astype(int)) else: # Point 13, calculate point 5 kp_5 = off_line_point + (np.array(best_vertical_line_points_left[4][1]) - proj_point) result[4] = tuple(kp_5.astype(int)) total_left_side_count += 1 all_available_points_left[4] = tuple(kp_5.astype(int)) elif best_vertical_line_name_left == '7-8': # Line 7-8: can calculate points on horizontal lines 3-7, 4-8 if off_line_idx == 2 or off_line_idx == 3: # Point 3 or 4 is off-line, calculate point 7 or 8 if off_line_idx == 2: # Point 3, calculate point 7 kp_7 = off_line_point + (np.array(best_vertical_line_points_left[0][1]) - proj_point) result[6] = tuple(kp_7.astype(int)) total_left_side_count += 1 all_available_points_left[6] = tuple(kp_7.astype(int)) else: # Point 4, calculate point 8 kp_8 = off_line_point + (np.array(best_vertical_line_points_left[-1][1]) - proj_point) result[7] = tuple(kp_8.astype(int)) total_left_side_count += 1 all_available_points_left[7] = tuple(kp_8.astype(int)) # Check if we can now form a horizontal line with the newly calculated points for line_name, (indices, line_type) in line_groups_left.items(): if line_type == 'horizontal': line_points = [(idx, all_available_points_left[idx]) for idx in indices if idx in all_available_points_left] if len(line_points) > max_horizontal_points_left: max_horizontal_points_left = len(line_points) best_horizontal_line_name_left = line_name best_horizontal_line_points_left = line_points elif best_horizontal_line_name_left is not None and best_vertical_line_name_left is None: # We have horizontal line but no vertical line # Find an off-line point (not on the horizontal line) off_line_point = None off_line_idx = None horizontal_line_indices = line_groups_left[best_horizontal_line_name_left][0] for idx, kp in all_available_points_left.items(): if idx not in horizontal_line_indices: off_line_point = kp off_line_idx = idx break if off_line_point is not None: # Project off_line_point onto horizontal line template_off_line = template_coords_left[off_line_idx] template_horizontal_start = template_coords_left[best_horizontal_line_points_left[0][0]] template_horizontal_end = template_coords_left[best_horizontal_line_points_left[-1][0]] # Project at same x as off_line_point template_x_off = template_off_line[0] template_x_horizontal_start = template_horizontal_start[0] template_x_horizontal_end = template_horizontal_end[0] if abs(template_x_horizontal_end - template_x_horizontal_start) > 1e-6: ratio_proj = (template_x_off - template_x_horizontal_start) / (template_x_horizontal_end - template_x_horizontal_start) else: ratio_proj = 0.5 frame_horizontal_start = best_horizontal_line_points_left[0][1] frame_horizontal_end = best_horizontal_line_points_left[-1][1] proj_x = frame_horizontal_start[0] + (frame_horizontal_end[0] - frame_horizontal_start[0]) * ratio_proj proj_y = frame_horizontal_start[1] + (frame_horizontal_end[1] - frame_horizontal_start[1]) * ratio_proj proj_point = np.array([proj_x, proj_y]) off_line_point = np.array(off_line_point) # Calculate vertical line points based on which horizontal line we have if best_horizontal_line_name_left == '2-10': # Line 2-10: can calculate points on vertical lines 1-6, 10-13 if off_line_idx == 0 or off_line_idx == 5: # Point 1 or 6 is off-line, calculate point 2 kp_2 = off_line_point + (np.array(best_horizontal_line_points_left[0][1]) - proj_point) result[1] = tuple(kp_2.astype(int)) total_left_side_count += 1 all_available_points_left[1] = tuple(kp_2.astype(int)) elif off_line_idx == 9 or off_line_idx == 12: # Point 10 or 13 is off-line, calculate point 10 kp_10 = off_line_point + (np.array(best_horizontal_line_points_left[-1][1]) - proj_point) result[9] = tuple(kp_10.astype(int)) total_left_side_count += 1 all_available_points_left[9] = tuple(kp_10.astype(int)) elif best_horizontal_line_name_left == '3-7': # Line 3-7: can calculate points on vertical lines 1-6, 7-8 if off_line_idx == 0 or off_line_idx == 5: # Point 1 or 6 is off-line, calculate point 3 kp_3 = off_line_point + (np.array(best_horizontal_line_points_left[0][1]) - proj_point) result[2] = tuple(kp_3.astype(int)) total_left_side_count += 1 all_available_points_left[2] = tuple(kp_3.astype(int)) elif off_line_idx == 6 or off_line_idx == 7: # Point 7 or 8 is off-line, calculate point 7 kp_7 = off_line_point + (np.array(best_horizontal_line_points_left[-1][1]) - proj_point) result[6] = tuple(kp_7.astype(int)) total_left_side_count += 1 all_available_points_left[6] = tuple(kp_7.astype(int)) elif best_horizontal_line_name_left == '4-8': # Line 4-8: can calculate points on vertical lines 1-6, 7-8 if off_line_idx == 0 or off_line_idx == 5: # Point 1 or 6 is off-line, calculate point 4 kp_4 = off_line_point + (np.array(best_horizontal_line_points_left[0][1]) - proj_point) result[3] = tuple(kp_4.astype(int)) total_left_side_count += 1 all_available_points_left[3] = tuple(kp_4.astype(int)) elif off_line_idx == 6 or off_line_idx == 7: # Point 7 or 8 is off-line, calculate point 8 kp_8 = off_line_point + (np.array(best_horizontal_line_points_left[-1][1]) - proj_point) result[7] = tuple(kp_8.astype(int)) total_left_side_count += 1 all_available_points_left[7] = tuple(kp_8.astype(int)) elif best_horizontal_line_name_left == '5-13': # Line 5-13: can calculate points on vertical lines 1-6, 10-13 if off_line_idx == 0 or off_line_idx == 5: # Point 1 or 6 is off-line, calculate point 5 kp_5 = off_line_point + (np.array(best_horizontal_line_points_left[0][1]) - proj_point) result[4] = tuple(kp_5.astype(int)) total_left_side_count += 1 all_available_points_left[4] = tuple(kp_5.astype(int)) elif off_line_idx == 9 or off_line_idx == 12: # Point 10 or 13 is off-line, calculate point 13 kp_13 = off_line_point + (np.array(best_horizontal_line_points_left[-1][1]) - proj_point) result[12] = tuple(kp_13.astype(int)) total_left_side_count += 1 all_available_points_left[12] = tuple(kp_13.astype(int)) # Check if we can now form a vertical line with the newly calculated points for line_name, (indices, line_type) in line_groups_left.items(): if line_type == 'vertical': line_points = [(idx, all_available_points_left[idx]) for idx in indices if idx in all_available_points_left] if len(line_points) > max_vertical_points_left: max_vertical_points_left = len(line_points) best_vertical_line_name_left = line_name best_vertical_line_points_left = line_points # Calculate keypoint 9 if we have at least one line if best_vertical_line_name_left is not None and best_horizontal_line_name_left is not None: if kp_9 is None: print(f"Calculating keypoint 9 using both vertical and horizontal lines: {best_vertical_line_name_left} and {best_horizontal_line_name_left}") template_x_9 = 110 template_y_9 = 340 # Project keypoint 9 onto vertical line template_vertical_start = template_coords_left[best_vertical_line_points_left[0][0]] template_vertical_end = template_coords_left[best_vertical_line_points_left[-1][0]] # Project at y=340 (same y as keypoint 9) template_y_vertical_start = template_vertical_start[1] template_y_vertical_end = template_vertical_end[1] if abs(template_y_vertical_end - template_y_vertical_start) > 1e-6: ratio_9_vertical = (template_y_9 - template_y_vertical_start) / (template_y_vertical_end - template_y_vertical_start) else: ratio_9_vertical = 0.5 frame_vertical_start = best_vertical_line_points_left[0][1] frame_vertical_end = best_vertical_line_points_left[-1][1] proj_9_on_vertical_x = frame_vertical_start[0] + (frame_vertical_end[0] - frame_vertical_start[0]) * ratio_9_vertical proj_9_on_vertical_y = frame_vertical_start[1] + (frame_vertical_end[1] - frame_vertical_start[1]) * ratio_9_vertical proj_9_on_vertical = (proj_9_on_vertical_x, proj_9_on_vertical_y) # Project keypoint 9 onto horizontal line template_horizontal_start = template_coords_left[best_horizontal_line_points_left[0][0]] template_horizontal_end = template_coords_left[best_horizontal_line_points_left[-1][0]] # Project at x=110 (same x as keypoint 9) template_x_horizontal_start = template_horizontal_start[0] template_x_horizontal_end = template_horizontal_end[0] if abs(template_x_horizontal_end - template_x_horizontal_start) > 1e-6: ratio_9_horizontal = (template_x_9 - template_x_horizontal_start) / (template_x_horizontal_end - template_x_horizontal_start) else: ratio_9_horizontal = 0.5 frame_horizontal_start = best_horizontal_line_points_left[0][1] frame_horizontal_end = best_horizontal_line_points_left[-1][1] proj_9_on_horizontal_x = frame_horizontal_start[0] + (frame_horizontal_end[0] - frame_horizontal_start[0]) * ratio_9_horizontal proj_9_on_horizontal_y = frame_horizontal_start[1] + (frame_horizontal_end[1] - frame_horizontal_start[1]) * ratio_9_horizontal proj_9_on_horizontal = (proj_9_on_horizontal_x, proj_9_on_horizontal_y) # Calculate keypoint 9 as intersection of two lines # Line 1: Passes through proj_9_on_vertical, parallel to best_horizontal_line # Line 2: Passes through proj_9_on_horizontal, parallel to best_vertical_line # Calculate direction vector of best_horizontal_line horizontal_dir_x = frame_horizontal_end[0] - frame_horizontal_start[0] horizontal_dir_y = frame_horizontal_end[1] - frame_horizontal_start[1] horizontal_dir_length = np.sqrt(horizontal_dir_x**2 + horizontal_dir_y**2) # Calculate direction vector of best_vertical_line vertical_dir_x = frame_vertical_end[0] - frame_vertical_start[0] vertical_dir_y = frame_vertical_end[1] - frame_vertical_start[1] vertical_dir_length = np.sqrt(vertical_dir_x**2 + vertical_dir_y**2) if horizontal_dir_length > 1e-6 and vertical_dir_length > 1e-6: # Normalize direction vectors horizontal_dir_x /= horizontal_dir_length horizontal_dir_y /= horizontal_dir_length vertical_dir_x /= vertical_dir_length vertical_dir_y /= vertical_dir_length # Find intersection: proj_9_on_vertical + t * horizontal_dir = proj_9_on_horizontal + s * vertical_dir A = np.array([ [horizontal_dir_x, -vertical_dir_x], [horizontal_dir_y, -vertical_dir_y] ]) b = np.array([ proj_9_on_horizontal[0] - proj_9_on_vertical[0], proj_9_on_horizontal[1] - proj_9_on_vertical[1] ]) try: t, s = np.linalg.solve(A, b) # Calculate intersection point using line 1 x_9 = proj_9_on_vertical[0] + t * horizontal_dir_x y_9 = proj_9_on_vertical[1] + t * horizontal_dir_y result[8] = (int(round(x_9)), int(round(y_9))) total_left_side_count += 1 except np.linalg.LinAlgError: # Lines are parallel or nearly parallel, use simple intersection x_9 = proj_9_on_vertical[0] y_9 = proj_9_on_horizontal[1] result[8] = (int(round(x_9)), int(round(y_9))) total_left_side_count += 1 else: # Fallback: use simple intersection x_9 = proj_9_on_vertical[0] y_9 = proj_9_on_horizontal[1] result[8] = (int(round(x_9)), int(round(y_9))) total_left_side_count += 1 print(f"total_left_side_count: {total_left_side_count}, result: {result}") if total_left_side_count > 5: pass # Continue to right side logic # Calculate m_line and b_line from best vertical or horizontal line for use in calculating other points m_line_left = None b_line_left = None best_line_for_calc_left = None best_line_type_for_calc_left = None if best_vertical_line_name_left is not None and len(best_vertical_line_points_left) >= 2: best_line_for_calc_left = best_vertical_line_points_left best_line_type_for_calc_left = 'vertical' points_array = np.array([[kp[0], kp[1]] for _, kp in best_vertical_line_points_left]) x_coords = points_array[:, 0] y_coords = points_array[:, 1] A = np.vstack([x_coords, np.ones(len(x_coords))]).T m_line_left, b_line_left = np.linalg.lstsq(A, y_coords, rcond=None)[0] elif best_horizontal_line_name_left is not None and len(best_horizontal_line_points_left) >= 2: best_line_for_calc_left = best_horizontal_line_points_left best_line_type_for_calc_left = 'horizontal' points_array = np.array([[kp[0], kp[1]] for _, kp in best_horizontal_line_points_left]) x_coords = points_array[:, 0] y_coords = points_array[:, 1] A = np.vstack([x_coords, np.ones(len(x_coords))]).T m_line_left, b_line_left = np.linalg.lstsq(A, y_coords, rcond=None)[0] # Calculate missing points to reach exactly 5 points # Ensure 4 points aren't all on one line if total_left_side_count < 5 and (m_line_left is not None or (best_line_for_calc_left is not None and best_line_type_for_calc_left == 'vertical')): # Check current distribution counts_per_line = [ len(line_1_6_points), len(line_7_8_points), len(line_10_13_points) ] # Calculate points on line 1-6 if needed template_ys_1_6 = [5, 140, 250, 430, 540, 675] template_indices_1_6 = [0, 1, 2, 3, 4, 5] if best_vertical_line_name_left == '10-13': # Construct parallel line 1-6 from line 10-13 for template_y, idx in zip(template_ys_1_6, template_indices_1_6): if result[idx] is None and total_left_side_count < 5: # Check if adding this point would put 4 on one line new_counts = counts_per_line.copy() new_counts[0] += 1 # Adding to line 1-6 if max(new_counts) >= 4 and total_left_side_count == 4: # Would have 4 on one line, skip continue # Calculate y using scale from template ref_ys = [kp[1] for _, kp in line_10_13_points] ref_template_ys = [140, 270, 410, 540] ref_indices = [9, 10, 11, 12] matched_template_ys = [] for ref_idx, ref_kp in line_10_13_points: if ref_idx in ref_indices: template_idx = ref_indices.index(ref_idx) matched_template_ys.append((ref_template_ys[template_idx], ref_kp[1])) if len(matched_template_ys) >= 1: ref_template_y, ref_frame_y = matched_template_ys[0] if ref_template_y > 0: scale = ref_frame_y / ref_template_y y_new = int(round(template_y * scale)) else: y_new = ref_frame_y else: y_new = int(round(np.median(ref_ys))) if ref_ys else template_y # Calculate x using parallel line geometry if abs(m_line_left) > 1e-6: x_on_line_10_13 = (y_new - b_line_left) / m_line_left x_new = int(round(x_on_line_10_13 * 0.0303)) # 5/165 else: x_new = int(round(np.median([kp[0] for _, kp in line_10_13_points]) * 0.0303)) result[idx] = (x_new, y_new) total_left_side_count += 1 if total_left_side_count >= 5: break elif best_vertical_line_name_left == '1-6': # Calculate missing points on line 1-6 for template_y, idx in zip(template_ys_1_6, template_indices_1_6): if result[idx] is None and total_left_side_count < 5: # Check if adding this point would put 4 on one line new_counts = counts_per_line.copy() new_counts[0] += 1 # Adding to line 1-6 if max(new_counts) >= 4 and total_left_side_count == 4: # Would have 4 on one line, skip continue # Calculate x on the line if abs(m_line_left) > 1e-6: x_new = (template_y - b_line_left) / m_line_left else: x_new = np.median([kp[0] for _, kp in line_1_6_points]) # Scale y based on available points ref_ys = [kp[1] for _, kp in line_1_6_points] ref_template_ys = [] for ref_idx, _ in line_1_6_points: if ref_idx in template_indices_1_6: template_idx = template_indices_1_6.index(ref_idx) ref_template_ys.append(template_ys_1_6[template_idx]) if len(ref_ys) >= 1 and len(ref_template_ys) >= 1: ref_template_y = ref_template_ys[0] ref_frame_y = ref_ys[0] if ref_template_y > 0: scale = ref_frame_y / ref_template_y y_new = int(round(template_y * scale)) else: y_new = ref_frame_y else: y_new = int(round(np.median(ref_ys))) if ref_ys else template_y result[idx] = (int(round(x_new)), y_new) total_left_side_count += 1 if total_left_side_count >= 5: break print(f"total_left_side_count: {total_left_side_count}, result: {result}") # Case 2: Unified handling of right side keypoints (18-30) # Three parallel lines on right side: # - Line 18-21: keypoints 18, 19, 20, 21 (indices 17-20) # - Line 23-24: keypoints 23, 24 (indices 22-23) # - Line 25-30: keypoints 25, 26, 27, 28, 29, 30 (indices 24-29) # Keypoint 22 (index 21) is between line 18-21 and line 25-30 # Collect all right-side keypoints (18-30, indices 17-29) right_side_all = [] line_18_21_points = [] # Indices 17-20 line_23_24_points = [] # Indices 22-23 line_25_30_points = [] # Indices 24-29 for idx in range(17, 30): # Keypoints 18-30 (indices 17-29) kp = get_kp(idx) if kp: right_side_all.append((idx, kp)) if 17 <= idx <= 20: # Line 18-21 line_18_21_points.append((idx, kp)) elif 22 <= idx <= 23: # Line 23-24 line_23_24_points.append((idx, kp)) elif 24 <= idx <= 29: # Line 25-30 line_25_30_points.append((idx, kp)) kp_22 = get_kp(21) # Keypoint 22 if kp_22: right_side_all.append((21, kp_22)) total_right_side_count = len(right_side_all) # If we have 6 or more points, no need to calculate more if total_right_side_count >= 6: pass # Don't calculate more points elif total_right_side_count == 5: # Check if 4 points are on one line and 1 on another line counts_per_line = [ len(line_18_21_points), len(line_23_24_points), len(line_25_30_points) ] if max(counts_per_line) == 4 and sum(counts_per_line) == 4: # 4 points on one line, need to calculate 1 more point on another line # Determine which line has 4 points and calculate on a different line if len(line_18_21_points) == 4: # All 4 on line 18-21, calculate on line 25-30 or 23-24 # Prefer line 25-30 (right edge) if len(line_25_30_points) == 0: # Calculate a point on line 25-30 # Fit line through 18-21 points points_18_21 = np.array([[kp[0], kp[1]] for _, kp in line_18_21_points]) x_coords = points_18_21[:, 0] y_coords = points_18_21[:, 1] A = np.vstack([x_coords, np.ones(len(x_coords))]).T m_18_21, b_18_21 = np.linalg.lstsq(A, y_coords, rcond=None)[0] # Calculate a point on line 25-30 (parallel to 18-21) # Use template y-coordinate for one of 25-30 points template_ys_25_30 = [5, 140, 250, 430, 540, 675] # Template y for 25-30 template_indices_25_30 = [24, 25, 26, 27, 28, 29] # Use median y from 18-21 points to estimate scale median_y = np.median(y_coords) # Find closest template y ref_template_y = min(template_ys_25_30, key=lambda ty: abs(ty - np.median([kp[1] for _, kp in line_18_21_points]))) ref_idx = template_ys_25_30.index(ref_template_y) # Calculate y for the new point y_new = int(round(median_y)) # Calculate x using parallel line geometry # In template: line 25-30 is at x=1045, line 18-21 is at x=888 # Ratio: 1045/888 ≈ 1.177 if abs(m_18_21) > 1e-6: x_on_line_18_21 = (y_new - b_18_21) / m_18_21 x_new = int(round(x_on_line_18_21 * 1.177)) else: x_new = int(round(np.median(x_coords) * 1.177)) # Find first missing index in 25-30 range for template_y, idx in zip(template_ys_25_30, template_indices_25_30): if result[idx] is None: result[idx] = (x_new, y_new) break elif len(line_25_30_points) == 4: # All 4 on line 25-30, calculate on line 18-21 # Similar logic but in reverse points_25_30 = np.array([[kp[0], kp[1]] for _, kp in line_25_30_points]) x_coords = points_25_30[:, 0] y_coords = points_25_30[:, 1] A = np.vstack([x_coords, np.ones(len(x_coords))]).T m_25_30, b_25_30 = np.linalg.lstsq(A, y_coords, rcond=None)[0] # Calculate a point on line 18-21 template_ys_18_21 = [140, 270, 410, 540] # Template y for 18-21 template_indices_18_21 = [17, 18, 19, 20] median_y = np.median(y_coords) # Calculate x using parallel line geometry # Ratio: 888/1045 ≈ 0.850 if abs(m_25_30) > 1e-6: x_on_line_25_30 = (median_y - b_25_30) / m_25_30 x_new = int(round(x_on_line_25_30 * 0.850)) else: x_new = int(round(np.median(x_coords) * 0.850)) for template_y, idx in zip(template_ys_18_21, template_indices_18_21): if result[idx] is None: result[idx] = (x_new, int(round(median_y))) break elif total_right_side_count < 5: # Need to calculate missing keypoints to get exactly 5 points # Requirements: # 1. Must have keypoint 22 # 2. 4 points shouldn't be all on one line (need distribution) # Template coordinates for reference template_coords = { 17: (888, 140), # 18 18: (888, 270), # 19 19: (888, 410), # 20 20: (888, 540), # 21 21: (940, 340), # 22 (what we're calculating) 22: (998, 250), # 23 23: (998, 430), # 24 24: (1045, 5), # 25 25: (1045, 140), # 26 26: (1045, 250), # 27 27: (1045, 430), # 28 28: (1045, 540), # 29 29: (1045, 675), # 30 } # Define line groups (vertical and horizontal lines) # Vertical lines: 18-21, 23-24, 25-30 # Horizontal lines: 18-26, 23-27, 24-28, 21-29 line_groups = { '18-21': ([17, 18, 19, 20], 'vertical'), # indices: 18, 19, 20, 21 '23-24': ([22, 23], 'vertical'), # indices: 23, 24 '25-30': ([24, 25, 26, 27, 28, 29], 'vertical'), # indices: 25, 26, 27, 28, 29, 30 '18-26': ([17, 25], 'horizontal'), # indices: 18, 26 '23-27': ([22, 26], 'horizontal'), # indices: 23, 27 '24-28': ([23, 27], 'horizontal'), # indices: 24, 28 '21-29': ([20, 28], 'horizontal'), # indices: 21, 29 } # Collect all available points with their indices all_available_points = {} for idx, kp in line_18_21_points: all_available_points[idx] = kp for idx, kp in line_23_24_points: all_available_points[idx] = kp for idx, kp in line_25_30_points: all_available_points[idx] = kp # Step 1: Find the best vertical line and best horizontal line separately best_vertical_line_name = None best_vertical_line_points = [] max_vertical_points = 1 best_horizontal_line_name = None best_horizontal_line_points = [] max_horizontal_points = 1 for line_name, (indices, line_type) in line_groups.items(): line_points = [(idx, all_available_points[idx]) for idx in indices if idx in all_available_points] if line_type == 'vertical' and len(line_points) > max_vertical_points: max_vertical_points = len(line_points) best_vertical_line_name = line_name best_vertical_line_points = line_points elif line_type == 'horizontal' and len(line_points) > max_horizontal_points: max_horizontal_points = len(line_points) best_horizontal_line_name = line_name best_horizontal_line_points = line_points # Check and calculate missing points on detected lines # For vertical lines if best_vertical_line_name is not None: expected_indices = line_groups[best_vertical_line_name][0] detected_indices = {idx for idx, _ in best_vertical_line_points} missing_indices = [idx for idx in expected_indices if idx not in detected_indices] if len(missing_indices) > 0: # Calculate missing points using template ratios template_start = template_coords[best_vertical_line_points[0][0]] template_end = template_coords[best_vertical_line_points[-1][0]] frame_start = best_vertical_line_points[0][1] frame_end = best_vertical_line_points[-1][1] for missing_idx in missing_indices: template_missing = template_coords[missing_idx] # Calculate ratio along the line based on y-coordinate (vertical line) template_y_start = template_start[1] template_y_end = template_end[1] template_y_missing = template_missing[1] if abs(template_y_end - template_y_start) > 1e-6: ratio = (template_y_missing - template_y_start) / (template_y_end - template_y_start) else: ratio = 0.5 # Calculate frame coordinates x_new = frame_start[0] + (frame_end[0] - frame_start[0]) * ratio y_new = frame_start[1] + (frame_end[1] - frame_start[1]) * ratio new_point = (int(round(x_new)), int(round(y_new))) # Add to result and update collections result[missing_idx] = new_point best_vertical_line_points.append((missing_idx, new_point)) all_available_points[missing_idx] = new_point total_right_side_count += 1 max_vertical_points = len(best_vertical_line_points) # Sort by index to maintain order best_vertical_line_points.sort(key=lambda x: x[0]) # Check if we can now form a horizontal line with the newly calculated points for line_name, (indices, line_type) in line_groups.items(): if line_type == 'horizontal': line_points = [(idx, all_available_points[idx]) for idx in indices if idx in all_available_points] if len(line_points) > max_horizontal_points: max_horizontal_points = len(line_points) best_horizontal_line_name = line_name best_horizontal_line_points = line_points # For horizontal lines if best_horizontal_line_name is not None: expected_indices = line_groups[best_horizontal_line_name][0] detected_indices = {idx for idx, _ in best_horizontal_line_points} missing_indices = [idx for idx in expected_indices if idx not in detected_indices] if len(missing_indices) > 0: # Calculate missing points using template ratios template_start = template_coords[best_horizontal_line_points[0][0]] template_end = template_coords[best_horizontal_line_points[-1][0]] frame_start = best_horizontal_line_points[0][1] frame_end = best_horizontal_line_points[-1][1] for missing_idx in missing_indices: template_missing = template_coords[missing_idx] # Calculate ratio along the line based on x-coordinate (horizontal line) template_x_start = template_start[0] template_x_end = template_end[0] template_x_missing = template_missing[0] if abs(template_x_end - template_x_start) > 1e-6: ratio = (template_x_missing - template_x_start) / (template_x_end - template_x_start) else: ratio = 0.5 # Calculate frame coordinates x_new = frame_start[0] + (frame_end[0] - frame_start[0]) * ratio y_new = frame_start[1] + (frame_end[1] - frame_start[1]) * ratio new_point = (int(round(x_new)), int(round(y_new))) # Add to result and update collections result[missing_idx] = new_point best_horizontal_line_points.append((missing_idx, new_point)) all_available_points[missing_idx] = new_point total_right_side_count += 1 max_horizontal_points = len(best_horizontal_line_points) # Sort by index to maintain order best_horizontal_line_points.sort(key=lambda x: x[0]) # Check if we can now form a vertical line with the newly calculated points for line_name, (indices, line_type) in line_groups.items(): if line_type == 'vertical': line_points = [(idx, all_available_points[idx]) for idx in indices if idx in all_available_points] if len(line_points) > max_vertical_points: max_vertical_points = len(line_points) best_vertical_line_name = line_name best_vertical_line_points = line_points # If we only have one direction, try to calculate the other direction line if best_vertical_line_name is not None and best_horizontal_line_name is None: # possible cases: # line is 25-30 and off line point is 19, then we can calculate 18 so get horizontal line 18-26 # line is 25-30 and off line point is 20, then we can calculate 18 so get horizontal line 18-26 # line is 18-21 and off line point is 23, then we can calculate 27 so get horizontal line 23-27 # line is 18-21 and off line point is 24, then we can calculate 28 so get horizontal line 24-28 # line is 18-21 and off line point is 25, then we can calculate 26 so get horizontal line 18-26 # line is 18-21 and off line point is 27, then we can calculate 26 so get horizontal line 18-26 # line is 18-21 and off line point is 28, then we can calculate 29 so get horizontal line 21-29 # line is 18-21 and off line point is 30, then we can calculate 29 so get horizontal line 21-29 # line is 23-24 and off line point is 18, then we can calculate 26 so get horizontal line 18-26 # line is 23-24 and off line point is 19, then we can calculate 18 so get horizontal line 18-26 # line is 23-24 and off line point is 20, then we can calculate 21 so get horizontal line 21-29 # line is 23-24 and off line point is 21, then we can calculate 29 so get horizontal line 21-29 # line is 23-24 and off line point is 25, then we can calculate 27 so get horizontal line 23-27 # line is 23-24 and off line point is 26, then we can calculate 27 so get horizontal line 23-27 # line is 23-24 and off line point is 29, then we can calculate 28 so get horizontal line 24-28 # line is 23-24 and off line point is 30, then we can calculate 28 so get horizontal line 24-28 # We have vertical line but no horizontal line # Find an off-line point (not on the vertical line) off_line_point = None off_line_idx = None vertical_line_indices = line_groups[best_vertical_line_name][0] for idx, kp in all_available_points.items(): if idx not in vertical_line_indices: off_line_point = kp off_line_idx = idx break if off_line_point is not None: # Convert off_line_point to numpy array for arithmetic operations off_line_point = np.array(off_line_point) # Project off_line_point onto vertical line template_off_line = template_coords[off_line_idx] template_vertical_start_index = best_vertical_line_points[0][0] template_vertical_end_index = best_vertical_line_points[-1][0] template_vertical_start = template_coords[template_vertical_start_index] template_vertical_end = template_coords[template_vertical_end_index] # Project at same y as off_line_point template_y_off = template_off_line[1] template_y_vertical_start = template_vertical_start[1] template_y_vertical_end = template_vertical_end[1] if abs(template_y_vertical_end - template_y_vertical_start) > 1e-6: ratio_proj = (template_y_off - template_y_vertical_start) / (template_y_vertical_end - template_y_vertical_start) else: ratio_proj = 0.5 frame_vertical_start = best_vertical_line_points[0][1] frame_vertical_end = best_vertical_line_points[-1][1] proj_x = frame_vertical_start[0] + (frame_vertical_end[0] - frame_vertical_start[0]) * ratio_proj proj_y = frame_vertical_start[1] + (frame_vertical_end[1] - frame_vertical_start[1]) * ratio_proj proj_point = np.array([proj_x, proj_y]) if best_vertical_line_name == '25-30' and len(best_vertical_line_points) == 6: if off_line_idx == 18 or off_line_idx == 19: # 19 or 20 point is off line point, so we can calculate 18 kp_26 = np.array(best_vertical_line_points[1][1]) # 26 point kp_18 = off_line_point + (kp_26 - proj_point) result[17] = tuple(kp_18.astype(int)) total_right_side_count += 1 all_available_points[17] = tuple(kp_18.astype(int)) # 18 point is now available, index is 17 if best_vertical_line_name == '18-21' and len(best_vertical_line_points) == 4: if off_line_idx == 22 or off_line_idx == 23: # 23 or 24 point is off line point, so we can calculate 27 template_19 = template_coords[18] # 19 point, index is 18 template_23 = template_coords[22] # 23 point, index is 22 template_27 = template_coords[26] # 27 point, index is 26 ratio = (template_27[0] - template_19[0]) / (template_23[0] - template_19[0]) # ratio in x coordinates because y coordinates are the same expected_point = proj_point + (off_line_point - proj_point) * ratio if off_line_idx == 22: result[26] = tuple(expected_point.astype(int)) # 27 point, index is 26 total_right_side_count += 1 all_available_points[26] = tuple(expected_point.astype(int)) # 27 point is now available, index is 26 else: result[27] = tuple(expected_point.astype(int)) # 28 point, index is 27 total_right_side_count += 1 all_available_points[27] = tuple(expected_point.astype(int)) # 28 point is now available, index is 27 if off_line_idx == 24 or off_line_idx == 26: # 25 or 27 point is off line point, so we can calculate 26 kp_18 = np.array(best_vertical_line_points[0][1]) # 18 point kp_26 = off_line_point + (kp_18 - proj_point) result[25] = tuple(kp_26.astype(int)) total_right_side_count += 1 all_available_points[25] = tuple(kp_26.astype(int)) # 26 point is now available, index is 25 if off_line_idx == 27 or off_line_idx == 29: # 28 or 30 point is off line point, so we can calculate 29 kp_21 = np.array(best_vertical_line_points[-1][1]) # 21 point kp_29 = off_line_point + (kp_21 - proj_point) result[28] = tuple(kp_29.astype(int)) total_right_side_count += 1 all_available_points[28] = tuple(kp_29.astype(int)) # 29 point is now available, index is 28 if best_vertical_line_name == '23-24' and len(best_vertical_line_points) == 2: if off_line_idx == 17 or off_line_idx == 18 or off_line_idx == 19 or off_line_idx == 20: # 18 or 19 or 20 or 21 point is off line point, so we can calculate 26 template_18 = template_coords[17] # 18 point, index is 17 template_26 = template_coords[25] # 26 point, index is 25 template_23 = template_coords[22] # 23 point, index is 22 ratio_26 = (template_26[0] - template_18[0]) / (template_23[0] - template_18[0]) # ratio in x coordinates because y coordinates are the same kp_18 = None if off_line_idx == 17: kp_18 = off_line_point elif off_line_idx == 18 or off_line_idx == 19 or off_line_idx == 20: template_off_line = template_coords[off_line_idx] ratio = (template_18[1] - template_off_line[1]) / (template_23[1] - template_off_line[1]) kp_18 = off_line_point + (np.array(best_vertical_line_points[0][1]) - proj_point) * ratio if kp_18 is not None: kp_26 = kp_18 + (proj_point - off_line_point) * ratio_26 result[25] = tuple(kp_26.astype(int)) total_right_side_count += 1 all_available_points[25] = tuple(kp_26.astype(int)) # 26 point is now available, index is 25 if off_line_idx == 24 or off_line_idx == 25: # 25 or 26 point is off line point, so we can calculate 27 kp_27 = off_line_point + (np.array(best_vertical_line_points[0][1]) - proj_point) result[26] = tuple(kp_27.astype(int)) total_right_side_count += 1 all_available_points[26] = tuple(kp_27.astype(int)) # 27 point is now available, index is 26 if off_line_idx == 28 or off_line_idx == 29: # 29 or 30 point is off line point, so we can calculate 29 kp_29 = off_line_point + (np.array(best_vertical_line_points[-1][1]) - proj_point) result[28] = tuple(kp_29.astype(int)) total_right_side_count += 1 all_available_points[28] = tuple(kp_29.astype(int)) # 29 point is now available, index is 28 # Check if we can now form a horizontal line with the newly calculated points for line_name, (indices, line_type) in line_groups.items(): if line_type == 'horizontal': line_points = [(idx, all_available_points[idx]) for idx in indices if idx in all_available_points] if len(line_points) > max_horizontal_points: max_horizontal_points = len(line_points) best_horizontal_line_name = line_name best_horizontal_line_points = line_points elif best_horizontal_line_name is not None and best_vertical_line_name is None: # possible cases: # line is 18-26 and off line point is 23, then we can calculate 27 so get vertical line 25-30 # line is 18-26 and off line point is 24, then we can calculate 28 so get vertical line 25-30 # line is 23-27 and off line point is 18, then we can calculate 26 so get vertical line 25-30 # line is 23-27 and off line point is 19, then we can calculate 18 so get vertical line 18-21 # line is 23-27 and off line point is 20, then we can calculate 18 so get vertical line 18-21 # line is 23-27 and off line point is 21, then we can calculate 29 so get vertical line 25-30 # line is 24-28 and off line point is 18, then we can calculate 26 so get vertical line 25-30 # line is 24-28 and off line point is 19, then we can calculate 21 so get vertical line 18-21 # line is 24-28 and off line point is 20, then we can calculate 21 so get vertical line 18-21 # line is 24-28 and off line point is 21, then we can calculate 29 so get vertical line 25-30 # line is 21-29 and off line point is 23, then we can calculate 27 so get vertical line 25-30 # line is 21-29 and off line point is 24, then we can calculate 28 so get vertical line 25-30 # We have horizontal line but no vertical line # Find an off-line point (not on the horizontal line) off_line_point = None off_line_idx = None horizontal_line_indices = line_groups[best_horizontal_line_name][0] for idx, kp in all_available_points.items(): if idx not in horizontal_line_indices: off_line_point = kp off_line_idx = idx break if off_line_point is not None: # Project off_line_point onto horizontal line template_off_line = template_coords[off_line_idx] template_horizontal_start = template_coords[best_horizontal_line_points[0][0]] template_horizontal_end = template_coords[best_horizontal_line_points[-1][0]] # Project at same x as off_line_point template_x_off = template_off_line[0] template_x_horizontal_start = template_horizontal_start[0] template_x_horizontal_end = template_horizontal_end[0] if abs(template_x_horizontal_end - template_x_horizontal_start) > 1e-6: ratio_proj = (template_x_off - template_x_horizontal_start) / (template_x_horizontal_end - template_x_horizontal_start) else: ratio_proj = 0.5 frame_horizontal_start = best_horizontal_line_points[0][1] frame_horizontal_end = best_horizontal_line_points[-1][1] proj_x = frame_horizontal_start[0] + (frame_horizontal_end[0] - frame_horizontal_start[0]) * ratio_proj proj_y = frame_horizontal_start[1] + (frame_horizontal_end[1] - frame_horizontal_start[1]) * ratio_proj proj_point = np.array([proj_x, proj_y]) if best_horizontal_line_name == '18-26': if off_line_idx == 22 or off_line_idx == 23: # 23 or 24 point is off line point, so we can calculate 27 or 28 template_18 = template_coords[best_horizontal_line_points[0][0]] # 18 point, index is 17 template_26 = template_coords[best_horizontal_line_points[-1][0]] # 26 point, index is 25 template_23 = template_coords[off_line_idx] # 23 or 24 point, index is 22 or 23 ratio_26 = (template_26[0] - template_23[0]) / (template_26[0] - template_18[0]) # ratio in x coordinates because y coordinates are the same detected_point = off_line_point + (np.array(best_horizontal_line_points[-1][1]) - np.array(best_horizontal_line_points[0][1])) * ratio_26 if off_line_idx == 22: result[26] = tuple(detected_point.astype(int)) total_right_side_count += 1 all_available_points[26] = tuple(detected_point.astype(int)) # 26 point is now available, index is 26 else: result[27] = tuple(detected_point.astype(int)) total_right_side_count += 1 all_available_points[27] = tuple(detected_point.astype(int)) # 27 point is now available, index is 27 if best_horizontal_line_name == '23-27': if off_line_idx == 17 or off_line_idx == 20: template_18 = template_coords[17] # 18 point, index is 17 template_26 = template_coords[25] # 26 point, index is 25 template_23 = template_coords[best_horizontal_line_points[0][0]] # 23 , index is 22 ratio_26 = (template_26[0] - template_18[0]) / (template_26[0] - template_23[0]) # ratio in x coordinates because y coordinates are the same detected_point = off_line_point + (np.array(best_horizontal_line_points[-1][1]) - np.array(best_horizontal_line_points[0][1])) * ratio_26 if off_line_idx == 17: result[25] = tuple(detected_point.astype(int)) total_right_side_count += 1 all_available_points[25] = tuple(detected_point.astype(int)) # 26 point is now available, index is 25 else: result[28] = tuple(detected_point.astype(int)) total_right_side_count += 1 all_available_points[28] = tuple(detected_point.astype(int)) # 29 point is now available, index is 28 if off_line_idx == 18 or off_line_idx == 19: # 19 or 20 point is off line point, so we can calculate 18 template_18 = template_coords[17] # 18 point, index is 17 template_off_line = template_coords[off_line_idx] template_23 = template_coords[best_horizontal_line_points[0][0]] # 23 point, index is 22 ratio = (template_off_line[1] - template_18[1]) / (template_off_line[1] - template_23[1]) kp_18 = off_line_point + (proj_point - off_line_point) * ratio result[17] = tuple(kp_18.astype(int)) total_right_side_count += 1 all_available_points[17] = tuple(kp_18.astype(int)) # 18 point is now available, index is 17 if best_horizontal_line_name == '24-28': if off_line_idx == 17 or off_line_idx == 20: template_18 = template_coords[17] # 18 point, index is 17 template_26 = template_coords[25] # 26 point, index is 25 template_24 = template_coords[best_horizontal_line_points[0][0]] # 24 , index is 23 ratio_26 = (template_26[0] - template_18[0]) / (template_26[0] - template_24[0]) # ratio in x coordinates because y coordinates are the same detected_point = off_line_point + (np.array(best_horizontal_line_points[-1][1]) - np.array(best_horizontal_line_points[0][1])) * ratio_26 if off_line_idx == 17: result[25] = tuple(detected_point.astype(int)) total_right_side_count += 1 all_available_points[25] = tuple(detected_point.astype(int)) # 26 point is now available, index is 25 else: result[28] = tuple(detected_point.astype(int)) total_right_side_count += 1 all_available_points[28] = tuple(detected_point.astype(int)) # 29 point is now available, index is 28 if off_line_idx == 18 or off_line_idx == 19: # 19 or 20 point is off line point, so we can calculate 18 template_21 = template_coords[20] # 21 point, index is 20 template_off_line = template_coords[off_line_idx] template_24 = template_coords[best_horizontal_line_points[0][0]] # 24 point, index is 23 ratio = (template_21[1] - template_off_line[1]) / (template_24[1] - template_off_line[1]) kp_21 = off_line_point + (proj_point - off_line_point) * ratio result[20] = tuple(kp_18.astype(int)) total_right_side_count += 1 all_available_points[20] = tuple(kp_18.astype(int)) # 21 point is now available, index is 20 if best_horizontal_line_name == '21-29': if off_line_idx == 22 or off_line_idx == 23: # 23 or 24 point is off line point, so we can calculate 27 or 28 template_21 = template_coords[best_horizontal_line_points[0][0]] # 21 point, index is 20 template_29 = template_coords[best_horizontal_line_points[-1][0]] # 29 point, index is 28 template_23 = template_coords[off_line_idx] # 23 or 24 point, index is 22 or 23 ratio_29 = (template_29[0] - template_23[0]) / (template_29[0] - template_21[0]) # ratio in x coordinates because y coordinates are the same detected_point = off_line_point + (np.array(best_horizontal_line_points[-1][1]) - np.array(best_horizontal_line_points[0][1])) * ratio_29 if off_line_idx == 22: result[26] = tuple(detected_point.astype(int)) total_right_side_count += 1 all_available_points[26] = tuple(detected_point.astype(int)) # 26 point is now available, index is 26 else: result[27] = tuple(detected_point.astype(int)) total_right_side_count += 1 all_available_points[27] = tuple(detected_point.astype(int)) # 27 point is now available, index is 27 # Check if we can now form a vertical line with the newly calculated points for line_name, (indices, line_type) in line_groups.items(): if line_type == 'vertical': line_points = [(idx, all_available_points[idx]) for idx in indices if idx in all_available_points] if len(line_points) > max_vertical_points: max_vertical_points = len(line_points) best_vertical_line_name = line_name best_vertical_line_points = line_points # Calculate keypoint 22 if we have at least one line if best_vertical_line_name is not None and best_horizontal_line_name is not None: if kp_22 is None: print(f"Calculating keypoint 22 using both vertical and horizontal lines: {best_vertical_line_name} and {best_horizontal_line_name}") template_x_22 = 940 template_y_22 = 340 # Step 2: Project keypoint 22 onto vertical line (if available) template_vertical_start = template_coords[best_vertical_line_points[0][0]] template_vertical_end = template_coords[best_vertical_line_points[-1][0]] # Project at y=340 (same y as keypoint 22) template_y_vertical_start = template_vertical_start[1] template_y_vertical_end = template_vertical_end[1] if abs(template_y_vertical_end - template_y_vertical_start) > 1e-6: ratio_22_vertical = (template_y_22 - template_y_vertical_start) / (template_y_vertical_end - template_y_vertical_start) else: ratio_22_vertical = 0.5 frame_vertical_start = best_vertical_line_points[0][1] frame_vertical_end = best_vertical_line_points[-1][1] proj_22_on_vertical_x = frame_vertical_start[0] + (frame_vertical_end[0] - frame_vertical_start[0]) * ratio_22_vertical proj_22_on_vertical_y = frame_vertical_start[1] + (frame_vertical_end[1] - frame_vertical_start[1]) * ratio_22_vertical proj_22_on_vertical = (proj_22_on_vertical_x, proj_22_on_vertical_y) # Step 3: Project keypoint 22 onto horizontal line (if available) template_horizontal_start = template_coords[best_horizontal_line_points[0][0]] template_horizontal_end = template_coords[best_horizontal_line_points[-1][0]] # Project at x=940 (same x as keypoint 22) template_x_horizontal_start = template_horizontal_start[0] template_x_horizontal_end = template_horizontal_end[0] if abs(template_x_horizontal_end - template_x_horizontal_start) > 1e-6: ratio_22_horizontal = (template_x_22 - template_x_horizontal_start) / (template_x_horizontal_end - template_x_horizontal_start) else: ratio_22_horizontal = 0.5 frame_horizontal_start = best_horizontal_line_points[0][1] frame_horizontal_end = best_horizontal_line_points[-1][1] proj_22_on_horizontal_x = frame_horizontal_start[0] + (frame_horizontal_end[0] - frame_horizontal_start[0]) * ratio_22_horizontal proj_22_on_horizontal_y = frame_horizontal_start[1] + (frame_horizontal_end[1] - frame_horizontal_start[1]) * ratio_22_horizontal proj_22_on_horizontal = (proj_22_on_horizontal_x, proj_22_on_horizontal_y) # Step 4: Calculate keypoint 22 as intersection of two lines # Line 1: Passes through proj_22_on_vertical, parallel to best_horizontal_line # Line 2: Passes through proj_22_on_horizontal, parallel to best_vertical_line # Calculate direction vector of best_horizontal_line horizontal_dir_x = frame_horizontal_end[0] - frame_horizontal_start[0] horizontal_dir_y = frame_horizontal_end[1] - frame_horizontal_start[1] horizontal_dir_length = np.sqrt(horizontal_dir_x**2 + horizontal_dir_y**2) # Calculate direction vector of best_vertical_line vertical_dir_x = frame_vertical_end[0] - frame_vertical_start[0] vertical_dir_y = frame_vertical_end[1] - frame_vertical_start[1] vertical_dir_length = np.sqrt(vertical_dir_x**2 + vertical_dir_y**2) if horizontal_dir_length > 1e-6 and vertical_dir_length > 1e-6: # Normalize direction vectors horizontal_dir_x /= horizontal_dir_length horizontal_dir_y /= horizontal_dir_length vertical_dir_x /= vertical_dir_length vertical_dir_y /= vertical_dir_length # Line 1: passes through proj_22_on_vertical with direction of best_horizontal_line # Parametric: p1 = proj_22_on_vertical + t * horizontal_dir # Line 2: passes through proj_22_on_horizontal with direction of best_vertical_line # Parametric: p2 = proj_22_on_horizontal + s * vertical_dir # Find intersection: proj_22_on_vertical + t * horizontal_dir = proj_22_on_horizontal + s * vertical_dir # This gives us: # proj_22_on_vertical[0] + t * horizontal_dir_x = proj_22_on_horizontal[0] + s * vertical_dir_x # proj_22_on_vertical[1] + t * horizontal_dir_y = proj_22_on_horizontal[1] + s * vertical_dir_y # Rearranging: # t * horizontal_dir_x - s * vertical_dir_x = proj_22_on_horizontal[0] - proj_22_on_vertical[0] # t * horizontal_dir_y - s * vertical_dir_y = proj_22_on_horizontal[1] - proj_22_on_vertical[1] # Solve for t and s using linear algebra A = np.array([ [horizontal_dir_x, -vertical_dir_x], [horizontal_dir_y, -vertical_dir_y] ]) b = np.array([ proj_22_on_horizontal[0] - proj_22_on_vertical[0], proj_22_on_horizontal[1] - proj_22_on_vertical[1] ]) try: t, s = np.linalg.solve(A, b) # Calculate intersection point using line 1 x_22 = proj_22_on_vertical[0] + t * horizontal_dir_x y_22 = proj_22_on_vertical[1] + t * horizontal_dir_y result[21] = (int(round(x_22)), int(round(y_22))) total_right_side_count += 1 except np.linalg.LinAlgError: # Lines are parallel or nearly parallel, use simple intersection # If lines are parallel, use the projection points directly x_22 = proj_22_on_vertical[0] y_22 = proj_22_on_horizontal[1] result[21] = (int(round(x_22)), int(round(y_22))) total_right_side_count += 1 else: # Fallback: use simple intersection x_22 = proj_22_on_vertical[0] y_22 = proj_22_on_horizontal[1] result[21] = (int(round(x_22)), int(round(y_22))) total_right_side_count += 1 print(f"total_right_side_count: {total_right_side_count}, result: {result}") if total_right_side_count > 5: return result # Calculate m_line and b_line from best vertical or horizontal line for use in calculating other points m_line = None b_line = None best_line_for_calc = None best_line_type_for_calc = None if best_vertical_line_name is not None and len(best_vertical_line_points) >= 2: best_line_for_calc = best_vertical_line_points best_line_type_for_calc = 'vertical' points_array = np.array([[kp[0], kp[1]] for _, kp in best_vertical_line_points]) x_coords = points_array[:, 0] y_coords = points_array[:, 1] A = np.vstack([x_coords, np.ones(len(x_coords))]).T m_line, b_line = np.linalg.lstsq(A, y_coords, rcond=None)[0] elif best_horizontal_line_name is not None and len(best_horizontal_line_points) >= 2: best_line_for_calc = best_horizontal_line_points best_line_type_for_calc = 'horizontal' points_array = np.array([[kp[0], kp[1]] for _, kp in best_horizontal_line_points]) x_coords = points_array[:, 0] y_coords = points_array[:, 1] A = np.vstack([x_coords, np.ones(len(x_coords))]).T m_line, b_line = np.linalg.lstsq(A, y_coords, rcond=None)[0] # Calculate missing points to reach exactly 5 points # Ensure 4 points aren't all on one line if total_right_side_count < 5 and (m_line is not None or (best_line_for_calc is not None and best_line_type_for_calc == 'vertical')): # Check current distribution counts_per_line = [ len(line_18_21_points), len(line_23_24_points), len(line_25_30_points) ] # Calculate points on line 18-21 if needed template_ys_18_21 = [140, 270, 410, 540] template_indices_18_21 = [17, 18, 19, 20] if best_vertical_line_name == '25-30': # Construct parallel line 18-21 from line 25-30 for template_y, idx in zip(template_ys_18_21, template_indices_18_21): if result[idx] is None and total_right_side_count < 5: # Check if adding this point would put 4 on one line new_counts = counts_per_line.copy() new_counts[0] += 1 # Adding to line 18-21 if max(new_counts) >= 4 and total_right_side_count == 4: # Would have 4 on one line, skip continue # Calculate y using scale from template ref_ys = [kp[1] for _, kp in line_25_30_points] ref_template_ys = [5, 140, 250, 430, 540, 675] ref_indices = [24, 25, 26, 27, 28, 29] matched_template_ys = [] for ref_idx, ref_kp in line_25_30_points: if ref_idx in ref_indices: template_idx = ref_indices.index(ref_idx) matched_template_ys.append((ref_template_ys[template_idx], ref_kp[1])) if len(matched_template_ys) >= 1: ref_template_y, ref_frame_y = matched_template_ys[0] if ref_template_y > 0: scale = ref_frame_y / ref_template_y y_new = int(round(template_y * scale)) else: y_new = ref_frame_y else: y_new = int(round(np.median(ref_ys))) if ref_ys else template_y # Calculate x using parallel line geometry if abs(m_line) > 1e-6: x_on_line_25_30 = (y_new - b_line) / m_line x_new = int(round(x_on_line_25_30 * 0.850)) else: x_new = int(round(np.median([kp[0] for _, kp in line_25_30_points]) * 0.850)) result[idx] = (x_new, y_new) total_right_side_count += 1 if total_right_side_count >= 5: break elif best_vertical_line_name == '18-21': # Calculate missing points on line 18-21 for template_y, idx in zip(template_ys_18_21, template_indices_18_21): if result[idx] is None and total_right_side_count < 5: # Check if adding this point would put 4 on one line new_counts = counts_per_line.copy() new_counts[0] += 1 # Adding to line 18-21 if max(new_counts) >= 4 and total_right_side_count == 4: # Would have 4 on one line, skip continue # Calculate x on the line if abs(m_line) > 1e-6: x_new = (template_y - b_line) / m_line else: x_new = np.median([kp[0] for _, kp in line_18_21_points]) # Scale y based on available points ref_ys = [kp[1] for _, kp in line_18_21_points] ref_template_ys = [] for ref_idx, _ in line_18_21_points: if ref_idx in template_indices_18_21: template_idx = template_indices_18_21.index(ref_idx) ref_template_ys.append(template_ys_18_21[template_idx]) if len(ref_ys) >= 1 and len(ref_template_ys) >= 1: ref_template_y = ref_template_ys[0] ref_frame_y = ref_ys[0] if ref_template_y > 0: scale = ref_frame_y / ref_template_y y_new = int(round(template_y * scale)) else: y_new = ref_frame_y else: y_new = int(round(np.median(ref_ys))) if ref_ys else template_y result[idx] = (int(round(x_new)), y_new) total_right_side_count += 1 if total_right_side_count >= 5: break # Note: The unified approach above handles all cases (2a and 2b combined) # Legacy code removed - all logic is now in the unified case 2 above return result def check_keypoints_would_cause_invalid_mask( frame_keypoints: list[tuple[int, int]], template_keypoints: list[tuple[int, int]] = None, frame: np.ndarray = None, floor_markings_template: np.ndarray = None, return_warped_data: bool = False, ) -> tuple[bool, str] | tuple[bool, str, tuple]: """ Check if keypoints would cause InvalidMask errors during evaluation. Args: frame_keypoints: Frame keypoints to check template_keypoints: Template keypoints (defaults to TEMPLATE_KEYPOINTS) frame: Optional frame image for full validation floor_markings_template: Optional template image for full validation Returns: Tuple of (would_cause_error, error_message) """ try: from keypoint_evaluation import ( validate_projected_corners, TEMPLATE_KEYPOINTS, INDEX_KEYPOINT_CORNER_BOTTOM_LEFT, INDEX_KEYPOINT_CORNER_BOTTOM_RIGHT, INDEX_KEYPOINT_CORNER_TOP_LEFT, INDEX_KEYPOINT_CORNER_TOP_RIGHT, findHomography, InvalidMask, ) if template_keypoints is None: template_keypoints = TEMPLATE_KEYPOINTS # Filter valid keypoints filtered_template = [] filtered_frame = [] for i, (t_kp, f_kp) in enumerate(zip(template_keypoints, frame_keypoints)): if f_kp[0] > 0 and f_kp[1] > 0: filtered_template.append(t_kp) filtered_frame.append(f_kp) if len(filtered_template) < 4: if return_warped_data: return (True, "Not enough keypoints for homography", None) return (True, "Not enough keypoints for homography") # Compute homography src_pts = np.array(filtered_template, dtype=np.float32) dst_pts = np.array(filtered_frame, dtype=np.float32) result = findHomography(src_pts, dst_pts) if result is None: if return_warped_data: return (True, "Failed to compute homography", None) return (True, "Failed to compute homography") H, _ = result # Check for twisted projection (bowtie) try: validate_projected_corners( source_keypoints=template_keypoints, homography_matrix=H ) except Exception as e: error_msg = "Projection twisted (bowtie)" if "twisted" in str(e).lower() or "Projection twisted" in str(e).lower() else str(e) if return_warped_data: return (True, error_msg, None) return (True, error_msg) # If frame and template are provided, check mask validation if frame is not None and floor_markings_template is not None: try: from keypoint_evaluation import ( project_image_using_keypoints, extract_masks_for_ground_and_lines, InvalidMask, ) # project_image_using_keypoints can raise InvalidMask from validate_projected_corners try: # start_time = time.time() warped_template = project_image_using_keypoints( image=floor_markings_template, source_keypoints=template_keypoints, destination_keypoints=frame_keypoints, destination_width=frame.shape[1], destination_height=frame.shape[0], ) # end_time = time.time() # print(f"project_image_using_keypoints time: {end_time - start_time} seconds") except InvalidMask as e: if return_warped_data: return (True, f"Projection validation failed: {e}", None) return (True, f"Projection validation failed: {e}") except Exception as e: # Other errors (e.g., ValueError from homography failure) if return_warped_data: return (True, f"Projection failed: {e}", None) return (True, f"Projection failed: {e}") # extract_masks_for_ground_and_lines can raise InvalidMask from validation try: mask_ground, mask_lines_expected = extract_masks_for_ground_and_lines( image=warped_template ) except InvalidMask as e: if return_warped_data: return (True, f"Mask extraction validation failed: {e}", None) return (True, f"Mask extraction validation failed: {e}") except Exception as e: if return_warped_data: return (True, f"Mask extraction failed: {e}", None) return (True, f"Mask extraction failed: {e}") # Additional explicit validation (though extract_masks_for_ground_and_lines already validates) from keypoint_evaluation import validate_mask_lines, validate_mask_ground try: validate_mask_lines(mask_lines_expected) except InvalidMask as e: if return_warped_data: return (True, f"Mask lines validation failed: {e}", None) return (True, f"Mask lines validation failed: {e}") except Exception as e: if return_warped_data: return (True, f"Mask lines validation error: {e}", None) return (True, f"Mask lines validation error: {e}") try: validate_mask_ground(mask_ground) except InvalidMask as e: if return_warped_data: return (True, f"Mask ground validation failed: {e}", None) return (True, f"Mask ground validation failed: {e}") except Exception as e: if return_warped_data: return (True, f"Mask ground validation error: {e}", None) return (True, f"Mask ground validation error: {e}") # If return_warped_data is True and validation passed, return the computed data if return_warped_data: return (False, "", (warped_template, mask_ground, mask_lines_expected)) except ImportError: # If keypoint_evaluation is not available, skip validation pass except InvalidMask as e: # Catch any InvalidMask that wasn't caught above if return_warped_data: return (True, f"InvalidMask error: {e}", None) return (True, f"InvalidMask error: {e}") except Exception as e: # If we can't check masks for other reasons, assume it's okay # Don't let exceptions propagate pass # If we get here, keypoints should be valid if return_warped_data: return (False, "", None) # No warped data if frame/template not provided return (False, "") except ImportError: # If keypoint_evaluation is not available, skip validation if return_warped_data: return (False, "", None) return (False, "") except Exception as e: # Any other error - assume it would cause problems if return_warped_data: return (True, f"Validation error: {e}", None) return (True, f"Validation error: {e}") def evaluate_keypoints_with_cached_data( frame: np.ndarray, mask_ground: np.ndarray, mask_lines_expected: np.ndarray, ) -> float: """ Evaluate keypoints using pre-computed warped template and masks. This avoids redundant computation when we already have the warped data from validation. Args: frame: Frame image mask_ground: Pre-computed ground mask from warped template mask_lines_expected: Pre-computed expected lines mask from warped template Returns: Score between 0.0 and 1.0 """ try: from keypoint_evaluation import ( extract_mask_of_ground_lines_in_image, bitwise_and, ) # Only need to extract predicted lines from frame (uses cached mask_ground) mask_lines_predicted = extract_mask_of_ground_lines_in_image( image=frame, ground_mask=mask_ground ) pixels_overlapping = bitwise_and( mask_lines_expected, mask_lines_predicted ).sum() pixels_on_lines = mask_lines_expected.sum() score = pixels_overlapping / (pixels_on_lines + 1e-8) return min(1.0, max(0.0, score)) # Clamp to [0, 1] except Exception as e: print(f'Error in cached keypoint evaluation: {e}') return 0.0 def check_and_evaluate_keypoints( frame_keypoints: list[tuple[int, int]], template_keypoints: list[tuple[int, int]], frame: np.ndarray, floor_markings_template: np.ndarray, ) -> tuple[bool, float]: """ Check if keypoints would cause InvalidMask errors and evaluate them in one call. This reuses the warped template and masks computed during validation for evaluation. Args: frame_keypoints: Frame keypoints to check and evaluate template_keypoints: Template keypoints frame: Frame image floor_markings_template: Template image Returns: Tuple of (is_valid, score). If is_valid is False, score is 0.0. """ # Check with return_warped_data=True to get cached data # start_time = time.time() check_result = check_keypoints_would_cause_invalid_mask( frame_keypoints, template_keypoints, frame, floor_markings_template, return_warped_data=True ) # end_time = time.time() # print(f"check_keypoints_would_cause_invalid_mask time: {end_time - start_time} seconds") if len(check_result) == 3: would_cause_error, error_msg, warped_data = check_result else: would_cause_error, error_msg = check_result warped_data = None if would_cause_error: return (False, 0.0) # If we have cached warped data, use it for fast evaluation if warped_data is not None: _, mask_ground, mask_lines_expected = warped_data try: score = evaluate_keypoints_with_cached_data( frame, mask_ground, mask_lines_expected ) return (True, score) except Exception as e: print(f'Error evaluating with cached data: {e}') return (True, 0.0) # Fallback to regular evaluation if no cached data try: from keypoint_evaluation import evaluate_keypoints_for_frame score = evaluate_keypoints_for_frame( template_keypoints, frame_keypoints, frame, floor_markings_template ) return (True, score) except Exception as e: print(f'Error in regular evaluation: {e}') return (True, 0.0) def adjust_keypoints_to_avoid_invalid_mask( frame_keypoints: list[tuple[int, int]], template_keypoints: list[tuple[int, int]] = None, frame: np.ndarray = None, floor_markings_template: np.ndarray = None, max_iterations: int = 5, ) -> list[tuple[int, int]]: """ Adjust keypoints to avoid InvalidMask errors. This function tries to fix common issues: 1. Twisted projection (bowtie) - adjusts corner keypoints 2. Ground covers too much - shrinks projected area by moving corners inward 3. Other mask validation issues - adjusts keypoints to improve projection Args: frame_keypoints: Frame keypoints to adjust template_keypoints: Template keypoints frame: Optional frame image for validation floor_markings_template: Optional template image for validation max_iterations: Maximum number of adjustment iterations Returns: Adjusted keypoints that should avoid InvalidMask errors """ adjusted = list(frame_keypoints) # Check if adjustment is needed would_cause_error, error_msg = check_keypoints_would_cause_invalid_mask( adjusted, template_keypoints, frame, floor_markings_template ) print(f"Would cause error: {would_cause_error}, error_msg: {error_msg}") if not would_cause_error: return (True,adjusted) # Try to fix twisted projection (most common issue) if "twisted" in error_msg.lower() or "bowtie" in error_msg.lower() or "Projection twisted" in error_msg.lower(): # Use the existing _adjust_keypoints_to_pass_validation function adjusted = _adjust_keypoints_to_pass_validation( adjusted, template_keypoints, frame.shape[1] if frame is not None else None, frame.shape[0] if frame is not None else None ) # Check again after adjustment would_cause_error, error_msg = check_keypoints_would_cause_invalid_mask( adjusted, template_keypoints, frame, floor_markings_template ) if not would_cause_error: return (True,adjusted) start_time = time.time() # Handle "a projected line is too wide" error # This happens when projected lines are too thick/wide (aspect ratio too high) if "too wide" in error_msg.lower() or "wide line" in error_msg.lower(): print(f"Adjusting keypoints to fix 'a projected line is too wide' error") try: # This error usually means the projection is creating lines that are too thick # Strategy: Adjust keypoints to reduce projection distortion valid_keypoints = [] for idx in range(len(adjusted)): x, y = adjusted[idx] if x == 0 and y == 0: continue valid_keypoints.append((idx, x, y)) if len(valid_keypoints) >= 4: # Calculate center and spread of keypoints center_x = sum(x for _, x, y in valid_keypoints) / len(valid_keypoints) center_y = sum(y for _, x, y in valid_keypoints) / len(valid_keypoints) # Calculate distances from center distances = [] for idx, x, y in valid_keypoints: dist = np.sqrt((x - center_x)**2 + (y - center_y)**2) distances.append((idx, x, y, dist)) # Sort by distance distances.sort(key=lambda d: d[3], reverse=True) # Strategy 1: Try moving keypoints slightly outward to reduce compression # This can help if keypoints are too close together causing wide lines best_wide_kps = None best_wide_score = -1.0 # Try expanding keypoints slightly (opposite of shrinking) for expand_factor in [1.02, 1.05, 1.08, 1.10]: test_kps = list(adjusted) for idx, x, y, dist in distances: # Move keypoint slightly away from center new_x = int(round(center_x + (x - center_x) * expand_factor)) new_y = int(round(center_y + (y - center_y) * expand_factor)) test_kps[idx] = (new_x, new_y) # Check and get cached warped data if available check_result = check_keypoints_would_cause_invalid_mask( test_kps, template_keypoints, frame, floor_markings_template, return_warped_data=(frame is not None and floor_markings_template is not None) ) if len(check_result) == 3: would_cause_error, test_error_msg, warped_data = check_result else: would_cause_error, test_error_msg = check_result warped_data = None if not would_cause_error: # Evaluate score for this adjustment if frame is not None and floor_markings_template is not None: try: if warped_data is not None: # Use cached warped data for faster evaluation _, mask_ground, mask_lines_expected = warped_data score = evaluate_keypoints_with_cached_data( frame, mask_ground, mask_lines_expected ) else: # Fallback to regular evaluation from keypoint_evaluation import evaluate_keypoints_for_frame score = evaluate_keypoints_for_frame( template_keypoints, test_kps, frame, floor_markings_template ) if score > best_wide_score: best_wide_score = score best_wide_kps = test_kps print(f"Found valid wide-line adjustment (expand) with factor {expand_factor}, score: {score:.4f}") except Exception: # If score evaluation fails, use this adjustment anyway print(f"Successfully adjusted keypoints for wide line (expand) with factor {expand_factor}") return (True,test_kps) else: print(f"Successfully adjusted keypoints for wide line (expand) with factor {expand_factor}") return (True,test_kps) # Strategy 2: If expanding didn't work, try adjusting individual keypoints # Move keypoints that are too close together slightly apart for idx, x, y, dist in distances: # Try small adjustments to this keypoint for adjust_x in [-3, -2, -1, 1, 2, 3]: for adjust_y in [-3, -2, -1, 1, 2, 3]: test_kps = list(adjusted) test_kps[idx] = (x + adjust_x, y + adjust_y) # Use optimized check_and_evaluate to reuse warped data if frame is not None and floor_markings_template is not None: is_valid, score = check_and_evaluate_keypoints( test_kps, template_keypoints, frame, floor_markings_template ) if is_valid: if score > best_wide_score: best_wide_score = score best_wide_kps = test_kps print(f"Found valid wide-line adjustment (perturb) for keypoint {idx}, score: {score:.4f}") else: would_cause_error, _ = check_keypoints_would_cause_invalid_mask( test_kps, template_keypoints, frame, floor_markings_template ) if not would_cause_error: return (True, test_kps) # Return the best scoring adjustment if we found any if best_wide_kps is not None: end_time = time.time() print(f"Returning best scoring wide-line adjustment time: {end_time - start_time} seconds") print(f"Returning best scoring wide-line adjustment with score: {best_wide_score:.4f}") return (True,best_wide_kps) # Strategy 3: Try slight shrinking (opposite approach - reduce projection area) for shrink_factor in [0.98, 0.96, 0.94]: test_kps = list(adjusted) for idx, x, y, dist in distances: new_x = int(round(center_x + (x - center_x) * shrink_factor)) new_y = int(round(center_y + (y - center_y) * shrink_factor)) test_kps[idx] = (new_x, new_y) # Use optimized check_and_evaluate to reuse warped data if frame is not None and floor_markings_template is not None: try: is_valid, score = check_and_evaluate_keypoints( test_kps, template_keypoints, frame, floor_markings_template ) if is_valid: if score > best_wide_score: best_wide_score = score best_wide_kps = test_kps print(f"Found valid wide-line adjustment (shrink) with factor {shrink_factor}, score: {score:.4f}") except Exception: would_cause_error, _ = check_keypoints_would_cause_invalid_mask( test_kps, template_keypoints, frame, floor_markings_template ) if not would_cause_error: return (True, test_kps) else: would_cause_error, _ = check_keypoints_would_cause_invalid_mask( test_kps, template_keypoints, frame, floor_markings_template ) if not would_cause_error: return (True, test_kps) if best_wide_kps is not None: print(f"Returning best scoring wide-line adjustment with score: {best_wide_score:.4f}") return (True,best_wide_kps) except Exception as e: print(f"Error in wide line adjustment: {e}") pass # Handle "projected ground should be a single object" error # This happens when the ground mask has multiple disconnected regions if "should be a single" in error_msg.lower() or "single object" in error_msg.lower() or "distinct regions" in error_msg.lower(): print(f"Adjusting keypoints to fix 'projected ground should be a single object' error") try: # This error usually means the projection creates gaps/holes in the ground mask # We need to adjust keypoints to make the projection more continuous # Strategy 1: Try moving keypoints closer together to reduce gaps valid_keypoints = [] for idx in range(len(adjusted)): x, y = adjusted[idx] if x == 0 and y == 0: continue valid_keypoints.append((idx, x, y)) if len(valid_keypoints) >= 4: # Calculate center of all keypoints center_x = sum(x for _, x, y in valid_keypoints) / len(valid_keypoints) center_y = sum(y for _, x, y in valid_keypoints) / len(valid_keypoints) # Try moving keypoints slightly closer to center to reduce fragmentation # Use smaller adjustments to preserve geometry best_single_kps = None best_single_score = -1.0 for shrink_factor in [0.98, 0.96, 0.94, 0.92, 0.90]: test_kps = list(adjusted) for idx, x, y in valid_keypoints: # Move keypoint slightly toward center new_x = int(round(center_x + (x - center_x) * shrink_factor)) new_y = int(round(center_y + (y - center_y) * shrink_factor)) test_kps[idx] = (new_x, new_y) # Use optimized check_and_evaluate to reuse warped data if frame is not None and floor_markings_template is not None: try: is_valid, score = check_and_evaluate_keypoints( test_kps, template_keypoints, frame, floor_markings_template ) if is_valid: if score > best_single_score: best_single_score = score best_single_kps = test_kps print(f"Found valid single-object adjustment with shrink_factor {shrink_factor}, score: {score:.4f}") except Exception: # If score evaluation fails, use this adjustment anyway print(f"Successfully adjusted keypoints for single object with shrink_factor {shrink_factor}") return (True, test_kps) else: # No frame/template for score evaluation, use first valid adjustment would_cause_error, _ = check_keypoints_would_cause_invalid_mask( test_kps, template_keypoints, frame, floor_markings_template ) if not would_cause_error: print(f"Successfully adjusted keypoints for single object with shrink_factor {shrink_factor}") return (True, test_kps) # Return the best scoring adjustment if we found any if best_single_kps is not None: print(f"Returning best scoring single-object adjustment with score: {best_single_score:.4f}") return (True,best_single_kps) # Strategy 2: If moving toward center didn't work, try adjusting boundary keypoints # Calculate distances from center distances = [] for idx, x, y in valid_keypoints: dist = np.sqrt((x - center_x)**2 + (y - center_y)**2) distances.append((idx, x, y, dist)) # Sort by distance (farthest first) distances.sort(key=lambda d: d[3], reverse=True) # Try adjusting the farthest keypoints (which might be causing fragmentation) for shrink_factor in [0.95, 0.90, 0.85]: test_kps = list(adjusted) # Adjust top 25% of farthest keypoints boundary_count = max(1, len(distances) // 4) for idx, x, y, dist in distances[:boundary_count]: new_x = int(round(center_x + (x - center_x) * shrink_factor)) new_y = int(round(center_y + (y - center_y) * shrink_factor)) test_kps[idx] = (new_x, new_y) # Use optimized check_and_evaluate to reuse warped data if frame is not None and floor_markings_template is not None: try: is_valid, score = check_and_evaluate_keypoints( test_kps, template_keypoints, frame, floor_markings_template ) if is_valid: print(f"Found valid boundary adjustment for single object with shrink_factor {shrink_factor}, score: {score:.4f}") return (True, test_kps) except Exception: return (True, test_kps) else: would_cause_error, _ = check_keypoints_would_cause_invalid_mask( test_kps, template_keypoints, frame, floor_markings_template ) if not would_cause_error: return (True, test_kps) except Exception as e: print(f"Error in single object adjustment: {e}") pass # Handle "ground covers too much" error by shrinking the projected area if "ground covers" in error_msg.lower() or "covers more than" in error_msg.lower(): print(f"Adjusting keypoints to avoid 'ground covers too much' error") try: from keypoint_evaluation import ( INDEX_KEYPOINT_CORNER_BOTTOM_LEFT, INDEX_KEYPOINT_CORNER_BOTTOM_RIGHT, INDEX_KEYPOINT_CORNER_TOP_LEFT, INDEX_KEYPOINT_CORNER_TOP_RIGHT, ) # First, try adjusting corners if available corner_indices = [ INDEX_KEYPOINT_CORNER_TOP_LEFT, INDEX_KEYPOINT_CORNER_TOP_RIGHT, INDEX_KEYPOINT_CORNER_BOTTOM_RIGHT, INDEX_KEYPOINT_CORNER_BOTTOM_LEFT, ] # Get corner keypoints corners = [] center_x, center_y = 0, 0 valid_corners = 0 for corner_idx in corner_indices: if corner_idx < len(adjusted): x, y = adjusted[corner_idx] if x == 0 and y == 0: continue corners.append((corner_idx, x, y)) center_x += x center_y += y valid_corners += 1 if valid_corners >= 4: center_x /= valid_corners center_y /= valid_corners # Move corners inward toward center to reduce projected area # Try different shrink factors, starting with smaller adjustments to preserve score # Use score-aware adjustment: evaluate score for each candidate and pick the best best_corner_kps = None best_corner_score = -1.0 for shrink_factor in [0.95, 0.90, 0.85, 0.80, 0.75, 0.70, 0.65]: test_kps = list(adjusted) for corner_idx, x, y in corners: # Move corner toward center new_x = int(round(center_x + (x - center_x) * shrink_factor)) new_y = int(round(center_y + (y - center_y) * shrink_factor)) test_kps[corner_idx] = (new_x, new_y) # Use optimized check_and_evaluate to reuse warped data if frame is not None and floor_markings_template is not None: try: is_valid, score = check_and_evaluate_keypoints( test_kps, template_keypoints, frame, floor_markings_template ) if is_valid: if score > best_corner_score: best_corner_score = score best_corner_kps = test_kps print(f"Found valid corner adjustment with shrink_factor {shrink_factor}, score: {score:.4f}") except Exception: # If score evaluation fails, use this adjustment anyway return (True, test_kps) else: # No frame/template for score evaluation, use first valid adjustment would_cause_error, _ = check_keypoints_would_cause_invalid_mask( test_kps, template_keypoints, frame, floor_markings_template ) if not would_cause_error: return (True, test_kps) # Return the best scoring corner adjustment if we found any if best_corner_kps is not None: print(f"Returning best scoring corner adjustment with score: {best_corner_score:.4f}") return (True,best_corner_kps) # If corners adjustment didn't work or we don't have enough corners, # try adjusting individual keypoints one at a time # This handles cases where non-corner keypoints (like 15, 16, 17, 31, 32) are causing the issue valid_keypoints = [] all_center_x, all_center_y = 0, 0 valid_count = 0 for idx in range(len(adjusted)): x, y = adjusted[idx] if x == 0 and y == 0: continue valid_keypoints.append((idx, x, y)) all_center_x += x all_center_y += y valid_count += 1 if valid_count >= 4: all_center_x /= valid_count all_center_y /= valid_count # Calculate distances from center for each keypoint # Try adjusting keypoints farthest from center first (most likely to cause coverage issues) distances = [] for idx, x, y in valid_keypoints: dist = np.sqrt((x - all_center_x)**2 + (y - all_center_y)**2) distances.append((idx, x, y, dist)) # Sort by distance (farthest first) - these are most likely causing the coverage issue distances.sort(key=lambda d: d[3], reverse=True) # Try adjusting each keypoint individually, starting with farthest from center # Use score-aware adjustment: evaluate score for each candidate and pick the best best_kps = None best_score = -1.0 for idx, x, y, dist in distances: # Try different shrink factors for this single keypoint # Start with smaller adjustments to preserve score better for shrink_factor in [0.98, 0.95, 0.92, 0.90, 0.85, 0.80, 0.75, 0.70, 0.65]: test_kps = list(adjusted) # Start with original adjusted keypoints # Only adjust this one keypoint new_x = int(round(all_center_x + (x - all_center_x) * shrink_factor)) new_y = int(round(all_center_y + (y - all_center_y) * shrink_factor)) test_kps[idx] = (new_x, new_y) # Use optimized check_and_evaluate to reuse warped data if frame is not None and floor_markings_template is not None: try: is_valid, score = check_and_evaluate_keypoints( test_kps, template_keypoints, frame, floor_markings_template ) if is_valid: if score > best_score: best_score = score best_kps = test_kps print(f"Found valid adjustment for keypoint {idx} with shrink_factor {shrink_factor}, score: {score:.4f}") except Exception: # If score evaluation fails, use this adjustment anyway print(f"Successfully adjusted keypoint {idx} (distance {dist:.1f} from center) with shrink_factor {shrink_factor}") return (True, test_kps) else: # No frame/template for score evaluation, use first valid adjustment would_cause_error, _ = check_keypoints_would_cause_invalid_mask( test_kps, template_keypoints, frame, floor_markings_template ) if not would_cause_error: print(f"Successfully adjusted keypoint {idx} (distance {dist:.1f} from center) with shrink_factor {shrink_factor}") return (True, test_kps) # Return the best scoring adjustment if we found any if best_kps is not None: print(f"Returning best scoring adjustment with score: {best_score:.4f}") return (True,best_kps) # If adjusting individual keypoints didn't work, try adjusting pairs of keypoints # (but only if we have enough keypoints) if valid_count >= 6: # Try adjusting the two farthest keypoints together # Use score-aware adjustment here too best_pair_kps = None best_pair_score = -1.0 for shrink_factor in [0.95, 0.90, 0.85, 0.80, 0.75, 0.70]: test_kps = list(adjusted) # Adjust top 2 farthest keypoints for idx, x, y, dist in distances[:2]: new_x = int(round(all_center_x + (x - all_center_x) * shrink_factor)) new_y = int(round(all_center_y + (y - all_center_y) * shrink_factor)) test_kps[idx] = (new_x, new_y) # Use optimized check_and_evaluate to reuse warped data if frame is not None and floor_markings_template is not None: try: is_valid, score = check_and_evaluate_keypoints( test_kps, template_keypoints, frame, floor_markings_template ) if is_valid: if score > best_pair_score: best_pair_score = score best_pair_kps = test_kps print(f"Found valid pair adjustment with shrink_factor {shrink_factor}, score: {score:.4f}") except Exception: # If score evaluation fails, use this adjustment anyway print(f"Successfully adjusted 2 farthest keypoints with shrink_factor {shrink_factor}") return (True, test_kps) else: # No frame/template for score evaluation, use first valid adjustment would_cause_error, _ = check_keypoints_would_cause_invalid_mask( test_kps, template_keypoints, frame, floor_markings_template ) if not would_cause_error: print(f"Successfully adjusted 2 farthest keypoints with shrink_factor {shrink_factor}") return (True, test_kps) # Return the best scoring pair adjustment if we found any if best_pair_kps is not None: print(f"Returning best scoring pair adjustment with score: {best_pair_score:.4f}") return (True,best_pair_kps) except Exception as e: print(f"Error in ground coverage adjustment: {e}") pass # If still causing errors, try small perturbations to corner keypoints # This helps with mask validation issues if would_cause_error and max_iterations > 0: try: from keypoint_evaluation import ( INDEX_KEYPOINT_CORNER_BOTTOM_LEFT, INDEX_KEYPOINT_CORNER_BOTTOM_RIGHT, INDEX_KEYPOINT_CORNER_TOP_LEFT, INDEX_KEYPOINT_CORNER_TOP_RIGHT, ) corner_indices = [ INDEX_KEYPOINT_CORNER_TOP_LEFT, INDEX_KEYPOINT_CORNER_TOP_RIGHT, INDEX_KEYPOINT_CORNER_BOTTOM_RIGHT, INDEX_KEYPOINT_CORNER_BOTTOM_LEFT, ] # Try small adjustments to corners and evaluate to find the best one best_corner_kps = None best_corner_score = -1.0 for corner_idx in corner_indices: if corner_idx < len(adjusted): x, y = adjusted[corner_idx] if x == 0 and y == 0: continue # Try small perturbations for dx in [-5, -3, -1, 1, 3, 5]: for dy in [-5, -3, -1, 1, 3, 5]: test_kps = list(adjusted) test_kps[corner_idx] = (x + dx, y + dy) # Use optimized check_and_evaluate to reuse warped data if frame is not None and floor_markings_template is not None: try: is_valid, score = check_and_evaluate_keypoints( test_kps, template_keypoints, frame, floor_markings_template ) if is_valid: if score > best_corner_score: best_corner_score = score best_corner_kps = test_kps print(f"Found valid corner perturbation: corner {corner_idx} with adjust ({dx}, {dy}), score: {score:.4f}") except Exception: pass else: # No frame/template, just check validation would_cause_error, _ = check_keypoints_would_cause_invalid_mask( test_kps, template_keypoints, frame, floor_markings_template ) if not would_cause_error: return (True, test_kps) # Return the best scoring corner adjustment if we found any if best_corner_kps is not None: print(f"Returning best scoring corner perturbation with score: {best_corner_score:.4f}") return (True, best_corner_kps) except Exception: pass # If we can't fix it, return adjusted (best effort) return (False,adjusted) def _validate_keypoints_corners( frame_keypoints: list[tuple[int, int]], template_keypoints: list[tuple[int, int]] = None, ) -> bool: """ Validate that frame keypoints can form a valid homography with template keypoints (corners don't create twisted projection). Returns True if validation passes, False otherwise. """ try: from keypoint_evaluation import ( validate_projected_corners, TEMPLATE_KEYPOINTS, INDEX_KEYPOINT_CORNER_BOTTOM_LEFT, INDEX_KEYPOINT_CORNER_BOTTOM_RIGHT, INDEX_KEYPOINT_CORNER_TOP_LEFT, INDEX_KEYPOINT_CORNER_TOP_RIGHT, ) # Use provided template_keypoints or default TEMPLATE_KEYPOINTS if template_keypoints is None: template_keypoints = TEMPLATE_KEYPOINTS # Filter valid keypoints (non-zero) filtered_template = [] filtered_frame = [] for i, (t_kp, f_kp) in enumerate(zip(template_keypoints, frame_keypoints)): if f_kp[0] > 0 and f_kp[1] > 0: # Frame keypoint is valid filtered_template.append(t_kp) filtered_frame.append(f_kp) if len(filtered_template) < 4: return False # Not enough keypoints for homography # Compute homography from template to frame src_pts = np.array(filtered_template, dtype=np.float32) dst_pts = np.array(filtered_frame, dtype=np.float32) H, mask = cv2.findHomography(src_pts, dst_pts) if H is None: return False # Homography computation failed # Validate corners using the homography try: validate_projected_corners( source_keypoints=template_keypoints, homography_matrix=H ) return True # Validation passed except Exception: return False # Validation failed (twisted projection) except ImportError: # If keypoint_evaluation is not available, skip validation return True except Exception: # Any other error - assume invalid return False def predict_failed_indices( results_frames: Sequence[Any], template_keypoints: list[tuple[int, int]] = None, frame_width: int = None, frame_height: int = None, frames: List[np.ndarray] = None, floor_markings_template: np.ndarray = None, offset: int = 0, ) -> list[int]: """ Predict failed frame indices based on: 1. Having <= 4 valid keypoints (after calculating missing ones) 2. Failing validate_projected_corners validation (twisted projection) For each frame, tries to calculate missing keypoints first. If after calculation we have more than 5 keypoints, the frame is not marked as failed. """ max_frames = len(results_frames) if max_frames == 0: return [] failed_indices: list[int] = [] for frame_index, frame_result in enumerate(results_frames): frame_keypoints = getattr(frame_result, "keypoints", []) or [] original_count = sum(1 for (x, y) in frame_keypoints if int(x) != 0 and int(y) != 0) # Try to calculate missing keypoints # First, remove duplicate/conflicting detections (e.g., same point detected as both 13 and 21) cleaned_keypoints = remove_duplicate_detections( frame_keypoints, frame_width, frame_height ) valid_keypoint_indices = [idx for idx, kp in enumerate(cleaned_keypoints) if kp[0] != 0 and kp[1] != 0] if len(valid_keypoint_indices) > 5: calculated_keypoints = cleaned_keypoints else: left_side_indices_range = range(0, 13) right_side_indices_range = range(17, 30) side_check_set = set() if len(valid_keypoint_indices) >= 4: for idx in valid_keypoint_indices: if idx in left_side_indices_range: side_check_set.add("left") elif idx in right_side_indices_range: side_check_set.add("right") else: side_check_set.add("center") if len(side_check_set) > 1: calculated_keypoints = cleaned_keypoints else: # Then calculate missing keypoints calculated_keypoints = calculate_missing_keypoints( cleaned_keypoints, frame_width, frame_height ) # Get frame image if available frame_image = None if frames is not None and frame_index < len(frames): frame_image = frames[frame_index] original_frame_number = offset + frame_index print(f"Frame {original_frame_number} (index {frame_index}): original_count: {original_count}, cleaned_keypoints: {len([kp for kp in cleaned_keypoints if kp[0] != 0 and kp[1] != 0])}, calculated_keypoints: {len([kp for kp in calculated_keypoints if kp[0] != 0 and kp[1] != 0])}") start_time = time.time() adjusted_success, calculated_keypoints = adjust_keypoints_to_avoid_invalid_mask( calculated_keypoints, template_keypoints, frame_image, floor_markings_template ) end_time = time.time() print(f"adjust_keypoints_to_avoid_invalid_mask time: {end_time - start_time} seconds") if not adjusted_success: failed_indices.append(frame_index) continue print(f"after adjustment, calculated_keypoints: {calculated_keypoints}") # Update the frame result with calculated keypoints setattr(frame_result, "keypoints", list(calculated_keypoints)) return failed_indices def _generate_sparse_template_keypoints( frame_width: int, frame_height: int, frame_image: np.ndarray = None, template_image: np.ndarray = None, template_keypoints: list[tuple[int, int]] = None ) -> list[tuple[int, int]]: # Calculate template dimensions from template_keypoints if available, otherwise use default if template_keypoints is not None and len(template_keypoints) > 0: valid_template_points = [(x, y) for x, y in template_keypoints if x > 0 and y > 0] if len(valid_template_points) > 0: template_max_x = max(x for x, y in valid_template_points) template_max_y = max(y for x, y in valid_template_points) else: template_max_x, template_max_y = (1045, 675) # Default fallback else: template_max_x, template_max_y = (1045, 675) # Default fallback # Calculate scaling factors for both dimensions sx = float(frame_width) / float(template_max_x if template_max_x != 0 else 1) sy = float(frame_height) / float(template_max_y if template_max_y != 0 else 1) # Always use uniform scaling to preserve pitch geometry and aspect ratio # This prevents distortion that creates square contours (like 3x3, 4x4) which fail the wide line check # Uniform scaling ensures the pitch maintains its shape and avoids twisted projections uniform_scale = min(sx, sy) # Scale down significantly to create a much smaller pitch in the warped template # Use a small fraction of the uniform scale to make the pitch as small as possible # This creates a small pitch centered in the frame, avoiding edge artifacts scale_factor = 0.25 # Use 25% of the frame-filling scale to make pitch much smaller uniform_scale = uniform_scale * scale_factor # Ensure minimum scale to avoid keypoints being too close together # Very small scales cause warping artifacts that create square contours (1x1, 2x2 pixels) # These single-pixel artifacts trigger the "too wide" error # Use a fixed minimum scale based on template dimensions to ensure keypoints are spaced properly # This prevents warping artifacts regardless of frame size # Template is 1045x675, need sufficient scale to avoid 1x1 pixel artifacts from warping # Higher minimum scale ensures warped template doesn't create tiny square artifacts min_scale_absolute = 0.5 # Fixed minimum 50% of template size to avoid 1x1 pixel warping artifacts # Higher scale is necessary to prevent warping interpolation from creating single-pixel squares uniform_scale = max(uniform_scale, min_scale_absolute) # Analyze line distribution to determine which keypoints to use and where to place them # Default: use center line keypoints (15, 16, 31, 32) - indices 14, 15, 30, 31 selected_keypoint_indices = set([14, 15, 30, 31]) # Default: 15, 16, 31, 32 line_distribution = None # Will store: (top_count, bottom_count, left_count, right_count, total_count) # If we have line distribution analysis, select appropriate keypoints if frame_image is not None and template_image is not None and template_keypoints is not None: try: from keypoint_evaluation import ( project_image_using_keypoints, extract_masks_for_ground_and_lines_no_validation, extract_mask_of_ground_lines_in_image ) # Generate initial keypoints for analysis using EXACT FITTING (full frame coverage) # This ensures we get correct line distribution analysis # Use non-uniform scaling to fit exactly to frame dimensions initial_sx = float(frame_width) / float(template_max_x if template_max_x != 0 else 1) initial_sy = float(frame_height) / float(template_max_y if template_max_y != 0 else 1) initial_scaled = [] num_template_kps = len(template_keypoints) if template_keypoints is not None else 32 for i in range(max(32, num_template_kps)): # Ensure we have at least 32 keypoints if i < num_template_kps: tx, ty = template_keypoints[i] if tx > 0 and ty > 0: # Only scale non-zero keypoints # Use non-uniform scaling for exact fit x_scaled = int(round(tx * initial_sx)) y_scaled = int(round(ty * initial_sy)) # Clamp to frame bounds x_scaled = max(0, min(x_scaled, frame_width - 1)) y_scaled = max(0, min(y_scaled, frame_height - 1)) initial_scaled.append((x_scaled, y_scaled)) else: initial_scaled.append((0, 0)) else: initial_scaled.append((0, 0)) # Pad to 32 if template_keypoints has fewer # With exact fitting, keypoints already fill the frame, no centering needed initial_centered = initial_scaled if len(initial_scaled) > 0: try: warped_template = project_image_using_keypoints( image=template_image, source_keypoints=template_keypoints, destination_keypoints=initial_centered, destination_width=frame_width, destination_height=frame_height, ) # Use non-validating version for line distribution analysis # Exact fitting might create invalid masks, but we still want to analyze line distribution mask_ground, mask_lines = extract_masks_for_ground_and_lines_no_validation(image=warped_template) mask_lines_predicted = extract_mask_of_ground_lines_in_image( image=frame_image, ground_mask=mask_ground ) h, w = mask_lines_predicted.shape top_half = mask_lines_predicted[:h//2, :] bottom_half = mask_lines_predicted[h//2:, :] left_half = mask_lines_predicted[:, :w//2] right_half = mask_lines_predicted[:, w//2:] top_line_count = np.sum(top_half > 0) bottom_line_count = np.sum(bottom_half > 0) left_line_count = np.sum(left_half > 0) right_line_count = np.sum(right_half > 0) total_line_count = top_line_count + bottom_line_count line_distribution = (top_line_count, bottom_line_count, left_line_count, right_line_count, total_line_count) # print(f"top_line_count: {top_line_count}, bottom_line_count: {bottom_line_count}, left_line_count: {left_line_count}, right_line_count: {right_line_count}, total_line_count: {total_line_count}") if total_line_count > 100: # Only use analysis if enough lines detected # Select keypoints based on where lines are detected # If lines at top -> use top part of pitch (so top part aligns with top where lines are) # If lines at bottom -> use bottom part of pitch (so bottom part aligns with bottom where lines are) # If lines at left -> use left part of pitch (so left part aligns with left where lines are) # If lines at right -> use right part of pitch (so right part aligns with right where lines are) # Define keypoint sets for different regions top_part = set([0, 1, 2, 9, 13, 14, 24, 25, 26]) # Top part of pitch bottom_part = set([3, 4, 5, 12, 15, 16, 27, 28, 29]) # Bottom part of pitch left_part = set(list(range(0, 13)) + [6, 7, 8]) # Left part of pitch right_part = set(list(range(17, 30)) + [21, 22, 23]) # Right part of pitch center_reference = set([13, 14, 15, 16, 30, 31]) # Center line and circle # Select vertical region - match where lines are detected vertical_selection = set() if top_line_count > bottom_line_count: # Lines at top -> use top part of pitch vertical_selection = top_part else: # Lines at bottom -> use bottom part of pitch vertical_selection = bottom_part # Select horizontal region - match where lines are detected horizontal_selection = set() if left_line_count > right_line_count: # Lines at left -> use left part of pitch horizontal_selection = left_part else: # Lines at right -> use right part of pitch horizontal_selection = right_part # Use intersection when both conditions are met, otherwise use the stronger signal # This ensures we select a coherent region (e.g., bottom-right corner) rather than union vertical_diff = abs(top_line_count - bottom_line_count) horizontal_diff = abs(left_line_count - right_line_count) if vertical_diff > horizontal_diff * 1.5: # Vertical signal is much stronger - use only vertical selection selected_keypoint_indices = vertical_selection elif horizontal_diff > vertical_diff * 1.5: # Horizontal signal is much stronger - use only horizontal selection selected_keypoint_indices = horizontal_selection else: # Both signals are similar - use intersection to get corner region selected_keypoint_indices = vertical_selection & horizontal_selection # If intersection is too small, fall back to union if len(selected_keypoint_indices) < 4: selected_keypoint_indices = vertical_selection | horizontal_selection # Always include center line and center circle for reference selected_keypoint_indices.update(center_reference) # Ensure we have at least 4 keypoints if len(selected_keypoint_indices) < 4: selected_keypoint_indices = set([14, 15, 30, 31]) # Fallback to default except Exception: pass # Use default keypoints if analysis fails except Exception: pass # Use default keypoints if analysis fails # Generate scaled keypoints only for selected indices # Use template_keypoints if available, otherwise fall back to FOOTBALL_KEYPOINTS source_keypoints = template_keypoints if template_keypoints is not None else FOOTBALL_KEYPOINTS num_keypoints = len(source_keypoints) if source_keypoints is not None else 32 scaled: list[tuple[int, int]] = [] for i in range(num_keypoints): if i in selected_keypoint_indices and i < len(source_keypoints): tx, ty = source_keypoints[i] if tx > 0 and ty > 0: # Only scale non-zero keypoints x_scaled = int(round(tx * uniform_scale)) y_scaled = int(round(ty * uniform_scale)) scaled.append((x_scaled, y_scaled)) else: scaled.append((0, 0)) else: scaled.append((0, 0)) # Set unselected keypoints to (0, 0) # Ensure minimum spacing between keypoints to avoid warping artifacts # Very close keypoints can create single-pixel square contours during warping # Check if any keypoints are too close and adjust scale if needed min_spacing = 5 # Minimum 5 pixels between keypoints to avoid 1x1 artifacts needs_adjustment = False for i in range(len(scaled)): if scaled[i][0] == 0 and scaled[i][1] == 0: continue x1, y1 = scaled[i] for j in range(i + 1, len(scaled)): if scaled[j][0] == 0 and scaled[j][1] == 0: continue x2, y2 = scaled[j] dist = np.sqrt((x2 - x1)**2 + (y2 - y1)**2) if dist > 0 and dist < min_spacing: needs_adjustment = True break if needs_adjustment: break # If keypoints are too close, slightly increase scale to maintain minimum spacing if needs_adjustment and uniform_scale < 0.25: uniform_scale = uniform_scale * 1.2 # Increase by 20% to ensure spacing uniform_scale = min(uniform_scale, 0.25) # Cap at 25% to keep it small # Recalculate selected keypoints with adjusted scale scaled = [] for k in range(num_keypoints): if k in selected_keypoint_indices and k < len(source_keypoints): tx, ty = source_keypoints[k] if tx > 0 and ty > 0: # Only scale non-zero keypoints x_scaled = int(round(tx * uniform_scale)) y_scaled = int(round(ty * uniform_scale)) scaled.append((x_scaled, y_scaled)) else: scaled.append((0, 0)) else: scaled.append((0, 0)) # Use line distribution analysis (already computed above) to determine optimal pitch placement offset_x = 0 offset_y = 0 if line_distribution is not None: top_line_count, bottom_line_count, left_line_count, right_line_count, total_line_count = line_distribution # Adjust keypoint placement based on line distribution valid_points = [(x, y) for x, y in scaled if x > 0 and y > 0] if len(valid_points) > 0: scaled_width = max(x for x, y in valid_points) scaled_height = max(y for x, y in valid_points) margin = 5 offset_x = max(margin, (frame_width - scaled_width) // 2) offset_x = min(offset_x, frame_width - scaled_width - margin) offset_x = max(0, offset_x) # Only use line distribution analysis if we detected a reasonable number of lines # Otherwise fall back to default centering if total_line_count > 100: # Minimum threshold to trust the analysis # Calculate the bounding box of selected keypoints in scaled coordinates selected_scaled_points = [(x, y) for i, (x, y) in enumerate(scaled) if i in selected_keypoint_indices and x > 0 and y > 0] if len(selected_scaled_points) > 0: min_y_selected = min(y for x, y in selected_scaled_points) max_y_selected = max(y for x, y in selected_scaled_points) min_x_selected = min(x for x, y in selected_scaled_points) max_x_selected = max(x for x, y in selected_scaled_points) # Determine which part of template the selected keypoints represent # Check template coordinates to determine if selected keypoints are from top/bottom/left/right if template_keypoints is not None: selected_template_points = [(template_keypoints[i][0], template_keypoints[i][1]) for i in selected_keypoint_indices if i < len(template_keypoints) and template_keypoints[i][0] > 0] if len(selected_template_points) > 0: template_min_y = min(y for x, y in selected_template_points) template_max_y = max(y for x, y in selected_template_points) template_min_x = min(x for x, y in selected_template_points) template_max_x = max(x for x, y in selected_template_points) template_height = max(y for x, y in template_keypoints if x > 0) if template_keypoints else 675 template_width = max(x for x, y in template_keypoints if x > 0) if template_keypoints else 1045 # Determine if selected keypoints are from top, bottom, left, or right part is_top_part = template_min_y < template_height * 0.4 # Top 40% of template is_bottom_part = template_max_y > template_height * 0.6 # Bottom 40% of template is_left_part = template_min_x < template_width * 0.4 # Left 40% of template is_right_part = template_max_x > template_width * 0.6 # Right 40% of template # Position selected keypoints to align with where lines are detected if top_line_count > bottom_line_count: # Lines detected at top -> align selected keypoints with top region if is_bottom_part: # Selected bottom part -> position so its top edge aligns with top of frame offset_y = margin - min_y_selected # Shift so min_y_selected aligns with margin elif is_top_part: # Selected top part -> position at top offset_y = margin - min_y_selected else: # Mixed or center -> position at top offset_y = margin else: # Lines detected at bottom -> align selected keypoints with bottom region if is_top_part: # Selected top part -> position so its bottom edge aligns with bottom of frame offset_y = frame_height - max_y_selected - margin # Shift so max_y_selected aligns with bottom elif is_bottom_part: # Selected bottom part -> position at bottom offset_y = frame_height - max_y_selected - margin else: # Mixed or center -> position at bottom offset_y = frame_height - scaled_height - margin # Horizontal alignment if left_line_count > right_line_count: # Lines detected at left -> align selected keypoints with left region if is_right_part: offset_x = margin - min_x_selected elif is_left_part: offset_x = margin - min_x_selected else: offset_x = max(margin, (frame_width - scaled_width) // 2) else: # Lines detected at right -> align selected keypoints with right region if is_left_part: offset_x = frame_width - max_x_selected - margin elif is_right_part: offset_x = frame_width - max_x_selected - margin else: offset_x = max(margin, (frame_width - scaled_width) // 2) else: # Fallback to simple positioning if top_line_count > bottom_line_count: offset_y = margin else: offset_y = frame_height - scaled_height - margin else: # Fallback if no template_keypoints available if top_line_count > bottom_line_count: offset_y = margin else: offset_y = frame_height - scaled_height - margin # Ensure reasonable bounds - keep pitch within frame min_visible_height = scaled_height * 0.3 offset_y = max(margin, min(offset_y, frame_height - scaled_height - margin)) # Ensure at least some portion of pitch is visible if offset_y + scaled_height < min_visible_height: offset_y = frame_height - min_visible_height - margin if offset_y > frame_height - min_visible_height: offset_y = margin else: # Not enough lines detected, use default centering offset_y = max(margin, (frame_height - scaled_height) // 2) offset_y = min(offset_y, frame_height - scaled_height - margin) offset_y = max(0, offset_y) else: # Default centering if no line distribution analysis valid_points = [(x, y) for x, y in scaled if x > 0 and y > 0] if len(valid_points) > 0: scaled_width = max(x for x, y in valid_points) scaled_height = max(y for x, y in valid_points) margin = 5 offset_x = max(margin, (frame_width - scaled_width) // 2) offset_y = max(margin, (frame_height - scaled_height) // 2) offset_x = min(offset_x, frame_width - scaled_width - margin) offset_y = min(offset_y, frame_height - scaled_height - margin) offset_x = max(0, offset_x) offset_y = max(0, offset_y) # Apply centering offset centered = [] for x, y in scaled: if x == 0 and y == 0: centered.append((0, 0)) else: new_x = x + offset_x new_y = y + offset_y # Allow negative y coordinates (pitch extends above frame) # But ensure x coordinates are within frame bounds to avoid warping artifacts new_x = max(0, min(new_x, frame_width - 1)) # Allow negative y, but ensure at least some keypoints are in frame # This prevents large square artifacts from warping centered.append((new_x, new_y)) # Ensure at least some keypoints have positive y coordinates (visible in frame) # This prevents warping from creating large square artifacts visible_keypoints = [kp for kp in centered if kp[1] > 0] if len(visible_keypoints) < 4: # Not enough visible keypoints - adjust offset_y to ensure visibility # This prevents warping artifacts that create large squares min_y = min(y for x, y in centered if y != 0) if visible_keypoints else 0 if min_y < 0: adjustment = abs(min_y) + 10 # Push down by at least 10 pixels centered = [] for x, y in scaled: if x == 0 and y == 0: centered.append((0, 0)) else: new_x = x + offset_x new_y = y + offset_y + adjustment new_x = max(0, min(new_x, frame_width - 1)) new_y = max(0, new_y) # Ensure at least some are visible centered.append((new_x, new_y)) return centered def _adjust_keypoints_to_pass_validation( keypoints: list[tuple[int, int]], template_keypoints: list[tuple[int, int]] = None, frame_width: int = None, frame_height: int = None, ) -> list[tuple[int, int]]: """ Adjust keypoints to pass validate_projected_corners. If validation fails, try to fix by ensuring corners form a valid quadrilateral. """ if _validate_keypoints_corners(keypoints, template_keypoints): return keypoints # Already valid # If validation fails, try to fix by ensuring corner keypoints are in correct order try: from keypoint_evaluation import ( TEMPLATE_KEYPOINTS, INDEX_KEYPOINT_CORNER_BOTTOM_LEFT, INDEX_KEYPOINT_CORNER_BOTTOM_RIGHT, INDEX_KEYPOINT_CORNER_TOP_LEFT, INDEX_KEYPOINT_CORNER_TOP_RIGHT, ) if template_keypoints is None: template_keypoints = TEMPLATE_KEYPOINTS # Get corner indices corner_indices = [ INDEX_KEYPOINT_CORNER_TOP_LEFT, INDEX_KEYPOINT_CORNER_TOP_RIGHT, INDEX_KEYPOINT_CORNER_BOTTOM_RIGHT, INDEX_KEYPOINT_CORNER_BOTTOM_LEFT, ] # Check if we have all corner keypoints corners = [] for idx in corner_indices: if idx < len(keypoints): x, y = keypoints[idx] if x > 0 and y > 0: corners.append((x, y, idx)) if len(corners) < 4: # Not enough corners - can't fix, return original return keypoints # Extract corner coordinates corner_coords = [(x, y) for x, y, _ in corners] # Check if corners form a bowtie (twisted quadrilateral) # A bowtie occurs when opposite edges intersect def segments_intersect(p1, p2, q1, q2): """Check if line segments p1-p2 and q1-q2 intersect.""" def ccw(a, b, c): return (c[1] - a[1]) * (b[0] - a[0]) > (b[1] - a[1]) * (c[0] - a[0]) return (ccw(p1, q1, q2) != ccw(p2, q1, q2)) and (ccw(p1, p2, q1) != ccw(p1, p2, q2)) # Try different corner orderings to find a valid one # Current order: top-left, top-right, bottom-right, bottom-left # If this creates a bowtie, we need to reorder # Sort corners by position to get proper order # Top row (smaller y values) top_corners = sorted([c for c in corners if c[1] <= np.mean([c[1] for c in corners])], key=lambda c: c[0]) # Bottom row (larger y values) bottom_corners = sorted([c for c in corners if c[1] > np.mean([c[1] for c in corners])], key=lambda c: c[0]) # If we have 2 top and 2 bottom corners, ensure proper ordering if len(top_corners) == 2 and len(bottom_corners) == 2: # Ensure left < right if top_corners[0][0] > top_corners[1][0]: top_corners = top_corners[::-1] if bottom_corners[0][0] > bottom_corners[1][0]: bottom_corners = bottom_corners[::-1] # Reconstruct with proper order: top-left, top-right, bottom-right, bottom-left result = list(keypoints) # Map to corner indices corner_mapping = { INDEX_KEYPOINT_CORNER_TOP_LEFT: top_corners[0], INDEX_KEYPOINT_CORNER_TOP_RIGHT: top_corners[1], INDEX_KEYPOINT_CORNER_BOTTOM_RIGHT: bottom_corners[1], INDEX_KEYPOINT_CORNER_BOTTOM_LEFT: bottom_corners[0], } for corner_idx, (x, y, _) in corner_mapping.items(): if corner_idx < len(result): result[corner_idx] = (x, y) # Validate the adjusted keypoints if _validate_keypoints_corners(result, template_keypoints): return result # Alternative: If we can't fix by reordering, try using template-based scaling # for corners only, keeping other keypoints as-is if len(corners) >= 4: # Calculate scale from non-corner keypoints if available non_corner_kps = [(i, keypoints[i]) for i in range(len(keypoints)) if i not in corner_indices and keypoints[i][0] > 0 and keypoints[i][1] > 0] if len(non_corner_kps) >= 2: # Use template scaling approach scales_x = [] scales_y = [] for idx, (x, y) in non_corner_kps: if idx < len(template_keypoints): tx, ty = template_keypoints[idx] if tx > 0: scales_x.append(x / tx) if ty > 0: scales_y.append(y / ty) if scales_x and scales_y: avg_scale_x = sum(scales_x) / len(scales_x) avg_scale_y = sum(scales_y) / len(scales_y) result = list(keypoints) # Recalculate corners using template scaling for corner_idx in corner_indices: if corner_idx < len(template_keypoints): tx, ty = template_keypoints[corner_idx] new_x = int(round(tx * avg_scale_x)) new_y = int(round(ty * avg_scale_y)) if corner_idx < len(result): result[corner_idx] = (new_x, new_y) # Validate again if _validate_keypoints_corners(result, template_keypoints): return result except Exception: pass # If we can't fix, return original return keypoints def fix_keypoints( results_frames: Sequence[Any], failed_indices: Sequence[int], frame_width: int, frame_height: int, template_keypoints: list[tuple[int, int]] = None, frames: List[np.ndarray] = None, floor_markings_template: np.ndarray = None, offset: int = 0, ) -> list[Any]: max_frames = len(results_frames) if max_frames == 0: return list(results_frames) failed_set = set(int(i) for i in failed_indices) all_indices = list(range(max_frames)) successful_indices = [i for i in all_indices if i not in failed_set] # Use actual frame dimensions instead of hardcoded values # Using actual dimensions ensures keypoints match the frame and avoid warping artifacts # if len(successful_indices) == 0: # for frame_result in results_frames: # setattr(frame_result, "keypoints", list(convert_keypoints_to_val_format(sparse_template))) # return list(results_frames) # seed_index = successful_indices[0] # seed_kps_raw = getattr(results_frames[seed_index], "keypoints", []) or [] # last_success_kps = convert_keypoints_to_val_format(seed_kps_raw) # # Validate and adjust seed keypoints # last_success_kps = _adjust_keypoints_to_pass_validation( # last_success_kps, template_keypoints, frame_width, frame_height # ) last_success_kps = None for frame_index in range(max_frames): frame_result = results_frames[frame_index] if frame_index in failed_set and last_success_kps is not None: # Substitute last_success_kps and validate/adjust adjusted_kps = _adjust_keypoints_to_pass_validation( last_success_kps, template_keypoints, frame_width, frame_height ) setattr(frame_result, "keypoints", list(adjusted_kps)) else: current_kps_raw = getattr(frame_result, "keypoints", []) or [] current_kps = convert_keypoints_to_val_format(current_kps_raw) last_success_kps = current_kps setattr(frame_result, "keypoints", list(current_kps)) # Get original detected keypoints from frame_result original_keypoints_raw = getattr(frame_result, "keypoints", []) or [] original_keypoints = convert_keypoints_to_val_format(original_keypoints_raw) # Check if original keypoints are valid (have at least some non-zero keypoints) original_keypoints_valid = len([kp for kp in original_keypoints if kp[0] != 0 or kp[1] != 0]) >= 4 # Generate sparse template keypoints with line distribution analysis for this frame frame_for_analysis = None template_for_analysis = None if frames is not None and frame_index < len(frames): frame_for_analysis = frames[frame_index] if floor_markings_template is not None: template_for_analysis = floor_markings_template # Generate sparse template keypoints for this specific frame sparse_template = _generate_sparse_template_keypoints( frame_width, frame_height, frame_image=frame_for_analysis, template_image=template_for_analysis, template_keypoints=template_keypoints ) # Evaluate both keypoint sets and choose the one with higher score final_keypoints = sparse_template sparse_score = 0.0 original_score = 0.0 # Only evaluate if we have frame/template and original keypoints might be better if frame_for_analysis is not None and template_for_analysis is not None and template_keypoints is not None: try: from keypoint_evaluation import evaluate_keypoints_for_frame # Evaluate sparse template keypoints first try: sparse_score = evaluate_keypoints_for_frame( template_keypoints=template_keypoints, frame_keypoints=sparse_template, frame=frame_for_analysis, floor_markings_template=template_for_analysis, ) except Exception as e: # If evaluation fails, use sparse_template as fallback sparse_score = 0.0 # Only evaluate original keypoints if: # 1. They are valid # 2. Sparse score is not already very high (>= 0.8) - skip if sparse is already good should_evaluate_original = original_keypoints_valid and sparse_score < 0.8 if should_evaluate_original: try: original_score = evaluate_keypoints_for_frame( template_keypoints=template_keypoints, frame_keypoints=original_keypoints, frame=frame_for_analysis, floor_markings_template=template_for_analysis, ) except Exception as e: # If evaluation fails, keep sparse_template original_score = 0.0 else: if not original_keypoints_valid: # Original keypoints are invalid, set score to -1 to ensure sparse_template is used original_score = -1.0 else: # Sparse score is already high, skip expensive original evaluation original_score = sparse_score - 0.01 # Slightly lower to prefer sparse # Choose the keypoints with higher score if original_score > sparse_score: final_keypoints = original_keypoints print(f"Frame {frame_index}: Using original keypoints (score: {original_score:.4f} > sparse: {sparse_score:.4f})") else: final_keypoints = sparse_template if original_keypoints_valid: print(f"Frame {frame_index}: Using sparse template keypoints (score: {sparse_score:.4f} >= original: {original_score:.4f})") else: print(f"Frame {frame_index}: Using sparse template keypoints (score: {sparse_score:.4f}, original keypoints invalid)") except Exception as e: # If evaluation fails completely, use sparse_template as default print(f"Frame {frame_index}: Could not evaluate keypoints, using sparse template: {e}") final_keypoints = sparse_template else: # If we don't have frame/template for evaluation, use sparse_template final_keypoints = sparse_template setattr(frame_result, "keypoints", list(convert_keypoints_to_val_format(final_keypoints))) # frame_image = frames[frame_index] # if frame_index in failed_set: # # Substitute last_success_kps and validate/adjust # adjusted_kps = _adjust_keypoints_to_pass_validation( # last_success_kps, template_keypoints, frame_width, frame_height # ) # check_success, adjusted_kps = adjust_keypoints_to_avoid_invalid_mask( # adjusted_kps, template_keypoints, frame_image, floor_markings_template # ) # if check_success: # setattr(frame_result, "keypoints", list(adjusted_kps)) # else: # setattr(frame_result, "keypoints", list(convert_keypoints_to_val_format(sparse_template))) # # setattr(frame_result, "keypoints", list(convert_keypoints_to_val_format(sparse_template))) # else: # current_kps_raw = getattr(frame_result, "keypoints", []) or [] # current_kps = convert_keypoints_to_val_format(current_kps_raw) # last_success_kps = current_kps # setattr(frame_result, "keypoints", list(current_kps)) return list(results_frames) def run_keypoints_post_processing( results_frames: Sequence[Any], frame_width: int, frame_height: int, frames: List[np.ndarray] = None, template_keypoints: list[tuple[int, int]] = None, floor_markings_template: np.ndarray = None, template_image_path: str = None, offset: int = 0, ) -> list[Any]: """ Post-process keypoints with validation and adjustment. Args: results_frames: Sequence of frame results with keypoints frame_width: Frame width frame_height: Frame height frames: Optional list of frame images for validation template_keypoints: Optional template keypoints (defaults to TEMPLATE_KEYPOINTS) floor_markings_template: Optional template image for validation template_image_path: Optional path to template image (will load if not provided) offset: Frame offset for tracking (defaults to 0) Returns: List of processed frame results """ # Load template_keypoints and floor_markings_template if not provided if template_keypoints is None or floor_markings_template is None: try: from keypoint_evaluation import ( load_template_from_file, TEMPLATE_KEYPOINTS, ) if template_keypoints is None: template_keypoints = TEMPLATE_KEYPOINTS if floor_markings_template is None: if template_image_path is None: # Try to find template in common locations from pathlib import Path possible_paths = [ Path("football_pitch_template.png"), Path("templates/football_pitch_template.png"), ] template_image_path = None for path in possible_paths: if path.exists(): template_image_path = str(path) break if template_image_path is not None: loaded_template_image, loaded_template_keypoints = load_template_from_file(template_image_path) floor_markings_template = loaded_template_image if template_keypoints is None: template_keypoints = loaded_template_keypoints else: # If not found, we'll skip full validation but still do basic checks print("Warning: Template image not found, skipping full mask validation") except ImportError: pass except Exception as e: print(f"Warning: Could not load template: {e}") failed_indices = predict_failed_indices( results_frames, template_keypoints, frame_width, frame_height, frames, floor_markings_template, offset ) return fix_keypoints( results_frames, failed_indices, frame_width, frame_height, template_keypoints, frames, floor_markings_template, offset )