import cv2 import numpy as np from pathlib import Path from ultralytics import YOLO from numpy import ndarray from pydantic import BaseModel from typing import List, Tuple, Dict from sklearn.cluster import KMeans from scipy.optimize import least_squares class BoundingBox(BaseModel): x1: int y1: int x2: int y2: int cls_id: int conf: float class TVFrameResult(BaseModel): frame_id: int boxes: list[BoundingBox] keypoints: list[Tuple[int, int]] def get_grass_color(img: np.ndarray) -> Tuple[int, int, int]: hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV) lower_green = np.array([30, 40, 40]) upper_green = np.array([80, 255, 255]) mask = cv2.inRange(hsv, lower_green, upper_green) grass_color = cv2.mean(img, mask=mask) return grass_color[:3] def get_players_boxes(result): players_imgs, players_boxes = [], [] for box in result.boxes: label = int(box.cls.cpu().numpy()[0]) if label == 0: # Player class (cls_id=0 is Player) x1, y1, x2, y2 = map(int, box.xyxy[0].cpu().numpy()) crop = result.orig_img[y1:y2, x1:x2] if crop.size > 0: players_imgs.append(crop) players_boxes.append((x1, y1, x2, y2)) return players_imgs, players_boxes def get_kits_colors(players, grass_hsv=None, frame=None): kits_colors = [] if grass_hsv is None: grass_color = get_grass_color(frame) grass_hsv = cv2.cvtColor(np.uint8([[list(grass_color)]]), cv2.COLOR_BGR2HSV) for player_img in players: hsv = cv2.cvtColor(player_img, cv2.COLOR_BGR2HSV) lower_green = np.array([grass_hsv[0, 0, 0] - 10, 40, 40]) upper_green = np.array([grass_hsv[0, 0, 0] + 10, 255, 255]) mask = cv2.inRange(hsv, lower_green, upper_green) mask = cv2.bitwise_not(mask) upper_mask = np.zeros(player_img.shape[:2], np.uint8) upper_mask[0 : player_img.shape[0] // 2, :] = 255 mask = cv2.bitwise_and(mask, upper_mask) kit_color = np.array(cv2.mean(player_img, mask=mask)[:3]) kits_colors.append(kit_color) return kits_colors def interpolate_pitch_keypoints(frames, K, verbose=False): frames = [np.array(f, dtype=float) for f in frames] num_frames = len(frames) num_points = 32 def is_valid_point(pt): """Check if point is valid (not (0,0) and not nan)""" return ( not (pt[0] == 0 and pt[1] == 0) and np.isfinite(pt[0]) and np.isfinite(pt[1]) ) frames_to_interpolate = [] for fi, frame in enumerate(frames): valid_count = sum(1 for pt in frame if is_valid_point(pt)) if 2 <= valid_count <= 3: frames_to_interpolate.append(fi) if verbose: print( f"Frame {fi}: {valid_count} valid points - will attempt interpolation" ) elif valid_count >= 4: if verbose: print( f"Frame {fi}: {valid_count} valid points - no interpolation needed" ) else: if verbose: print( f"Frame {fi}: {valid_count} valid points - too few to interpolate" ) for target_frame_idx in frames_to_interpolate: if verbose: print(f"\n{'='*60}") print(f"Processing frame {target_frame_idx}") print(f"{'='*60}") target_frame = frames[target_frame_idx] valid_points_in_target = [ pi for pi in range(num_points) if is_valid_point(target_frame[pi]) ] missing_points_in_target = [ pi for pi in range(num_points) if not is_valid_point(target_frame[pi]) ] if verbose: print(f"Valid points in target frame: {valid_points_in_target}") print(f"Missing points in target frame: {missing_points_in_target}") points_to_interpolate = [] for pi in missing_points_in_target: # Count how many frames (excluding target) have this point history_count = 0 for fi in range(num_frames): if fi != target_frame_idx and is_valid_point(frames[fi][pi]): history_count += 1 if history_count >= 2: # Need at least 2 observations to interpolate points_to_interpolate.append(pi) if verbose: print( f" Point {pi}: {history_count} observations in other frames - can interpolate" ) else: if verbose: print( f" Point {pi}: {history_count} observations in other frames - cannot interpolate" ) if not points_to_interpolate: if verbose: print(f"No points can be interpolated for frame {target_frame_idx}") continue # Build a subset optimization problem points_for_optimization = list( set(valid_points_in_target + points_to_interpolate) ) points_for_optimization.sort() if verbose: print(f"Points used for optimization: {points_for_optimization}") # Build observation list for these specific points obs = [] for fi in range(num_frames): for pi in points_for_optimization: pt = frames[fi][pi] if is_valid_point(pt): obs.append((fi, pi, pt[0], pt[1])) if verbose: print(f"Total observations: {len(obs)}") if len(obs) < 6: # Need at least 6 observations for meaningful optimization if verbose: print(f"Not enough observations for frame {target_frame_idx}") continue # Create mapping from point index to optimization index point_idx_to_opt_idx = {pi: i for i, pi in enumerate(points_for_optimization)} num_opt_points = len(points_for_optimization) # Initialize 3D points (simple grid initialization) points3d_init = [] for i, pi in enumerate(points_for_optimization): x = (pi % 8) * 1.0 y = (pi // 8) * 1.0 points3d_init.extend([x, y, 10.0]) points3d_init = np.array(points3d_init) # Initialize camera poses rvecs_init = np.zeros((num_frames, 3)) tvecs_init = np.array( [[0, 0, i * 0.1] for i in range(num_frames)], dtype=float ).ravel() # Pack parameters def pack_params(points3d, rvecs, tvecs): return np.hstack([points3d.ravel(), rvecs.ravel(), tvecs.ravel()]) def unpack_params(x): p = x[: num_opt_points * 3].reshape((num_opt_points, 3)) off = num_opt_points * 3 r = x[off : off + 3 * num_frames].reshape((num_frames, 3)) t = x[off + 3 * num_frames : off + 6 * num_frames].reshape((num_frames, 3)) return p, r, t # Residual function def reproj_residuals(x): P3d, rvecs, tvecs = unpack_params(x) res = [] for fi, pi, ux, uy in obs: opt_idx = point_idx_to_opt_idx[pi] rvec = rvecs[fi] tvec = tvecs[fi] R, _ = cv2.Rodrigues(rvec) Xc = R.dot(P3d[opt_idx]) + tvec if Xc[2] <= 0: # Behind camera res.extend([1000.0, 1000.0]) else: xproj = K.dot(Xc) xproj = xproj[:2] / xproj[2] res.extend([xproj[0] - ux, xproj[1] - uy]) return np.array(res) # Run optimization x0 = pack_params(points3d_init, rvecs_init, tvecs_init.reshape((num_frames, 3))) try: res_lsq = least_squares( reproj_residuals, x0, verbose=0, xtol=1e-8, ftol=1e-8, max_nfev=1000, loss="soft_l1", # Robust loss to handle outliers ) # Extract results P3d_est, rvecs_est, tvecs_est = unpack_params(res_lsq.x) # Project missing points in target frame rvec = rvecs_est[target_frame_idx] tvec = tvecs_est[target_frame_idx] R, _ = cv2.Rodrigues(rvec) if verbose: print(f"\nInterpolation results for frame {target_frame_idx}:") for pi in points_to_interpolate: opt_idx = point_idx_to_opt_idx[pi] Xc = R.dot(P3d_est[opt_idx]) + tvec if Xc[2] > 0: # In front of camera xproj = K.dot(Xc) predicted_pt = (xproj[:2] / xproj[2]).ravel() # Update the frame with interpolated point frames[target_frame_idx][pi] = predicted_pt if verbose: print( f" Point {pi}: ({predicted_pt[0]:.2f}, {predicted_pt[1]:.2f})" ) else: if verbose: print(f" Point {pi}: Behind camera, cannot interpolate") if verbose: print(f"Optimization cost: {res_lsq.cost:.4f}") except Exception as e: if verbose: print(f"Optimization failed for frame {target_frame_idx}: {e}") return frames class Miner: def __init__(self, path_hf_repo: Path) -> None: self.bbox_model = YOLO(path_hf_repo / "detection.pt") print("✅ BBox Model Loaded") self.keypoints_model = YOLO(path_hf_repo / "keypoint.pt") print("✅ Keypoints Model Loaded") self.team_kmeans = None self.left_team_label = 0 self.grass_hsv = None self.team_classifier_fitted = False self.labels = [] self.corner_indices = {0, 5, 24, 29} def __repr__(self) -> str: return ( f"BBox Model: {type(self.bbox_model).__name__}\n" f"Keypoints Model: {type(self.keypoints_model).__name__}\n" ) def fit_team_classifier(self, frame: np.ndarray) -> None: result = self.bbox_model(frame, conf=0.2, verbose=False)[0] players_imgs, players_boxes = get_players_boxes(result) if len(players_imgs) == 0: print("⚠️ No players found for team fitting.") return kits_colors = get_kits_colors(players_imgs, frame=frame) if len(kits_colors) < 2: print( f"⚠️ Chỉ tìm thấy {len(kits_colors)} cầu thủ, không đủ để phân thành 2 đội. Bỏ qua việc fit." ) return self.team_kmeans = KMeans(n_clusters=2, random_state=42) self.team_kmeans.fit(kits_colors) self.team_classifier_fitted = True print(f"✅ Team KMeans fitted on {len(kits_colors)} players") team_assignments = self.team_kmeans.predict(kits_colors) team_0_x = [ players_boxes[i][0] for i, t in enumerate(team_assignments) if t == 0 ] team_1_x = [ players_boxes[i][0] for i, t in enumerate(team_assignments) if t == 1 ] if len(team_0_x) and len(team_1_x): avg0, avg1 = np.mean(team_0_x), np.mean(team_1_x) self.left_team_label = 0 if avg0 < avg1 else 1 print(f"🏳️ Left team label: {self.left_team_label}") grass_color = get_grass_color(frame) self.grass_hsv = cv2.cvtColor( np.uint8([[list(grass_color)]]), cv2.COLOR_BGR2HSV ) def predict_batch( self, batch_images: list[ndarray], offset: int, n_keypoints: int, ) -> list[TVFrameResult]: if batch_images: height, width = batch_images[0].shape[:2] K = np.array( [[1200, 0, width / 2], [0, 1200, height / 2], [0, 0, 1]], dtype=float ) results: list[TVFrameResult] = [] detected_boxes = [] for i, frame in enumerate(batch_images): if not self.team_classifier_fitted: self.fit_team_classifier(frame) bbox_result = self.bbox_model(frame, conf=0.29, verbose=False)[0] boxes = [] if bbox_result and bbox_result.boxes is not None: players_imgs, players_boxes = get_players_boxes(bbox_result) kits_colors = get_kits_colors(players_imgs, self.grass_hsv, frame) if len(kits_colors) > 0 and self.team_kmeans is not None: teams = self.team_kmeans.predict(kits_colors) else: teams = [] player_indices = [] for idx, box in enumerate(bbox_result.boxes): cls_id = int(box.cls.cpu().numpy()[0]) if cls_id == 0: player_indices.append(idx) team_predictions = {} if len(player_indices) > 0 and len(teams) > 0: for player_idx, team_id in zip(player_indices, teams): if team_id == self.left_team_label: team_predictions[player_idx] = 6 else: team_predictions[player_idx] = 7 for idx, box in enumerate(bbox_result.boxes): x1, y1, x2, y2 = map(int, box.xyxy[0].cpu().numpy()) conf = float(box.conf.cpu().numpy()[0]) cls_id = int(box.cls.cpu().numpy()[0]) if cls_id not in self.labels: self.labels.append(cls_id) if idx in team_predictions: cls_id = team_predictions[idx] elif cls_id == 0: cls_id = 2 elif cls_id == 1: cls_id = 1 elif cls_id == 2: cls_id = 0 elif cls_id in [ 3, 4, ]: cls_id = 3 else: continue boxes.append( BoundingBox( x1=x1, y1=y1, x2=x2, y2=y2, cls_id=cls_id, conf=conf ) ) if 2 not in self.labels: bbox_result = self.bbox_model(frame, conf=0.1, verbose=False)[0] for idx, box in enumerate(bbox_result.boxes): x1, y1, x2, y2 = map(int, box.xyxy[0].cpu().numpy()) conf = float(box.conf.cpu().numpy()[0]) cls_id = int(box.cls.cpu().numpy()[0]) if cls_id == 2: # YOLO Ball -> Validator Ball (0) boxes.append( BoundingBox(x1=x1, y1=y1, x2=x2, y2=y2, cls_id=0, conf=conf) ) detected_boxes.append(boxes) keypoints: Dict[int, List[Tuple[int, int]]] = {} original_keypints = [] keypoints_model_results = self.keypoints_model.predict(batch_images) for frame_idx_in_batch, detection in enumerate(keypoints_model_results): if not hasattr(detection, "keypoints") or detection.keypoints is None: continue frame_keypoints_with_conf: List[Tuple[int, int, float]] = [] for i, part_points in enumerate(detection.keypoints.data): for k_id, (x, y, _) in enumerate(part_points): confidence = float(detection.keypoints.conf[i][k_id]) frame_keypoints_with_conf.append((int(x), int(y), confidence)) if len(frame_keypoints_with_conf) < n_keypoints: frame_keypoints_with_conf.extend( [(0, 0, 0.0)] * (n_keypoints - len(frame_keypoints_with_conf)) ) else: frame_keypoints_with_conf = frame_keypoints_with_conf[:n_keypoints] filtered_keypoints: List[Tuple[int, int]] = [] for idx, (x, y, confidence) in enumerate(frame_keypoints_with_conf): if idx in self.corner_indices: if confidence < 0.3: filtered_keypoints.append((0, 0)) else: filtered_keypoints.append((int(x), int(y))) else: if confidence < 0.4: filtered_keypoints.append((0, 0)) else: filtered_keypoints.append((int(x), int(y))) keypoints[offset + frame_idx_in_batch] = filtered_keypoints sanitized_keypoints = [] for x, y in filtered_keypoints: if x == width or y == height: sanitized_keypoints.append((0, 0)) else: sanitized_keypoints.append((x, y)) original_keypints.append(sanitized_keypoints) interploated_keypoints = interpolate_pitch_keypoints(original_keypints, K) print(original_keypints) print(K) print(interploated_keypoints) for frame_number in range(offset, offset + len(batch_images)): results.append( TVFrameResult( frame_id=frame_number, boxes=detected_boxes[frame_number - offset], # keypoints=keypoints.get( # frame_number, # [(0, 0) for _ in range(n_keypoints)], # ), keypoints=[ (int(x), int(y)) for (x, y) in interploated_keypoints[frame_number - offset] ], ) ) return results