Download miner.py from vincentwarrior/score: direct link, hf CLI and curl.
- Browser
- Download file 18.4 kB
-
https://huggingface.co/vincentwarrior/score/resolve/main/miner.py
- Command line
-
hf download hf://vincentwarrior/score/miner.py
-
curl -L -o miner.py https://huggingface.co/vincentwarrior/score/resolve/main/miner.py
18.4 kB
| import cv2 | |
| import numpy as np | |
| from pathlib import Path | |
| from ultralytics import YOLO | |
| from numpy import ndarray | |
| from pydantic import BaseModel | |
| from typing import List, Tuple, Dict | |
| from sklearn.cluster import KMeans | |
| from scipy.optimize import least_squares | |
| class BoundingBox(BaseModel): | |
| x1: int | |
| y1: int | |
| x2: int | |
| y2: int | |
| cls_id: int | |
| conf: float | |
| class TVFrameResult(BaseModel): | |
| frame_id: int | |
| boxes: list[BoundingBox] | |
| keypoints: list[Tuple[int, int]] | |
| def get_grass_color(img: np.ndarray) -> Tuple[int, int, int]: | |
| hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV) | |
| lower_green = np.array([30, 40, 40]) | |
| upper_green = np.array([80, 255, 255]) | |
| mask = cv2.inRange(hsv, lower_green, upper_green) | |
| grass_color = cv2.mean(img, mask=mask) | |
| return grass_color[:3] | |
| def get_players_boxes(result): | |
| players_imgs, players_boxes = [], [] | |
| for box in result.boxes: | |
| label = int(box.cls.cpu().numpy()[0]) | |
| if label == 0: # Player class (cls_id=0 is Player) | |
| x1, y1, x2, y2 = map(int, box.xyxy[0].cpu().numpy()) | |
| crop = result.orig_img[y1:y2, x1:x2] | |
| if crop.size > 0: | |
| players_imgs.append(crop) | |
| players_boxes.append((x1, y1, x2, y2)) | |
| return players_imgs, players_boxes | |
| def get_kits_colors(players, grass_hsv=None, frame=None): | |
| kits_colors = [] | |
| if grass_hsv is None: | |
| grass_color = get_grass_color(frame) | |
| grass_hsv = cv2.cvtColor(np.uint8([[list(grass_color)]]), cv2.COLOR_BGR2HSV) | |
| for player_img in players: | |
| hsv = cv2.cvtColor(player_img, cv2.COLOR_BGR2HSV) | |
| lower_green = np.array([grass_hsv[0, 0, 0] - 10, 40, 40]) | |
| upper_green = np.array([grass_hsv[0, 0, 0] + 10, 255, 255]) | |
| mask = cv2.inRange(hsv, lower_green, upper_green) | |
| mask = cv2.bitwise_not(mask) | |
| upper_mask = np.zeros(player_img.shape[:2], np.uint8) | |
| upper_mask[0 : player_img.shape[0] // 2, :] = 255 | |
| mask = cv2.bitwise_and(mask, upper_mask) | |
| kit_color = np.array(cv2.mean(player_img, mask=mask)[:3]) | |
| kits_colors.append(kit_color) | |
| return kits_colors | |
| def interpolate_pitch_keypoints(frames, K, verbose=False): | |
| frames = [np.array(f, dtype=float) for f in frames] | |
| num_frames = len(frames) | |
| num_points = 32 | |
| def is_valid_point(pt): | |
| """Check if point is valid (not (0,0) and not nan)""" | |
| return ( | |
| not (pt[0] == 0 and pt[1] == 0) | |
| and np.isfinite(pt[0]) | |
| and np.isfinite(pt[1]) | |
| ) | |
| frames_to_interpolate = [] | |
| for fi, frame in enumerate(frames): | |
| valid_count = sum(1 for pt in frame if is_valid_point(pt)) | |
| if 2 <= valid_count <= 3: | |
| frames_to_interpolate.append(fi) | |
| if verbose: | |
| print( | |
| f"Frame {fi}: {valid_count} valid points - will attempt interpolation" | |
| ) | |
| elif valid_count >= 4: | |
| if verbose: | |
| print( | |
| f"Frame {fi}: {valid_count} valid points - no interpolation needed" | |
| ) | |
| else: | |
| if verbose: | |
| print( | |
| f"Frame {fi}: {valid_count} valid points - too few to interpolate" | |
| ) | |
| for target_frame_idx in frames_to_interpolate: | |
| if verbose: | |
| print(f"\n{'='*60}") | |
| print(f"Processing frame {target_frame_idx}") | |
| print(f"{'='*60}") | |
| target_frame = frames[target_frame_idx] | |
| valid_points_in_target = [ | |
| pi for pi in range(num_points) if is_valid_point(target_frame[pi]) | |
| ] | |
| missing_points_in_target = [ | |
| pi for pi in range(num_points) if not is_valid_point(target_frame[pi]) | |
| ] | |
| if verbose: | |
| print(f"Valid points in target frame: {valid_points_in_target}") | |
| print(f"Missing points in target frame: {missing_points_in_target}") | |
| points_to_interpolate = [] | |
| for pi in missing_points_in_target: | |
| # Count how many frames (excluding target) have this point | |
| history_count = 0 | |
| for fi in range(num_frames): | |
| if fi != target_frame_idx and is_valid_point(frames[fi][pi]): | |
| history_count += 1 | |
| if history_count >= 2: # Need at least 2 observations to interpolate | |
| points_to_interpolate.append(pi) | |
| if verbose: | |
| print( | |
| f" Point {pi}: {history_count} observations in other frames - can interpolate" | |
| ) | |
| else: | |
| if verbose: | |
| print( | |
| f" Point {pi}: {history_count} observations in other frames - cannot interpolate" | |
| ) | |
| if not points_to_interpolate: | |
| if verbose: | |
| print(f"No points can be interpolated for frame {target_frame_idx}") | |
| continue | |
| # Build a subset optimization problem | |
| points_for_optimization = list( | |
| set(valid_points_in_target + points_to_interpolate) | |
| ) | |
| points_for_optimization.sort() | |
| if verbose: | |
| print(f"Points used for optimization: {points_for_optimization}") | |
| # Build observation list for these specific points | |
| obs = [] | |
| for fi in range(num_frames): | |
| for pi in points_for_optimization: | |
| pt = frames[fi][pi] | |
| if is_valid_point(pt): | |
| obs.append((fi, pi, pt[0], pt[1])) | |
| if verbose: | |
| print(f"Total observations: {len(obs)}") | |
| if len(obs) < 6: # Need at least 6 observations for meaningful optimization | |
| if verbose: | |
| print(f"Not enough observations for frame {target_frame_idx}") | |
| continue | |
| # Create mapping from point index to optimization index | |
| point_idx_to_opt_idx = {pi: i for i, pi in enumerate(points_for_optimization)} | |
| num_opt_points = len(points_for_optimization) | |
| # Initialize 3D points (simple grid initialization) | |
| points3d_init = [] | |
| for i, pi in enumerate(points_for_optimization): | |
| x = (pi % 8) * 1.0 | |
| y = (pi // 8) * 1.0 | |
| points3d_init.extend([x, y, 10.0]) | |
| points3d_init = np.array(points3d_init) | |
| # Initialize camera poses | |
| rvecs_init = np.zeros((num_frames, 3)) | |
| tvecs_init = np.array( | |
| [[0, 0, i * 0.1] for i in range(num_frames)], dtype=float | |
| ).ravel() | |
| # Pack parameters | |
| def pack_params(points3d, rvecs, tvecs): | |
| return np.hstack([points3d.ravel(), rvecs.ravel(), tvecs.ravel()]) | |
| def unpack_params(x): | |
| p = x[: num_opt_points * 3].reshape((num_opt_points, 3)) | |
| off = num_opt_points * 3 | |
| r = x[off : off + 3 * num_frames].reshape((num_frames, 3)) | |
| t = x[off + 3 * num_frames : off + 6 * num_frames].reshape((num_frames, 3)) | |
| return p, r, t | |
| # Residual function | |
| def reproj_residuals(x): | |
| P3d, rvecs, tvecs = unpack_params(x) | |
| res = [] | |
| for fi, pi, ux, uy in obs: | |
| opt_idx = point_idx_to_opt_idx[pi] | |
| rvec = rvecs[fi] | |
| tvec = tvecs[fi] | |
| R, _ = cv2.Rodrigues(rvec) | |
| Xc = R.dot(P3d[opt_idx]) + tvec | |
| if Xc[2] <= 0: # Behind camera | |
| res.extend([1000.0, 1000.0]) | |
| else: | |
| xproj = K.dot(Xc) | |
| xproj = xproj[:2] / xproj[2] | |
| res.extend([xproj[0] - ux, xproj[1] - uy]) | |
| return np.array(res) | |
| # Run optimization | |
| x0 = pack_params(points3d_init, rvecs_init, tvecs_init.reshape((num_frames, 3))) | |
| try: | |
| res_lsq = least_squares( | |
| reproj_residuals, | |
| x0, | |
| verbose=0, | |
| xtol=1e-8, | |
| ftol=1e-8, | |
| max_nfev=1000, | |
| loss="soft_l1", # Robust loss to handle outliers | |
| ) | |
| # Extract results | |
| P3d_est, rvecs_est, tvecs_est = unpack_params(res_lsq.x) | |
| # Project missing points in target frame | |
| rvec = rvecs_est[target_frame_idx] | |
| tvec = tvecs_est[target_frame_idx] | |
| R, _ = cv2.Rodrigues(rvec) | |
| if verbose: | |
| print(f"\nInterpolation results for frame {target_frame_idx}:") | |
| for pi in points_to_interpolate: | |
| opt_idx = point_idx_to_opt_idx[pi] | |
| Xc = R.dot(P3d_est[opt_idx]) + tvec | |
| if Xc[2] > 0: # In front of camera | |
| xproj = K.dot(Xc) | |
| predicted_pt = (xproj[:2] / xproj[2]).ravel() | |
| # Update the frame with interpolated point | |
| frames[target_frame_idx][pi] = predicted_pt | |
| if verbose: | |
| print( | |
| f" Point {pi}: ({predicted_pt[0]:.2f}, {predicted_pt[1]:.2f})" | |
| ) | |
| else: | |
| if verbose: | |
| print(f" Point {pi}: Behind camera, cannot interpolate") | |
| if verbose: | |
| print(f"Optimization cost: {res_lsq.cost:.4f}") | |
| except Exception as e: | |
| if verbose: | |
| print(f"Optimization failed for frame {target_frame_idx}: {e}") | |
| return frames | |
| class Miner: | |
| def __init__(self, path_hf_repo: Path) -> None: | |
| self.bbox_model = YOLO(path_hf_repo / "detection.pt") | |
| print("✅ BBox Model Loaded") | |
| self.keypoints_model = YOLO(path_hf_repo / "keypoint.pt") | |
| print("✅ Keypoints Model Loaded") | |
| self.team_kmeans = None | |
| self.left_team_label = 0 | |
| self.grass_hsv = None | |
| self.team_classifier_fitted = False | |
| self.labels = [] | |
| self.corner_indices = {0, 5, 24, 29} | |
| def __repr__(self) -> str: | |
| return ( | |
| f"BBox Model: {type(self.bbox_model).__name__}\n" | |
| f"Keypoints Model: {type(self.keypoints_model).__name__}\n" | |
| ) | |
| def fit_team_classifier(self, frame: np.ndarray) -> None: | |
| result = self.bbox_model(frame, conf=0.2, verbose=False)[0] | |
| players_imgs, players_boxes = get_players_boxes(result) | |
| if len(players_imgs) == 0: | |
| print("⚠️ No players found for team fitting.") | |
| return | |
| kits_colors = get_kits_colors(players_imgs, frame=frame) | |
| if len(kits_colors) < 2: | |
| print( | |
| f"⚠️ Chỉ tìm thấy {len(kits_colors)} cầu thủ, không đủ để phân thành 2 đội. Bỏ qua việc fit." | |
| ) | |
| return | |
| self.team_kmeans = KMeans(n_clusters=2, random_state=42) | |
| self.team_kmeans.fit(kits_colors) | |
| self.team_classifier_fitted = True | |
| print(f"✅ Team KMeans fitted on {len(kits_colors)} players") | |
| team_assignments = self.team_kmeans.predict(kits_colors) | |
| team_0_x = [ | |
| players_boxes[i][0] for i, t in enumerate(team_assignments) if t == 0 | |
| ] | |
| team_1_x = [ | |
| players_boxes[i][0] for i, t in enumerate(team_assignments) if t == 1 | |
| ] | |
| if len(team_0_x) and len(team_1_x): | |
| avg0, avg1 = np.mean(team_0_x), np.mean(team_1_x) | |
| self.left_team_label = 0 if avg0 < avg1 else 1 | |
| print(f"🏳️ Left team label: {self.left_team_label}") | |
| grass_color = get_grass_color(frame) | |
| self.grass_hsv = cv2.cvtColor( | |
| np.uint8([[list(grass_color)]]), cv2.COLOR_BGR2HSV | |
| ) | |
| def predict_batch( | |
| self, | |
| batch_images: list[ndarray], | |
| offset: int, | |
| n_keypoints: int, | |
| ) -> list[TVFrameResult]: | |
| if batch_images: | |
| height, width = batch_images[0].shape[:2] | |
| K = np.array( | |
| [[1200, 0, width / 2], [0, 1200, height / 2], [0, 0, 1]], dtype=float | |
| ) | |
| results: list[TVFrameResult] = [] | |
| detected_boxes = [] | |
| for i, frame in enumerate(batch_images): | |
| if not self.team_classifier_fitted: | |
| self.fit_team_classifier(frame) | |
| bbox_result = self.bbox_model(frame, conf=0.29, verbose=False)[0] | |
| boxes = [] | |
| if bbox_result and bbox_result.boxes is not None: | |
| players_imgs, players_boxes = get_players_boxes(bbox_result) | |
| kits_colors = get_kits_colors(players_imgs, self.grass_hsv, frame) | |
| if len(kits_colors) > 0 and self.team_kmeans is not None: | |
| teams = self.team_kmeans.predict(kits_colors) | |
| else: | |
| teams = [] | |
| player_indices = [] | |
| for idx, box in enumerate(bbox_result.boxes): | |
| cls_id = int(box.cls.cpu().numpy()[0]) | |
| if cls_id == 0: | |
| player_indices.append(idx) | |
| team_predictions = {} | |
| if len(player_indices) > 0 and len(teams) > 0: | |
| for player_idx, team_id in zip(player_indices, teams): | |
| if team_id == self.left_team_label: | |
| team_predictions[player_idx] = 6 | |
| else: | |
| team_predictions[player_idx] = 7 | |
| for idx, box in enumerate(bbox_result.boxes): | |
| x1, y1, x2, y2 = map(int, box.xyxy[0].cpu().numpy()) | |
| conf = float(box.conf.cpu().numpy()[0]) | |
| cls_id = int(box.cls.cpu().numpy()[0]) | |
| if cls_id not in self.labels: | |
| self.labels.append(cls_id) | |
| if idx in team_predictions: | |
| cls_id = team_predictions[idx] | |
| elif cls_id == 0: | |
| cls_id = 2 | |
| elif cls_id == 1: | |
| cls_id = 1 | |
| elif cls_id == 2: | |
| cls_id = 0 | |
| elif cls_id in [ | |
| 3, | |
| 4, | |
| ]: | |
| cls_id = 3 | |
| else: | |
| continue | |
| boxes.append( | |
| BoundingBox( | |
| x1=x1, y1=y1, x2=x2, y2=y2, cls_id=cls_id, conf=conf | |
| ) | |
| ) | |
| if 2 not in self.labels: | |
| bbox_result = self.bbox_model(frame, conf=0.1, verbose=False)[0] | |
| for idx, box in enumerate(bbox_result.boxes): | |
| x1, y1, x2, y2 = map(int, box.xyxy[0].cpu().numpy()) | |
| conf = float(box.conf.cpu().numpy()[0]) | |
| cls_id = int(box.cls.cpu().numpy()[0]) | |
| if cls_id == 2: # YOLO Ball -> Validator Ball (0) | |
| boxes.append( | |
| BoundingBox(x1=x1, y1=y1, x2=x2, y2=y2, cls_id=0, conf=conf) | |
| ) | |
| detected_boxes.append(boxes) | |
| keypoints: Dict[int, List[Tuple[int, int]]] = {} | |
| original_keypints = [] | |
| keypoints_model_results = self.keypoints_model.predict(batch_images) | |
| for frame_idx_in_batch, detection in enumerate(keypoints_model_results): | |
| if not hasattr(detection, "keypoints") or detection.keypoints is None: | |
| continue | |
| frame_keypoints_with_conf: List[Tuple[int, int, float]] = [] | |
| for i, part_points in enumerate(detection.keypoints.data): | |
| for k_id, (x, y, _) in enumerate(part_points): | |
| confidence = float(detection.keypoints.conf[i][k_id]) | |
| frame_keypoints_with_conf.append((int(x), int(y), confidence)) | |
| if len(frame_keypoints_with_conf) < n_keypoints: | |
| frame_keypoints_with_conf.extend( | |
| [(0, 0, 0.0)] * (n_keypoints - len(frame_keypoints_with_conf)) | |
| ) | |
| else: | |
| frame_keypoints_with_conf = frame_keypoints_with_conf[:n_keypoints] | |
| filtered_keypoints: List[Tuple[int, int]] = [] | |
| for idx, (x, y, confidence) in enumerate(frame_keypoints_with_conf): | |
| if idx in self.corner_indices: | |
| if confidence < 0.3: | |
| filtered_keypoints.append((0, 0)) | |
| else: | |
| filtered_keypoints.append((int(x), int(y))) | |
| else: | |
| if confidence < 0.4: | |
| filtered_keypoints.append((0, 0)) | |
| else: | |
| filtered_keypoints.append((int(x), int(y))) | |
| keypoints[offset + frame_idx_in_batch] = filtered_keypoints | |
| sanitized_keypoints = [] | |
| for x, y in filtered_keypoints: | |
| if x == width or y == height: | |
| sanitized_keypoints.append((0, 0)) | |
| else: | |
| sanitized_keypoints.append((x, y)) | |
| original_keypints.append(sanitized_keypoints) | |
| interploated_keypoints = interpolate_pitch_keypoints(original_keypints, K) | |
| print(original_keypints) | |
| print(K) | |
| print(interploated_keypoints) | |
| for frame_number in range(offset, offset + len(batch_images)): | |
| results.append( | |
| TVFrameResult( | |
| frame_id=frame_number, | |
| boxes=detected_boxes[frame_number - offset], | |
| # keypoints=keypoints.get( | |
| # frame_number, | |
| # [(0, 0) for _ in range(n_keypoints)], | |
| # ), | |
| keypoints=[ | |
| (int(x), int(y)) | |
| for (x, y) in interploated_keypoints[frame_number - offset] | |
| ], | |
| ) | |
| ) | |
| return results | |