score / miner.py
vincentwarrior's picture
Rename Miner.py to miner.py
c17b96f verified
Raw History Blame Contribute Delete
18.4 kB
import cv2
import numpy as np
from pathlib import Path
from ultralytics import YOLO
from numpy import ndarray
from pydantic import BaseModel
from typing import List, Tuple, Dict
from sklearn.cluster import KMeans
from scipy.optimize import least_squares
class BoundingBox(BaseModel):
x1: int
y1: int
x2: int
y2: int
cls_id: int
conf: float
class TVFrameResult(BaseModel):
frame_id: int
boxes: list[BoundingBox]
keypoints: list[Tuple[int, int]]
def get_grass_color(img: np.ndarray) -> Tuple[int, int, int]:
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
lower_green = np.array([30, 40, 40])
upper_green = np.array([80, 255, 255])
mask = cv2.inRange(hsv, lower_green, upper_green)
grass_color = cv2.mean(img, mask=mask)
return grass_color[:3]
def get_players_boxes(result):
players_imgs, players_boxes = [], []
for box in result.boxes:
label = int(box.cls.cpu().numpy()[0])
if label == 0: # Player class (cls_id=0 is Player)
x1, y1, x2, y2 = map(int, box.xyxy[0].cpu().numpy())
crop = result.orig_img[y1:y2, x1:x2]
if crop.size > 0:
players_imgs.append(crop)
players_boxes.append((x1, y1, x2, y2))
return players_imgs, players_boxes
def get_kits_colors(players, grass_hsv=None, frame=None):
kits_colors = []
if grass_hsv is None:
grass_color = get_grass_color(frame)
grass_hsv = cv2.cvtColor(np.uint8([[list(grass_color)]]), cv2.COLOR_BGR2HSV)
for player_img in players:
hsv = cv2.cvtColor(player_img, cv2.COLOR_BGR2HSV)
lower_green = np.array([grass_hsv[0, 0, 0] - 10, 40, 40])
upper_green = np.array([grass_hsv[0, 0, 0] + 10, 255, 255])
mask = cv2.inRange(hsv, lower_green, upper_green)
mask = cv2.bitwise_not(mask)
upper_mask = np.zeros(player_img.shape[:2], np.uint8)
upper_mask[0 : player_img.shape[0] // 2, :] = 255
mask = cv2.bitwise_and(mask, upper_mask)
kit_color = np.array(cv2.mean(player_img, mask=mask)[:3])
kits_colors.append(kit_color)
return kits_colors
def interpolate_pitch_keypoints(frames, K, verbose=False):
frames = [np.array(f, dtype=float) for f in frames]
num_frames = len(frames)
num_points = 32
def is_valid_point(pt):
"""Check if point is valid (not (0,0) and not nan)"""
return (
not (pt[0] == 0 and pt[1] == 0)
and np.isfinite(pt[0])
and np.isfinite(pt[1])
)
frames_to_interpolate = []
for fi, frame in enumerate(frames):
valid_count = sum(1 for pt in frame if is_valid_point(pt))
if 2 <= valid_count <= 3:
frames_to_interpolate.append(fi)
if verbose:
print(
f"Frame {fi}: {valid_count} valid points - will attempt interpolation"
)
elif valid_count >= 4:
if verbose:
print(
f"Frame {fi}: {valid_count} valid points - no interpolation needed"
)
else:
if verbose:
print(
f"Frame {fi}: {valid_count} valid points - too few to interpolate"
)
for target_frame_idx in frames_to_interpolate:
if verbose:
print(f"\n{'='*60}")
print(f"Processing frame {target_frame_idx}")
print(f"{'='*60}")
target_frame = frames[target_frame_idx]
valid_points_in_target = [
pi for pi in range(num_points) if is_valid_point(target_frame[pi])
]
missing_points_in_target = [
pi for pi in range(num_points) if not is_valid_point(target_frame[pi])
]
if verbose:
print(f"Valid points in target frame: {valid_points_in_target}")
print(f"Missing points in target frame: {missing_points_in_target}")
points_to_interpolate = []
for pi in missing_points_in_target:
# Count how many frames (excluding target) have this point
history_count = 0
for fi in range(num_frames):
if fi != target_frame_idx and is_valid_point(frames[fi][pi]):
history_count += 1
if history_count >= 2: # Need at least 2 observations to interpolate
points_to_interpolate.append(pi)
if verbose:
print(
f" Point {pi}: {history_count} observations in other frames - can interpolate"
)
else:
if verbose:
print(
f" Point {pi}: {history_count} observations in other frames - cannot interpolate"
)
if not points_to_interpolate:
if verbose:
print(f"No points can be interpolated for frame {target_frame_idx}")
continue
# Build a subset optimization problem
points_for_optimization = list(
set(valid_points_in_target + points_to_interpolate)
)
points_for_optimization.sort()
if verbose:
print(f"Points used for optimization: {points_for_optimization}")
# Build observation list for these specific points
obs = []
for fi in range(num_frames):
for pi in points_for_optimization:
pt = frames[fi][pi]
if is_valid_point(pt):
obs.append((fi, pi, pt[0], pt[1]))
if verbose:
print(f"Total observations: {len(obs)}")
if len(obs) < 6: # Need at least 6 observations for meaningful optimization
if verbose:
print(f"Not enough observations for frame {target_frame_idx}")
continue
# Create mapping from point index to optimization index
point_idx_to_opt_idx = {pi: i for i, pi in enumerate(points_for_optimization)}
num_opt_points = len(points_for_optimization)
# Initialize 3D points (simple grid initialization)
points3d_init = []
for i, pi in enumerate(points_for_optimization):
x = (pi % 8) * 1.0
y = (pi // 8) * 1.0
points3d_init.extend([x, y, 10.0])
points3d_init = np.array(points3d_init)
# Initialize camera poses
rvecs_init = np.zeros((num_frames, 3))
tvecs_init = np.array(
[[0, 0, i * 0.1] for i in range(num_frames)], dtype=float
).ravel()
# Pack parameters
def pack_params(points3d, rvecs, tvecs):
return np.hstack([points3d.ravel(), rvecs.ravel(), tvecs.ravel()])
def unpack_params(x):
p = x[: num_opt_points * 3].reshape((num_opt_points, 3))
off = num_opt_points * 3
r = x[off : off + 3 * num_frames].reshape((num_frames, 3))
t = x[off + 3 * num_frames : off + 6 * num_frames].reshape((num_frames, 3))
return p, r, t
# Residual function
def reproj_residuals(x):
P3d, rvecs, tvecs = unpack_params(x)
res = []
for fi, pi, ux, uy in obs:
opt_idx = point_idx_to_opt_idx[pi]
rvec = rvecs[fi]
tvec = tvecs[fi]
R, _ = cv2.Rodrigues(rvec)
Xc = R.dot(P3d[opt_idx]) + tvec
if Xc[2] <= 0: # Behind camera
res.extend([1000.0, 1000.0])
else:
xproj = K.dot(Xc)
xproj = xproj[:2] / xproj[2]
res.extend([xproj[0] - ux, xproj[1] - uy])
return np.array(res)
# Run optimization
x0 = pack_params(points3d_init, rvecs_init, tvecs_init.reshape((num_frames, 3)))
try:
res_lsq = least_squares(
reproj_residuals,
x0,
verbose=0,
xtol=1e-8,
ftol=1e-8,
max_nfev=1000,
loss="soft_l1", # Robust loss to handle outliers
)
# Extract results
P3d_est, rvecs_est, tvecs_est = unpack_params(res_lsq.x)
# Project missing points in target frame
rvec = rvecs_est[target_frame_idx]
tvec = tvecs_est[target_frame_idx]
R, _ = cv2.Rodrigues(rvec)
if verbose:
print(f"\nInterpolation results for frame {target_frame_idx}:")
for pi in points_to_interpolate:
opt_idx = point_idx_to_opt_idx[pi]
Xc = R.dot(P3d_est[opt_idx]) + tvec
if Xc[2] > 0: # In front of camera
xproj = K.dot(Xc)
predicted_pt = (xproj[:2] / xproj[2]).ravel()
# Update the frame with interpolated point
frames[target_frame_idx][pi] = predicted_pt
if verbose:
print(
f" Point {pi}: ({predicted_pt[0]:.2f}, {predicted_pt[1]:.2f})"
)
else:
if verbose:
print(f" Point {pi}: Behind camera, cannot interpolate")
if verbose:
print(f"Optimization cost: {res_lsq.cost:.4f}")
except Exception as e:
if verbose:
print(f"Optimization failed for frame {target_frame_idx}: {e}")
return frames
class Miner:
def __init__(self, path_hf_repo: Path) -> None:
self.bbox_model = YOLO(path_hf_repo / "detection.pt")
print("✅ BBox Model Loaded")
self.keypoints_model = YOLO(path_hf_repo / "keypoint.pt")
print("✅ Keypoints Model Loaded")
self.team_kmeans = None
self.left_team_label = 0
self.grass_hsv = None
self.team_classifier_fitted = False
self.labels = []
self.corner_indices = {0, 5, 24, 29}
def __repr__(self) -> str:
return (
f"BBox Model: {type(self.bbox_model).__name__}\n"
f"Keypoints Model: {type(self.keypoints_model).__name__}\n"
)
def fit_team_classifier(self, frame: np.ndarray) -> None:
result = self.bbox_model(frame, conf=0.2, verbose=False)[0]
players_imgs, players_boxes = get_players_boxes(result)
if len(players_imgs) == 0:
print("⚠️ No players found for team fitting.")
return
kits_colors = get_kits_colors(players_imgs, frame=frame)
if len(kits_colors) < 2:
print(
f"⚠️ Chỉ tìm thấy {len(kits_colors)} cầu thủ, không đủ để phân thành 2 đội. Bỏ qua việc fit."
)
return
self.team_kmeans = KMeans(n_clusters=2, random_state=42)
self.team_kmeans.fit(kits_colors)
self.team_classifier_fitted = True
print(f"✅ Team KMeans fitted on {len(kits_colors)} players")
team_assignments = self.team_kmeans.predict(kits_colors)
team_0_x = [
players_boxes[i][0] for i, t in enumerate(team_assignments) if t == 0
]
team_1_x = [
players_boxes[i][0] for i, t in enumerate(team_assignments) if t == 1
]
if len(team_0_x) and len(team_1_x):
avg0, avg1 = np.mean(team_0_x), np.mean(team_1_x)
self.left_team_label = 0 if avg0 < avg1 else 1
print(f"🏳️ Left team label: {self.left_team_label}")
grass_color = get_grass_color(frame)
self.grass_hsv = cv2.cvtColor(
np.uint8([[list(grass_color)]]), cv2.COLOR_BGR2HSV
)
def predict_batch(
self,
batch_images: list[ndarray],
offset: int,
n_keypoints: int,
) -> list[TVFrameResult]:
if batch_images:
height, width = batch_images[0].shape[:2]
K = np.array(
[[1200, 0, width / 2], [0, 1200, height / 2], [0, 0, 1]], dtype=float
)
results: list[TVFrameResult] = []
detected_boxes = []
for i, frame in enumerate(batch_images):
if not self.team_classifier_fitted:
self.fit_team_classifier(frame)
bbox_result = self.bbox_model(frame, conf=0.29, verbose=False)[0]
boxes = []
if bbox_result and bbox_result.boxes is not None:
players_imgs, players_boxes = get_players_boxes(bbox_result)
kits_colors = get_kits_colors(players_imgs, self.grass_hsv, frame)
if len(kits_colors) > 0 and self.team_kmeans is not None:
teams = self.team_kmeans.predict(kits_colors)
else:
teams = []
player_indices = []
for idx, box in enumerate(bbox_result.boxes):
cls_id = int(box.cls.cpu().numpy()[0])
if cls_id == 0:
player_indices.append(idx)
team_predictions = {}
if len(player_indices) > 0 and len(teams) > 0:
for player_idx, team_id in zip(player_indices, teams):
if team_id == self.left_team_label:
team_predictions[player_idx] = 6
else:
team_predictions[player_idx] = 7
for idx, box in enumerate(bbox_result.boxes):
x1, y1, x2, y2 = map(int, box.xyxy[0].cpu().numpy())
conf = float(box.conf.cpu().numpy()[0])
cls_id = int(box.cls.cpu().numpy()[0])
if cls_id not in self.labels:
self.labels.append(cls_id)
if idx in team_predictions:
cls_id = team_predictions[idx]
elif cls_id == 0:
cls_id = 2
elif cls_id == 1:
cls_id = 1
elif cls_id == 2:
cls_id = 0
elif cls_id in [
3,
4,
]:
cls_id = 3
else:
continue
boxes.append(
BoundingBox(
x1=x1, y1=y1, x2=x2, y2=y2, cls_id=cls_id, conf=conf
)
)
if 2 not in self.labels:
bbox_result = self.bbox_model(frame, conf=0.1, verbose=False)[0]
for idx, box in enumerate(bbox_result.boxes):
x1, y1, x2, y2 = map(int, box.xyxy[0].cpu().numpy())
conf = float(box.conf.cpu().numpy()[0])
cls_id = int(box.cls.cpu().numpy()[0])
if cls_id == 2: # YOLO Ball -> Validator Ball (0)
boxes.append(
BoundingBox(x1=x1, y1=y1, x2=x2, y2=y2, cls_id=0, conf=conf)
)
detected_boxes.append(boxes)
keypoints: Dict[int, List[Tuple[int, int]]] = {}
original_keypints = []
keypoints_model_results = self.keypoints_model.predict(batch_images)
for frame_idx_in_batch, detection in enumerate(keypoints_model_results):
if not hasattr(detection, "keypoints") or detection.keypoints is None:
continue
frame_keypoints_with_conf: List[Tuple[int, int, float]] = []
for i, part_points in enumerate(detection.keypoints.data):
for k_id, (x, y, _) in enumerate(part_points):
confidence = float(detection.keypoints.conf[i][k_id])
frame_keypoints_with_conf.append((int(x), int(y), confidence))
if len(frame_keypoints_with_conf) < n_keypoints:
frame_keypoints_with_conf.extend(
[(0, 0, 0.0)] * (n_keypoints - len(frame_keypoints_with_conf))
)
else:
frame_keypoints_with_conf = frame_keypoints_with_conf[:n_keypoints]
filtered_keypoints: List[Tuple[int, int]] = []
for idx, (x, y, confidence) in enumerate(frame_keypoints_with_conf):
if idx in self.corner_indices:
if confidence < 0.3:
filtered_keypoints.append((0, 0))
else:
filtered_keypoints.append((int(x), int(y)))
else:
if confidence < 0.4:
filtered_keypoints.append((0, 0))
else:
filtered_keypoints.append((int(x), int(y)))
keypoints[offset + frame_idx_in_batch] = filtered_keypoints
sanitized_keypoints = []
for x, y in filtered_keypoints:
if x == width or y == height:
sanitized_keypoints.append((0, 0))
else:
sanitized_keypoints.append((x, y))
original_keypints.append(sanitized_keypoints)
interploated_keypoints = interpolate_pitch_keypoints(original_keypints, K)
print(original_keypints)
print(K)
print(interploated_keypoints)
for frame_number in range(offset, offset + len(batch_images)):
results.append(
TVFrameResult(
frame_id=frame_number,
boxes=detected_boxes[frame_number - offset],
# keypoints=keypoints.get(
# frame_number,
# [(0, 0) for _ in range(n_keypoints)],
# ),
keypoints=[
(int(x), int(y))
for (x, y) in interploated_keypoints[frame_number - offset]
],
)
)
return results