mastefan commited on
Commit
a1d6f70
·
verified ·
1 Parent(s): e550069

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +204 -498
app.py CHANGED
@@ -13,578 +13,284 @@
13
  # 5) Gradio UI: video in → gallery of clips + status text out.
14
  #
15
  # Fencing Scoreboard Clips - YOLO x AutoGluon (Gradio)
16
-
17
- import os, sys, zipfile, shutil, subprocess, tempfile, pathlib
18
  from typing import List, Tuple
19
- import uuid
20
-
21
- import numpy as np
22
- import pandas as pd
23
- import cv2
24
  import gradio as gr
25
 
26
- # ---- Robust imports/installs for Colab/Spaces ----
27
- def _pip(pkgs: List[str]):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  import subprocess, sys
29
  subprocess.check_call([sys.executable, "-m", "pip", "install", "--quiet", *pkgs])
30
 
31
  try:
32
- import ultralytics
33
- except:
34
- _pip(["ultralytics"])
35
- import ultralytics
36
-
37
- try:
38
- import ffmpeg # optional helper for duration probe
39
  except:
40
- try:
41
- _pip(["ffmpeg-python"])
42
- import ffmpeg
43
- except Exception:
44
- ffmpeg = None
45
-
46
  try:
47
  from autogluon.tabular import TabularPredictor
48
  except:
49
- _pip(["autogluon.tabular"])
50
- from autogluon.tabular import TabularPredictor
51
-
52
  try:
53
  from huggingface_hub import hf_hub_download
54
  except:
55
- _pip(["huggingface_hub"])
56
- from huggingface_hub import hf_hub_download
57
-
58
- from ultralytics import YOLO
59
 
60
- # ----------------------------
61
- # Config — HF Hub repositories
62
- # ----------------------------
63
- # YOLO scoreboard detector weights (pushed by your training file)
64
- YOLO_REPO_ID = os.getenv("YOLO_REPO_ID", "mastefan/fencing-scoreboard-yolov8")
65
- YOLO_FILENAME = os.getenv("YOLO_FILENAME", "best.pt")
66
 
67
- # AutoGluon Tabular detector (your color/timeseries model zip)
68
- AG_REPO_ID = os.getenv("AG_REPO_ID", "emkessle/2024-24679-fencing-touch-predictor")
69
- AG_ZIP_NAME = os.getenv("AG_ZIP_NAME", "autogluon_predictor_dir.zip")
70
-
71
- # Processing parameters
72
- FRAME_SKIP = int(os.getenv("FRAME_SKIP", "2")) # process every Nth frame
73
- KEEP_CONF = float(os.getenv("KEEP_CONF", "0.70"))# YOLO conf to keep color inside bbox
74
- YOLO_CONF = float(os.getenv("YOLO_CONF", "0.25"))
75
- YOLO_IOU = float(os.getenv("YOLO_IOU", "0.50"))
76
- MIN_SEP_S = float(os.getenv("MIN_SEP_S", "1.2")) # min gap between events (s)
77
- CLIP_PAD_S = float(os.getenv("CLIP_PAD_S","2.0")) # before/after padding each hit
78
- GROUP_GAP_S = float(os.getenv("GROUP_GAP_S","1.5"))# cluster close frames to single event
79
-
80
- # ----------------
81
- # Model loaders
82
- # ----------------
83
- CACHE_DIR = pathlib.Path("hf_assets")
84
- CACHE_DIR.mkdir(parents=True, exist_ok=True)
85
-
86
- def load_yolo_from_hub() -> YOLO:
87
  w = hf_hub_download(repo_id=YOLO_REPO_ID, filename=YOLO_FILENAME, cache_dir=CACHE_DIR)
 
88
  return YOLO(w)
89
 
90
- def load_autogluon_tabular_from_hub() -> TabularPredictor:
91
  z = hf_hub_download(repo_id=AG_REPO_ID, filename=AG_ZIP_NAME, cache_dir=CACHE_DIR)
92
  extract_dir = CACHE_DIR / "ag_predictor_native"
93
- if extract_dir.exists():
94
- shutil.rmtree(extract_dir)
95
- with zipfile.ZipFile(z, "r") as zip_ref:
96
- zip_ref.extractall(extract_dir)
97
  return TabularPredictor.load(str(extract_dir))
98
 
99
  _YOLO = None
100
- _AG_PRED = None
101
-
102
- def yolo() -> YOLO:
103
  global _YOLO
104
- if _YOLO is None:
105
- _YOLO = load_yolo_from_hub()
106
  return _YOLO
 
 
 
 
107
 
108
- def ag_predictor() -> TabularPredictor:
109
- global _AG_PRED
110
- if _AG_PRED is None:
111
- _AG_PRED = load_autogluon_tabular_from_hub()
112
- return _AG_PRED
113
-
114
- # ----------------------------
115
- # Vision helpers
116
- # ----------------------------
117
- DEBUG_DIR = pathlib.Path("debug_frames")
118
- DEBUG_DIR.mkdir(exist_ok=True)
119
 
120
  def isolate_scoreboard_color(frame_bgr: np.ndarray,
121
  conf: float = YOLO_CONF,
122
  iou: float = YOLO_IOU,
123
  keep_conf: float = KEEP_CONF,
124
- debug: bool = False,
125
  frame_id: int = None) -> np.ndarray:
126
- """
127
- Reverted version:
128
- - Choose the largest bbox among candidates meeting confidence.
129
- - Primary threshold: >= max(0.80, keep_conf)
130
- - Fallback threshold: >= (primary - 0.02) (i.e., ~0.78 by default)
131
- - Entire chosen bbox is restored to color; everything else is grayscale.
132
- - Single safeguard: reject very low-saturation ROIs (likely flat/neutral areas).
133
- """
134
  H, W = frame_bgr.shape[:2]
135
-
136
- # start fully grayscale
137
  gray = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2GRAY)
138
  gray = cv2.cvtColor(gray, cv2.COLOR_GRAY2BGR)
139
 
140
- primary_thr = max(0.80, keep_conf) # accept ≥0.80 as "good"
141
- fallback_thr = max(0.7, primary_thr - 0.05) # accept ≥0.75 as fallback
142
-
143
-
144
  chosen_box = None
145
  res = yolo().predict(frame_bgr, conf=conf, iou=iou, verbose=False)
146
  if len(res):
147
  r = res[0]
148
  if getattr(r, "boxes", None) is not None and len(r.boxes) > 0:
149
- boxes = r.boxes.xyxy.cpu().numpy()
150
  scores = r.boxes.conf.cpu().numpy()
151
- candidates = list(zip(boxes, scores))
152
-
153
- # Prefer largest box that meets primary threshold
154
- strong = [(b, s) for (b, s) in candidates if float(s) >= primary_thr]
155
- if strong:
156
- chosen_box, _ = max(strong, key=lambda bs: (bs[0][2]-bs[0][0]) * (bs[0][3]-bs[0][1]))
157
- else:
158
- # Fallback: largest box meeting fallback threshold
159
- medium = [(b, s) for (b, s) in candidates if float(s) >= fallback_thr]
160
- if medium:
161
- chosen_box, _ = max(medium, key=lambda bs: (bs[0][2]-bs[0][0]) * (bs[0][3]-bs[0][1]))
162
 
163
- if chosen_box is not None:
164
- x1, y1, x2, y2 = [int(round(v)) for v in chosen_box]
165
- x1, y1 = max(0, x1), max(0, y1)
166
- x2, y2 = min(W-1, x2), min(H-1, y2)
167
-
168
- if x2 > x1 and y2 > y1:
169
- # Single safeguard: reject very low-saturation ROIs
170
- roi_color = frame_bgr[y1:y2, x1:x2]
171
- if roi_color.size > 0:
172
- hsv = cv2.cvtColor(roi_color, cv2.COLOR_BGR2HSV)
173
- sat_mean = hsv[:, :, 1].mean()
174
- if sat_mean < 25: # flat/neutral area → reject
175
- print(f"[WARN] Rejected bbox due to low saturation (mean={sat_mean:.1f})")
176
- chosen_box = None
177
-
178
- # If accepted, restore whole bbox to color
179
- if chosen_box is not None:
180
- gray[y1:y2, x1:x2] = frame_bgr[y1:y2, x1:x2]
181
-
182
- # Optional debug save
183
  if debug and frame_id is not None:
184
  dbg = gray.copy()
185
  if chosen_box is not None:
186
- x1, y1, x2, y2 = [int(round(v)) for v in chosen_box]
187
- cv2.rectangle(dbg, (x1, y1), (x2, y2), (0, 255, 0), 2)
188
- out_path = DEBUG_DIR / f"frame_{frame_id:06d}.jpg"
189
- cv2.imwrite(str(out_path), dbg)
190
- print(f"[DEBUG] Saved debug frame → {out_path}")
191
-
192
  return gray
193
 
194
-
195
- # Color features
196
- def _count_color_pixels(rgb: np.ndarray, ch: int,
197
- red_thresh=150, green_thresh=100,
198
- red_dom=1.2, green_dom=1.05) -> int:
199
- R, G, B = rgb[:, :, 0], rgb[:, :, 1], rgb[:, :, 2]
200
- if ch == 0:
201
- mask = (R > red_thresh) & (R > red_dom*G) & (R > red_dom*B)
202
- elif ch == 1:
203
- mask = (G > green_thresh) & (G > green_dom*R) & (G > green_dom*B)
204
- else:
205
- raise ValueError("ch must be 0 (red) or 1 (green)")
206
  return int(np.sum(mask))
207
 
208
- def color_pixel_ratio(rgb: np.ndarray, ch: int) -> float:
209
- return _count_color_pixels(rgb, ch) / float(rgb.shape[0]*rgb.shape[1] + 1e-9)
210
-
211
- def rolling_z(series: pd.Series, win: int = 45) -> pd.Series:
212
- med = series.rolling(win, min_periods=5).median()
213
- mad = series.rolling(win, min_periods=5).apply(
214
- lambda x: np.median(np.abs(x - np.median(x))), raw=True
215
- )
216
- mad = mad.replace(0, mad[mad > 0].min() if (mad > 0).any() else 1.0)
217
- return (series - med) / mad
218
-
219
- # ----------------------------
220
- # Video → feature table
221
- # ----------------------------
222
- def extract_feature_timeseries(video_path: str,
223
- frame_skip: int = FRAME_SKIP,
224
- debug: bool = False) -> Tuple[pd.DataFrame, float]:
225
- print("[INFO] Starting frame extraction...")
226
- cap = cv2.VideoCapture(video_path)
227
- if not cap.isOpened():
228
- print("[ERROR] Could not open video.")
229
- return pd.DataFrame(), 0.0
230
-
231
- fps = cap.get(cv2.CAP_PROP_FPS) or 30.0
232
- records, frame_idx = [], 0
233
- total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
234
-
235
  while True:
236
- ret, frame = cap.read()
237
- if not ret:
238
- break
239
- if frame_idx % frame_skip == 0:
240
- ts = frame_idx / fps
241
- masked = isolate_scoreboard_color(frame, debug=debug, frame_id=frame_idx)
242
- rgb = cv2.cvtColor(masked, cv2.COLOR_BGR2RGB)
243
- red_ratio = color_pixel_ratio(rgb, 0)
244
- green_ratio = color_pixel_ratio(rgb, 1)
245
- records.append({
246
- "frame_id": frame_idx,
247
- "timestamp": ts,
248
- "red_ratio": red_ratio,
249
- "green_ratio": green_ratio,
250
  })
251
- frame_idx += 1
252
-
253
  cap.release()
254
- df = pd.DataFrame(records)
255
- print(f"[INFO] Processed {len(df)} frames out of {total_frames} (fps={fps:.2f})")
256
-
257
- if df.empty:
258
- return df, fps
259
-
260
- df["red_diff"] = df["red_ratio"].diff().fillna(0)
261
- df["green_diff"] = df["green_ratio"].diff().fillna(0)
262
- df["z_red"] = rolling_z(df["red_ratio"])
263
- df["z_green"] = rolling_z(df["green_ratio"])
264
-
265
- if debug:
266
- out_csv = DEBUG_DIR / f"features_{uuid.uuid4().hex}.csv"
267
- df.to_csv(out_csv, index=False)
268
- print(f"[DEBUG] Saved feature CSV → {out_csv}")
269
-
270
- return df, fps
271
-
272
-
273
- # ----------------------------
274
- # AutoGluon inference + event picking
275
- # ----------------------------
276
- def predict_scores(df: pd.DataFrame) -> pd.Series:
277
- feat_cols = ["red_ratio", "green_ratio", "red_diff", "green_diff", "z_red", "z_green"]
278
- X = df[feat_cols].copy()
279
- pred = ag_predictor().predict(X)
280
-
281
- # Prefer classification proba if available
282
  try:
283
- proba = ag_predictor().predict_proba(X)
284
- if isinstance(proba, pd.DataFrame) and (1 in proba.columns):
285
- return proba[1]
286
- except Exception:
287
- pass
288
-
289
- # Fallback: normalize regression-like output to 0..1 robustly
290
- s = pd.Series(pred).astype(float)
291
- rng = (s.quantile(0.95) - s.quantile(0.05)) or 1.0
292
- return ((s - s.quantile(0.05)) / rng).clip(0, 1)
293
-
294
- def pick_events(df: pd.DataFrame, score: pd.Series, fps: float) -> List[float]:
295
- """
296
- Adaptive hybrid event detection:
297
- - Adaptive raw threshold = 0.7 × max score
298
- - Adaptive z-threshold = max(2.0, 0.6 × max z-score)
299
- - Must be a local peak
300
- - Enforce min spacing (1.0s) and group gap (1.5s)
301
- - Ignore any detections before 1.0s
302
- """
303
- max_score = score.max()
304
- raw_cutoff = 0.7 * max_score if max_score > 0 else 0.4
305
-
306
- z = rolling_z(score, win=45)
307
- max_z = z.max()
308
- z_cutoff = max(2.0, 0.6 * max_z)
309
-
310
- print(f"[DEBUG] Predictor score stats: min={score.min():.3f}, max={max_score:.3f}, mean={score.mean():.3f}")
311
- print(f"[DEBUG] Adaptive thresholds: raw>{raw_cutoff:.3f}, z>{z_cutoff:.2f}")
312
-
313
- out_times = []
314
- min_dist_frames = max(1, int(1.0 * max(1.0, fps))) # 1.0s spacing
315
- y = score.values
316
- last_kept = -min_dist_frames
317
-
318
- for i in range(1, len(y)-1):
319
- ts = float(df.iloc[i]["timestamp"])
320
- local_peak = y[i] > y[i-1] and y[i] > y[i+1]
321
- if ts >= 1.0 and ((z.iloc[i] > z_cutoff) or (y[i] > raw_cutoff)) and local_peak and (i - last_kept) >= min_dist_frames:
322
- out_times.append(ts)
323
- last_kept = i
324
-
325
- if not out_times and len(y) > 0:
326
- best_idx = int(np.argmax(y))
327
- ts = float(df.iloc[best_idx]["timestamp"])
328
- if ts >= 1.0:
329
- out_times = [ts]
330
- print(f"[DEBUG] Fallback → using global max at {ts:.2f}s")
331
- else:
332
- print(f"[DEBUG] Ignored fallback at {ts:.2f}s (within first second)")
333
-
334
- out_times.sort()
335
-
336
- grouped = []
337
- for t in out_times:
338
- if (not grouped) or (t - grouped[-1]) > GROUP_GAP_S:
339
- grouped.append(t)
340
-
341
- print(f"[DEBUG] Final detected events: {grouped}")
342
  return grouped
343
 
344
- def save_event_snapshot(video_path: str, timestamp: float, out_path: str, fps: float):
345
- """Save a snapshot frame at timestamp with YOLO bbox drawn."""
346
- cap = cv2.VideoCapture(video_path)
347
- if not cap.isOpened():
348
- print("[ERROR] Could not open video for snapshot.")
349
- return None
350
-
351
- frame_idx = int(timestamp * fps)
352
- cap.set(cv2.CAP_PROP_POS_FRAMES, frame_idx)
353
- ret, frame = cap.read()
354
- cap.release()
355
-
356
- if not ret or frame is None:
357
- print(f"[WARN] Could not grab frame at {timestamp:.2f}s")
358
- return None
359
-
360
- masked = isolate_scoreboard_color(frame, debug=False)
361
- res = yolo().predict(frame, conf=YOLO_CONF, iou=YOLO_IOU, verbose=False)
362
-
363
- if len(res) and getattr(res[0], "boxes", None) is not None and len(res[0].boxes) > 0:
364
- boxes = res[0].boxes.xyxy.cpu().numpy()
365
- scores = res[0].boxes.conf.cpu().numpy()
366
- valid = [(box, score) for box, score in zip(boxes, scores) if float(score) >= KEEP_CONF]
367
- if valid:
368
- largest, _ = max(valid, key=lambda bs: (bs[0][2]-bs[0][0])*(bs[0][3]-bs[0][1]))
369
- x1, y1, x2, y2 = [int(round(v)) for v in largest]
370
- cv2.rectangle(masked, (x1, y1), (x2, y2), (0, 255, 0), 3)
371
-
372
- cv2.imwrite(out_path, masked)
373
- print(f"[DEBUG] Saved snapshot → {out_path}")
374
- return out_path
375
-
376
- import matplotlib.pyplot as plt
377
- def save_debug_plot(df: pd.DataFrame, score: pd.Series, events: List[float], base_name="debug_plot"):
378
- plt.figure(figsize=(12, 5))
379
- plt.plot(df["timestamp"], score, label="Predicted Score")
380
- plt.axhline(y=0.5, color="gray", linestyle="--", alpha=0.5)
381
- first = True
382
- for ev in events:
383
- plt.axvline(x=ev, color="red", linestyle="--", label="Detected Event" if first else None)
384
- first = False
385
- plt.xlabel("Time (s)")
386
- plt.ylabel("Score")
387
- plt.title("AutoGluon Score vs Time")
388
- plt.legend()
389
- out_path = DEBUG_DIR / f"{base_name}.png"
390
- plt.savefig(out_path)
391
- plt.close()
392
- print(f"[DEBUG] Saved debug score plot → {out_path}")
393
-
394
-
395
- # ----------------------------
396
- # Clip cutting (ffmpeg w/ moviepy fallback)
397
- # ----------------------------
398
- def _probe_duration(video_path: str) -> float:
399
  try:
400
- if ffmpeg is None:
401
- raise RuntimeError("ffmpeg-python not available")
402
- meta = ffmpeg.probe(video_path)
403
  return float(meta["format"]["duration"])
404
- except Exception:
405
- return 0.0
406
 
407
- def cut_clip(video_path: str, start: float, end: float, out_path: str) -> str:
408
- # Fast path (copy) if ffmpeg available
409
  try:
410
- cmd = ["ffmpeg", "-y", "-ss", str(max(0, start)), "-to", str(max(start, end)),
411
- "-i", video_path, "-c", "copy", out_path]
412
- sp = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
413
- if sp.returncode == 0 and os.path.exists(out_path):
414
- return out_path
415
- except Exception:
416
- pass
417
-
418
- # Fallback: moviepy re-encode
419
  from moviepy.editor import VideoFileClip
420
- clip = VideoFileClip(video_path).subclip(max(0, start), max(start, end))
421
- clip.write_videofile(out_path, codec="libx264", audio_codec="aac", verbose=False, logger=None)
422
  return out_path
423
 
424
- # ----------------------------
425
- # Orchestrator: detect + cut + debug
426
- # ----------------------------
427
-
428
- def extract_score_clips(video_path: str, debug: bool = True) -> Tuple[List[Tuple[str, str]], str]:
429
- print("[INFO] Running full detection pipeline...")
430
- df, fps = extract_feature_timeseries(video_path, frame_skip=FRAME_SKIP, debug=debug)
431
- if df.empty:
432
- print("[WARN] Empty dataframe — no frames processed.")
433
- return [], "No frames processed."
434
-
435
- print("[INFO] Feature extraction done. Running predictor...")
436
- score = predict_scores(df)
437
-
438
- # Bail early if the model produced no signal at all
439
- if score.max() <= 1e-6:
440
- print("[WARN] Flat scores from predictor (possible YOLO miss or feature mismatch).")
441
- return [], "⚠️ No scoreboard detected or illumination scores flat. Please check video or model."
442
-
443
- print("[INFO] Picking events from predictor scores...")
444
- events = pick_events(df, score, fps)
445
- print(f"[INFO] Picked {len(events)} event(s): {events}")
446
-
447
- if not events:
448
- topk = np.argsort(score.values)[-5:][::-1]
449
- dbg = [(float(df.iloc[i]['timestamp']), float(score.iloc[i])) for i in topk]
450
- print(f"[DEBUG] Top-5 peaks (ts,score): {dbg}")
451
- return [], "⚠️ No touches confidently detected in this video."
452
-
453
- duration = _probe_duration(video_path)
454
- if duration <= 0:
455
- duration = float(df["timestamp"].max() + CLIP_PAD_S + 0.5)
456
-
457
- clips = []
458
- snapshots = []
459
- base = os.path.splitext(os.path.basename(video_path))[0]
460
- for i, t in enumerate(events):
461
- s = max(0.0, t - CLIP_PAD_S)
462
- e = min(duration, t + CLIP_PAD_S)
463
- clip_path = os.path.join(tempfile.gettempdir(), f"{base}_score_{i+1:02d}.mp4")
464
- img_path = os.path.join(tempfile.gettempdir(), f"{base}_score_{i+1:02d}.jpg")
465
- cut_clip(video_path, s, e, clip_path)
466
- save_event_snapshot(video_path, t, img_path, fps)
467
- label = f"Touch {i+1} @ {t:.2f}s"
468
- clips.append((clip_path, label))
469
- snapshots.append(img_path)
470
-
471
- if debug:
472
- debug_csv = DEBUG_DIR / f"scores_{base}.csv"
473
- pd.DataFrame({"timestamp": df["timestamp"], "score": score}).to_csv(debug_csv, index=False)
474
- print(f"[DEBUG] Saved score debug CSV → {debug_csv}")
475
- save_debug_plot(df, score, events, base_name=base)
476
- print(f"[DEBUG] Saved debug frames in {DEBUG_DIR}/")
477
-
478
- return clips, f"✅ Detected {len(clips)} event(s). Snapshots saved to temp."
479
-
480
- import time
481
-
482
- def looping_progress():
483
- """
484
- Infinite generator that loops the fencer animation from 0 → 100%.
485
- Yields progress bar HTML until stopped by the pipeline finishing.
486
- """
487
- while True:
488
- for i in range(101):
489
- bar = _make_progress_bar(i)
490
- yield gr.update(value=bar, visible=True)
491
- time.sleep(0.05) # controls speed of march (~5s per loop)
492
-
493
- # ----------------------------
494
- # Gradio UI
495
- # ----------------------------
496
  CSS = """
497
- .gradio-container {max-width: 900px; margin: auto;}
498
- .header {text-align: center; margin-bottom: 20px;}
499
- .full-width {width: 100% !important;}
500
- .progress-bar {
501
- width: 100%;
502
- height: 30px;
503
- background-color: #e0e0e0;
504
- border-radius: 15px;
505
- margin: 15px 0;
506
- position: relative;
507
- overflow: hidden;
508
- }
509
- .progress-fill {
510
- height: 100%;
511
- background-color: #4CAF50;
512
- border-radius: 15px;
513
- text-align: center;
514
- line-height: 30px;
515
- color: white;
516
- font-weight: bold;
517
- transition: width 0.3s;
518
- }
519
- .fencer {
520
- position: absolute;
521
- top: -5px;
522
- font-size: 24px;
523
- transition: left 0.3s;
524
- transform: scaleX(-1); /* flip to face right */
525
- }
526
  """
527
 
528
- def _make_progress_bar(percent: int, final_text: str = None):
529
- text = f"{percent}%" if not final_text else final_text
530
  return f"""
531
  <div class="progress-bar">
532
- <div id="progress-fill" class="progress-fill" style="width:{percent}%">{text}</div>
533
  <div id="fencer" class="fencer" style="left:{percent}%">🤺</div>
534
  </div>
535
  """
536
 
537
  def run_with_progress(video_file):
538
  if not video_file:
539
- yield [], "Please upload a video file.", gr.update(visible=False)
540
  return
541
-
542
- # Step 1: Extract frames + features
543
- yield [], "🔄 Extracting frames...", _make_progress_bar(20)
544
- df, fps = extract_feature_timeseries(video_file, frame_skip=FRAME_SKIP, debug=False)
545
  if df.empty:
546
- yield [], "❌ No frames processed!", _make_progress_bar(100, "No Frames ❌")
547
- return
548
-
549
- # Step 2–4: Predict & pick events via the single orchestrator
550
- yield [], "🔄 Scoring & detecting touches...", _make_progress_bar(80)
551
- clips, status_msg = extract_score_clips(video_file, debug=True)
552
-
553
- # Step 5: Done (and cutting already handled in orchestrator)
554
- final_bar = _make_progress_bar(100, f"Detected {len(clips)} Touches ⚡" if clips else "No Touches")
555
- yield clips, status_msg, final_bar
556
-
557
- with gr.Blocks(css=CSS, title="Fencing Scoreboard Detector") as demo:
558
- with gr.Row(elem_classes="header"):
559
- gr.Markdown(
560
- "## 🤺 Fencing Score Detector\n"
561
- "Upload a fencing bout video. We’ll detect scoreboard lights (YOLO + AutoGluon), "
562
- "and return 4-second highlight clips around each scoring event."
563
- )
564
-
565
- in_video = gr.Video(label="Upload Bout Video", elem_classes="full-width", height=400)
566
- run_btn = gr.Button("⚡ Detect Touches", elem_classes="full-width")
567
-
568
- progress_html = gr.HTML(value="", label="Processing Progress", visible=False)
569
- status = gr.Markdown("Ready.")
570
- gallery = gr.Gallery(
571
- label="Detected Clips",
572
- columns=1,
573
- height=400,
574
- preview=True,
575
- allow_preview=True,
576
- show_download_button=True,
577
- visible=False
578
- )
579
-
580
- def run(video_file):
581
- if not video_file:
582
- return [], "Please upload a video file."
583
- clips, status_msg = extract_score_clips(video_file, debug=False)
584
- return clips, status_msg
585
-
586
- run_btn.click(fn=run, inputs=in_video, outputs=[gallery, status])
587
-
588
- if __name__ == "__main__":
589
- demo.launch(server_name="0.0.0.0", server_port=7860, show_api=False)
590
-
 
13
  # 5) Gradio UI: video in → gallery of clips + status text out.
14
  #
15
  # Fencing Scoreboard Clips - YOLO x AutoGluon (Gradio)
16
+ import os, cv2, zipfile, shutil, tempfile, subprocess, pathlib
17
+ import numpy as np, pandas as pd
18
  from typing import List, Tuple
 
 
 
 
 
19
  import gradio as gr
20
 
21
+ # =====================================================
22
+ # Configuration
23
+ # =====================================================
24
+ YOLO_REPO_ID = "mastefan/fencing-scoreboard-yolov8"
25
+ YOLO_FILENAME = "best.pt"
26
+
27
+ AG_REPO_ID = "emkessle/2024-24679-fencing-touch-predictor"
28
+ AG_ZIP_NAME = "autogluon_predictor_dir.zip"
29
+
30
+ FRAME_SKIP = 2
31
+ KEEP_CONF = 0.70
32
+ YOLO_CONF = 0.25
33
+ YOLO_IOU = 0.50
34
+ CLIP_PAD_S = 2.0
35
+ MIN_SEP_S = 1.2
36
+ GROUP_GAP_S = 1.5
37
+
38
+ DEBUG_MODE = False # set True to save debug images/CSVs
39
+
40
+ # =====================================================
41
+ # Dependency setup
42
+ # =====================================================
43
+ def _pip(pkgs):
44
  import subprocess, sys
45
  subprocess.check_call([sys.executable, "-m", "pip", "install", "--quiet", *pkgs])
46
 
47
  try:
48
+ from ultralytics import YOLO
 
 
 
 
 
 
49
  except:
50
+ _pip(["ultralytics"]); from ultralytics import YOLO
 
 
 
 
 
51
  try:
52
  from autogluon.tabular import TabularPredictor
53
  except:
54
+ _pip(["autogluon.tabular"]); from autogluon.tabular import TabularPredictor
 
 
55
  try:
56
  from huggingface_hub import hf_hub_download
57
  except:
58
+ _pip(["huggingface_hub"]); from huggingface_hub import hf_hub_download
 
 
 
59
 
60
+ # =====================================================
61
+ # Model loading
62
+ # =====================================================
63
+ CACHE_DIR = pathlib.Path("hf_assets"); CACHE_DIR.mkdir(exist_ok=True)
 
 
64
 
65
+ def load_yolo_from_hub():
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
66
  w = hf_hub_download(repo_id=YOLO_REPO_ID, filename=YOLO_FILENAME, cache_dir=CACHE_DIR)
67
+ print(f"[INFO] Loaded YOLO weights from {w}")
68
  return YOLO(w)
69
 
70
+ def load_autogluon_tabular_from_hub():
71
  z = hf_hub_download(repo_id=AG_REPO_ID, filename=AG_ZIP_NAME, cache_dir=CACHE_DIR)
72
  extract_dir = CACHE_DIR / "ag_predictor_native"
73
+ if extract_dir.exists(): shutil.rmtree(extract_dir)
74
+ with zipfile.ZipFile(z, "r") as zip_ref: zip_ref.extractall(extract_dir)
75
+ print(f"[INFO] Loaded AutoGluon predictor from {extract_dir}")
 
76
  return TabularPredictor.load(str(extract_dir))
77
 
78
  _YOLO = None
79
+ _AGP = None
80
+ def yolo(): # lazy load
 
81
  global _YOLO
82
+ if _YOLO is None: _YOLO = load_yolo_from_hub()
 
83
  return _YOLO
84
+ def ag_predictor():
85
+ global _AGP
86
+ if _AGP is None: _AGP = load_autogluon_tabular_from_hub()
87
+ return _AGP
88
 
89
+ # =====================================================
90
+ # Image + feature utilities
91
+ # =====================================================
92
+ DEBUG_DIR = pathlib.Path("debug_frames"); DEBUG_DIR.mkdir(exist_ok=True)
 
 
 
 
 
 
 
93
 
94
  def isolate_scoreboard_color(frame_bgr: np.ndarray,
95
  conf: float = YOLO_CONF,
96
  iou: float = YOLO_IOU,
97
  keep_conf: float = KEEP_CONF,
98
+ debug: bool = DEBUG_MODE,
99
  frame_id: int = None) -> np.ndarray:
100
+ """Grayscale everything except the largest YOLO box ≥ keep_conf."""
 
 
 
 
 
 
 
101
  H, W = frame_bgr.shape[:2]
 
 
102
  gray = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2GRAY)
103
  gray = cv2.cvtColor(gray, cv2.COLOR_GRAY2BGR)
104
 
 
 
 
 
105
  chosen_box = None
106
  res = yolo().predict(frame_bgr, conf=conf, iou=iou, verbose=False)
107
  if len(res):
108
  r = res[0]
109
  if getattr(r, "boxes", None) is not None and len(r.boxes) > 0:
110
+ boxes = r.boxes.xyxy.cpu().numpy()
111
  scores = r.boxes.conf.cpu().numpy()
112
+ valid = [(b, s) for b, s in zip(boxes, scores) if float(s) >= keep_conf]
113
+ if valid:
114
+ chosen_box, _ = max(valid, key=lambda bs: (bs[0][2]-bs[0][0])*(bs[0][3]-bs[0][1]))
115
+ x1, y1, x2, y2 = [int(v) for v in chosen_box]
116
+ gray[y1:y2, x1:x2] = frame_bgr[y1:y2, x1:x2]
 
 
 
 
 
 
117
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
118
  if debug and frame_id is not None:
119
  dbg = gray.copy()
120
  if chosen_box is not None:
121
+ x1, y1, x2, y2 = [int(v) for v in chosen_box]
122
+ cv2.rectangle(dbg, (x1, y1), (x2, y2), (0,255,0), 2)
123
+ cv2.imwrite(str(DEBUG_DIR / f"frame_{frame_id:06d}.jpg"), dbg)
 
 
 
124
  return gray
125
 
126
+ def _count_color_pixels(rgb, ch):
127
+ R, G, B = rgb[:,:,0], rgb[:,:,1], rgb[:,:,2]
128
+ if ch==0: mask=(R>150)&(R>1.2*G)&(R>1.2*B)
129
+ else: mask=(G>100)&(G>1.05*R)&(G>1.05*B)
 
 
 
 
 
 
 
 
130
  return int(np.sum(mask))
131
 
132
+ def color_pixel_ratio(rgb,ch): return _count_color_pixels(rgb,ch)/(rgb.shape[0]*rgb.shape[1]+1e-9)
133
+
134
+ def rolling_z(series, win=45):
135
+ med = series.rolling(win,min_periods=5).median()
136
+ mad = series.rolling(win,min_periods=5).apply(lambda x: np.median(np.abs(x-np.median(x))),raw=True)
137
+ mad = mad.replace(0, mad[mad>0].min() if (mad>0).any() else 1.0)
138
+ return (series-med)/mad
139
+
140
+ # =====================================================
141
+ # Video feature extraction
142
+ # =====================================================
143
+ def extract_feature_timeseries(video_path:str, frame_skip:int=FRAME_SKIP, debug:bool=DEBUG_MODE):
144
+ cap=cv2.VideoCapture(video_path)
145
+ if not cap.isOpened(): return pd.DataFrame(),0.0
146
+ fps=cap.get(cv2.CAP_PROP_FPS) or 30.0
147
+ total=int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) or 1
148
+ print(f"[INFO] Reading {total} frames @ {fps:.2f}fps ...")
149
+
150
+ rec,idx=[],0
 
 
 
 
 
 
 
 
151
  while True:
152
+ ret,frame=cap.read()
153
+ if not ret: break
154
+ if idx%frame_skip==0:
155
+ ts=idx/fps
156
+ masked=isolate_scoreboard_color(frame,debug=debug,frame_id=idx)
157
+ rgb=cv2.cvtColor(masked,cv2.COLOR_BGR2RGB)
158
+ rec.append({
159
+ "frame_id":idx,"timestamp":ts,
160
+ "red_ratio":color_pixel_ratio(rgb,0),
161
+ "green_ratio":color_pixel_ratio(rgb,1)
 
 
 
 
162
  })
163
+ idx+=1
 
164
  cap.release()
165
+ df=pd.DataFrame(rec)
166
+ if df.empty: return df,fps
167
+ df["red_diff"]=df["red_ratio"].diff().fillna(0)
168
+ df["green_diff"]=df["green_ratio"].diff().fillna(0)
169
+ df["z_red"]=rolling_z(df["red_ratio"])
170
+ df["z_green"]=rolling_z(df["green_ratio"])
171
+ print(f"[INFO] Extracted {len(df)} processed frames.")
172
+ return df,fps
173
+
174
+ # =====================================================
175
+ # Predictor & event logic
176
+ # =====================================================
177
+ def predict_scores(df):
178
+ feats=["red_ratio","green_ratio","red_diff","green_diff","z_red","z_green"]
179
+ X=df[feats].copy()
180
+ ag=ag_predictor()
 
 
 
 
 
 
 
 
 
 
 
 
181
  try:
182
+ proba=ag.predict_proba(X)
183
+ if isinstance(proba,pd.DataFrame) and (1 in proba.columns): return proba[1]
184
+ except: pass
185
+ s=pd.Series(ag.predict(X)).astype(float)
186
+ rng=(s.quantile(0.95)-s.quantile(0.05)) or 1.0
187
+ return ((s-s.quantile(0.05))/rng).clip(0,1)
188
+
189
+ def pick_events(df,score,fps):
190
+ z=rolling_z(score,45); strong=(z>4.0); keep=strong.rolling(3,min_periods=1).sum()>=2
191
+ min_dist=max(1,int(MIN_SEP_S*fps))
192
+ y=score.values; out=[]; last=-min_dist
193
+ for i in range(1,len(y)-1):
194
+ if keep.iloc[i] and y[i]>y[i-1] and y[i]>y[i+1] and (i-last)>=min_dist:
195
+ out.append(float(df.iloc[i]["timestamp"])); last=i
196
+ if not out and len(y)>0: out=[float(df.iloc[int(np.argmax(y))]["timestamp"])]
197
+ grouped=[]
198
+ for t in sorted(out):
199
+ if (not grouped) or (t-grouped[-1])>GROUP_GAP_S: grouped.append(t)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
200
  return grouped
201
 
202
+ # =====================================================
203
+ # Clip utilities
204
+ # =====================================================
205
+ def _probe_duration(video_path):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
206
  try:
207
+ import ffmpeg
208
+ meta=ffmpeg.probe(video_path)
 
209
  return float(meta["format"]["duration"])
210
+ except: return 0.0
 
211
 
212
+ def cut_clip(video_path,start,end,out_path):
 
213
  try:
214
+ cmd=["ffmpeg","-y","-ss",str(start),"-to",str(end),"-i",video_path,"-c","copy",out_path]
215
+ sp=subprocess.run(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
216
+ if sp.returncode==0 and os.path.exists(out_path): return out_path
217
+ except: pass
 
 
 
 
 
218
  from moviepy.editor import VideoFileClip
219
+ clip=VideoFileClip(video_path).subclip(start,end)
220
+ clip.write_videofile(out_path,codec="libx264",audio_codec="aac",verbose=False,logger=None)
221
  return out_path
222
 
223
+ def extract_score_clips(video_path:str,debug:bool=DEBUG_MODE):
224
+ df,fps=extract_feature_timeseries(video_path,FRAME_SKIP,debug)
225
+ if df.empty: return [],"No frames processed."
226
+ score=predict_scores(df); events=pick_events(df,score,fps)
227
+ print(f"[INFO] Detected {len(events)} potential events: {events}")
228
+ dur=_probe_duration(video_path) or float(df["timestamp"].max()+CLIP_PAD_S+0.5)
229
+ out=[]; base=os.path.splitext(os.path.basename(video_path))[0]
230
+ for i,t in enumerate(events):
231
+ s=max(0,t-CLIP_PAD_S); e=min(dur,t+CLIP_PAD_S)
232
+ tmp=os.path.join(tempfile.gettempdir(),f"{base}_score_{i+1:02d}.mp4")
233
+ print(f"[INFO] Cutting clip {i+1}: {s:.2f}s→{e:.2f}s")
234
+ cut_clip(video_path,s,e,tmp)
235
+ out.append((tmp,f"Touch {i+1} @ {t:.2f}s"))
236
+ return out,f"✅ Detected {len(out)} event(s)."
237
+
238
+ # =====================================================
239
+ # Progress GUI helpers
240
+ # =====================================================
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
241
  CSS = """
242
+ .gradio-container {max-width:900px;margin:auto;}
243
+ .full-width{width:100%!important;}
244
+ .progress-bar{width:100%;height:30px;background:#e0e0e0;border-radius:15px;margin:15px 0;position:relative;overflow:hidden;}
245
+ .progress-fill{height:100%;background:#4CAF50;border-radius:15px;text-align:center;line-height:30px;color:white;font-weight:bold;transition:width .3s;}
246
+ .fencer{position:absolute;top:-5px;font-size:24px;transition:left .3s;transform:scaleX(-1);}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
247
  """
248
 
249
+ def _make_progress_bar(percent:int,final_text:str=None,label:str=""):
250
+ text=f"{percent}%" if not final_text else final_text
251
  return f"""
252
  <div class="progress-bar">
253
+ <div id="progress-fill" class="progress-fill" style="width:{percent}%">{label} {text}</div>
254
  <div id="fencer" class="fencer" style="left:{percent}%">🤺</div>
255
  </div>
256
  """
257
 
258
  def run_with_progress(video_file):
259
  if not video_file:
260
+ yield [],"Please upload a video.",_make_progress_bar(0)
261
  return
262
+ print("[GUI] Starting processing...")
263
+ yield [],"🔄 Extracting frames...",_make_progress_bar(20,"","Pipeline")
264
+ df,fps=extract_feature_timeseries(video_file,FRAME_SKIP,DEBUG_MODE)
 
265
  if df.empty:
266
+ yield [],"❌ No frames processed!",_make_progress_bar(100,"No Frames ❌","Pipeline");return
267
+ yield [],"🔄 YOLO masking...",_make_progress_bar(40,"","Pipeline")
268
+ yield [],"🔄 Feature analysis...",_make_progress_bar(60,"","Pipeline")
269
+ yield [],"🔄 Scoring...",_make_progress_bar(80,"","Pipeline")
270
+ clips,msg=extract_score_clips(video_file,DEBUG_MODE)
271
+ final=_make_progress_bar(100,f"Detected {len(clips)} Touches ⚡","Pipeline")
272
+ print("[GUI] Finished.")
273
+ yield clips,msg,final
274
+
275
+ # =====================================================
276
+ # Gradio interface
277
+ # =====================================================
278
+ with gr.Blocks(css=CSS,title="Fencing Scoreboard Detector") as demo:
279
+ gr.Markdown("## 🤺 Fencing Score Detector\nUpload a fencing bout video and automatically detect scoreboard lights using YOLO + AutoGluon.")
280
+ in_video=gr.Video(label="Upload Bout Video",elem_classes="full-width",height=400)
281
+ run_btn=gr.Button("⚡ Detect Touches",elem_classes="full-width")
282
+ progress_html=gr.HTML(value="",label="Progress",visible=False)
283
+ status=gr.Markdown("Ready.")
284
+ gallery=gr.Gallery(label="Detected Clips",columns=1,height=400,visible=False)
285
+
286
+ def wrapped_run(video_file):
287
+ print("[SYSTEM] User started detection.")
288
+ yield [],"Processing started...",gr.update(value=_make_progress_bar(0),visible=True)
289
+ for clips,msg,bar in run_with_progress(video_file):
290
+ print(f"[SYSTEM] {msg}")
291
+ yield gr.update(value=clips,visible=bool(clips)),msg,gr.update(value=bar,visible=True)
292
+
293
+ run_btn.click(fn=wrapped_run,inputs=in_video,outputs=[gallery,status,progress_html])
294
+
295
+ if __name__=="__main__":
296
+ demo.launch(debug=True)