Claude commited on
Commit
3a2f49a
·
1 Parent(s): f2b292d

Filter: detect instructor frames the frontal-face cascade misses

Browse files

Frontal-face Haar misses faces at angles, with hands raised, partly
turned, or partly off-frame — so the threshold check never even fired
for those frames and they slipped through regardless of how strict
the slider was set. Now we run three cascades in sequence and take
the largest detected region across all of them:

- frontal face (existing — head-on shots)
- profile face (instructor turned to one side; runs on both the
image and its horizontal flip so either side is
covered without doubling cascade count)
- upper body (head + shoulders silhouette — robust when the
face cascade fails because the angle, lighting,
or partial occlusion blocks face detection but
the silhouette is still obvious)

The three cascades are parameterised individually (different
min_neighbors / min_size) since upper-body benefits from stricter
clustering and a larger min size to avoid false positives on
diagrams. Existing threshold semantics unchanged: lower → drop more.

Five new tests in tests/test_filter.py:
- cascades load and include the new profile/upperbody detectors;
- blank uniform frames don't trigger (no false positives);
- missing file path doesn't crash;
- filter_instructor_frames keeps non-person frames untouched and
leaves their files on disk;
- threshold monotonicity invariant: a stricter threshold can never
keep MORE frames than a lenient one.

Files changed (2) hide show
  1. app/pipeline/filter.py +69 -19
  2. tests/test_filter.py +79 -0
app/pipeline/filter.py CHANGED
@@ -1,34 +1,86 @@
1
- """Filter out instructor-only (talking-head) frames using face detection."""
2
  from __future__ import annotations
3
 
4
  from pathlib import Path
 
5
 
6
  import cv2
7
 
8
  from .types import SceneFrame
9
 
10
- _CASCADE_PATH = Path(cv2.data.haarcascades) / "haarcascade_frontalface_default.xml"
11
 
 
 
 
 
 
 
12
 
13
- def _load_cascade() -> cv2.CascadeClassifier:
14
- cascade = cv2.CascadeClassifier(str(_CASCADE_PATH))
15
- if cascade.empty():
16
- raise RuntimeError(f"Failed to load Haar cascade at {_CASCADE_PATH}")
17
- return cascade
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18
 
19
 
20
  def is_instructor_frame(
21
  image_path: Path,
22
  face_area_threshold: float = 0.12,
23
- cascade: cv2.CascadeClassifier | None = None,
24
  ) -> bool:
25
  """Return True if the frame appears to be a talking-head shot.
26
 
27
- Heuristic: a frame is instructor-only if a detected face occupies more
28
- than `face_area_threshold` of the image area. Slides with a small
29
- inset webcam fall well below this threshold and are kept.
 
 
30
  """
31
- cascade = cascade or _load_cascade()
32
  img = cv2.imread(str(image_path))
33
  if img is None:
34
  return False
@@ -36,12 +88,10 @@ def is_instructor_frame(
36
  if h == 0 or w == 0:
37
  return False
38
  gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
39
- faces = cascade.detectMultiScale(gray, scaleFactor=1.2, minNeighbors=5, minSize=(60, 60))
40
- if len(faces) == 0:
41
  return False
42
- frame_area = float(h * w)
43
- largest = max((fw * fh) for (_, _, fw, fh) in faces)
44
- return (largest / frame_area) >= face_area_threshold
45
 
46
 
47
  def filter_instructor_frames(
@@ -49,11 +99,11 @@ def filter_instructor_frames(
49
  face_area_threshold: float = 0.12,
50
  ) -> tuple[list[SceneFrame], list[SceneFrame]]:
51
  """Split frames into (kept, dropped). Dropped images are deleted from disk."""
52
- cascade = _load_cascade()
53
  kept: list[SceneFrame] = []
54
  dropped: list[SceneFrame] = []
55
  for frame in frames:
56
- if is_instructor_frame(frame.image_path, face_area_threshold, cascade):
57
  dropped.append(frame)
58
  try:
59
  frame.image_path.unlink(missing_ok=True)
 
1
+ """Filter out instructor-only (talking-head) frames using face/upper-body detection."""
2
  from __future__ import annotations
3
 
4
  from pathlib import Path
5
+ from typing import NamedTuple
6
 
7
  import cv2
8
 
9
  from .types import SceneFrame
10
 
 
11
 
12
+ class _CascadeSpec(NamedTuple):
13
+ name: str
14
+ filename: str
15
+ min_neighbors: int
16
+ min_size: tuple[int, int]
17
+ flip: bool # also run on horizontally-flipped image (profile faces face one way)
18
 
19
+
20
+ # Multiple cascades catch person-frames the frontal-face detector misses:
21
+ # - profileface: instructor turned to the side (flipped also covers the other side)
22
+ # - upperbody: head+shoulders silhouette, robust when face is angled or partly off-frame
23
+ _CASCADE_SPECS: list[_CascadeSpec] = [
24
+ _CascadeSpec("frontal", "haarcascade_frontalface_default.xml", 5, (50, 50), flip=False),
25
+ _CascadeSpec("profile", "haarcascade_profileface.xml", 4, (50, 50), flip=True),
26
+ _CascadeSpec("upperbody", "haarcascade_upperbody.xml", 6, (90, 90), flip=False),
27
+ ]
28
+
29
+
30
+ def _load_cascades() -> list[tuple[_CascadeSpec, cv2.CascadeClassifier]]:
31
+ out: list[tuple[_CascadeSpec, cv2.CascadeClassifier]] = []
32
+ for spec in _CASCADE_SPECS:
33
+ path = Path(cv2.data.haarcascades) / spec.filename
34
+ cascade = cv2.CascadeClassifier(str(path))
35
+ if not cascade.empty():
36
+ out.append((spec, cascade))
37
+ if not out:
38
+ raise RuntimeError(f"No Haar cascades loaded under {cv2.data.haarcascades}")
39
+ return out
40
+
41
+
42
+ def _largest_region_area(
43
+ gray, cascades: list[tuple[_CascadeSpec, cv2.CascadeClassifier]]
44
+ ) -> int:
45
+ largest = 0
46
+ flipped = None
47
+ for spec, cascade in cascades:
48
+ regions = cascade.detectMultiScale(
49
+ gray,
50
+ scaleFactor=1.15,
51
+ minNeighbors=spec.min_neighbors,
52
+ minSize=spec.min_size,
53
+ )
54
+ for (_, _, fw, fh) in regions:
55
+ largest = max(largest, fw * fh)
56
+ if spec.flip:
57
+ if flipped is None:
58
+ flipped = cv2.flip(gray, 1)
59
+ regions = cascade.detectMultiScale(
60
+ flipped,
61
+ scaleFactor=1.15,
62
+ minNeighbors=spec.min_neighbors,
63
+ minSize=spec.min_size,
64
+ )
65
+ for (_, _, fw, fh) in regions:
66
+ largest = max(largest, fw * fh)
67
+ return largest
68
 
69
 
70
  def is_instructor_frame(
71
  image_path: Path,
72
  face_area_threshold: float = 0.12,
73
+ cascades: list[tuple[_CascadeSpec, cv2.CascadeClassifier]] | None = None,
74
  ) -> bool:
75
  """Return True if the frame appears to be a talking-head shot.
76
 
77
+ A frame is treated as instructor-only when the largest detected
78
+ face / upper-body region occupies more than `face_area_threshold` of
79
+ the image area. Multiple cascades run in sequence so an angled face
80
+ or hands-raised pose still registers; the largest region across all
81
+ of them is compared to the threshold.
82
  """
83
+ cascades = cascades or _load_cascades()
84
  img = cv2.imread(str(image_path))
85
  if img is None:
86
  return False
 
88
  if h == 0 or w == 0:
89
  return False
90
  gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
91
+ largest = _largest_region_area(gray, cascades)
92
+ if largest == 0:
93
  return False
94
+ return (largest / float(h * w)) >= face_area_threshold
 
 
95
 
96
 
97
  def filter_instructor_frames(
 
99
  face_area_threshold: float = 0.12,
100
  ) -> tuple[list[SceneFrame], list[SceneFrame]]:
101
  """Split frames into (kept, dropped). Dropped images are deleted from disk."""
102
+ cascades = _load_cascades()
103
  kept: list[SceneFrame] = []
104
  dropped: list[SceneFrame] = []
105
  for frame in frames:
106
+ if is_instructor_frame(frame.image_path, face_area_threshold, cascades):
107
  dropped.append(frame)
108
  try:
109
  frame.image_path.unlink(missing_ok=True)
tests/test_filter.py ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Tests for the instructor-frame filter."""
2
+ from pathlib import Path
3
+
4
+ import cv2
5
+ import numpy as np
6
+ import pytest
7
+ from PIL import Image
8
+
9
+ from app.pipeline.filter import (
10
+ _CASCADE_SPECS,
11
+ _load_cascades,
12
+ is_instructor_frame,
13
+ filter_instructor_frames,
14
+ )
15
+ from app.pipeline.types import SceneFrame
16
+
17
+
18
+ def _save(path: Path, color) -> None:
19
+ path.parent.mkdir(parents=True, exist_ok=True)
20
+ Image.new("RGB", (320, 180), color).save(path, "JPEG")
21
+
22
+
23
+ def test_load_cascades_returns_multiple_detectors():
24
+ cascades = _load_cascades()
25
+ names = {spec.name for spec, _ in cascades}
26
+ # Frontal face is the baseline; the other two are the additions that
27
+ # actually catch instructor frames in the wild.
28
+ assert "frontal" in names
29
+ assert names & {"profile", "upperbody"}, (
30
+ f"need profile or upperbody for angled-face / hands-up coverage; got {names}"
31
+ )
32
+
33
+
34
+ def test_is_instructor_frame_returns_false_on_uniform_image(tmp_path):
35
+ # A solid-color frame has no detectable face, no upper body.
36
+ p = tmp_path / "blank.jpg"
37
+ _save(p, (255, 255, 255))
38
+ assert is_instructor_frame(p, face_area_threshold=0.01) is False
39
+
40
+
41
+ def test_is_instructor_frame_handles_unreadable_path(tmp_path):
42
+ # Missing file: don't crash, treat as non-instructor.
43
+ assert is_instructor_frame(tmp_path / "does-not-exist.jpg") is False
44
+
45
+
46
+ def test_filter_keeps_blank_frames_and_unlinks_dropped(tmp_path):
47
+ # No real faces here; the filter should keep both frames and the
48
+ # filesystem should be untouched.
49
+ a = tmp_path / "a.jpg"
50
+ b = tmp_path / "b.jpg"
51
+ _save(a, (255, 255, 255))
52
+ _save(b, (10, 10, 10))
53
+ frames = [
54
+ SceneFrame(0, 0.0, 5.0, a),
55
+ SceneFrame(1, 5.0, 10.0, b),
56
+ ]
57
+ kept, dropped = filter_instructor_frames(frames, face_area_threshold=0.05)
58
+ assert len(kept) == 2
59
+ assert dropped == []
60
+ assert a.exists() and b.exists()
61
+
62
+
63
+ def test_threshold_changes_drop_decision(tmp_path):
64
+ # Build a frame with a synthetic large oval the upperbody cascade is
65
+ # at least a candidate to fire on — but this is only meant to verify
66
+ # the threshold knob still gates decisions, NOT the cascade quality.
67
+ # If detection happens to miss this synthetic shape on the test box,
68
+ # the test just confirms both calls return False (consistent), which
69
+ # is still a valid invariant: same input + lower threshold ≠ more
70
+ # frames kept.
71
+ p = tmp_path / "shape.jpg"
72
+ arr = np.full((360, 640, 3), 255, dtype=np.uint8)
73
+ cv2.ellipse(arr, (320, 220), (90, 130), 0, 0, 360, (60, 60, 60), -1)
74
+ cv2.imwrite(str(p), arr)
75
+ cascades = _load_cascades()
76
+ strict = is_instructor_frame(p, face_area_threshold=0.001, cascades=cascades)
77
+ lenient = is_instructor_frame(p, face_area_threshold=0.99, cascades=cascades)
78
+ # Lower threshold can never be MORE permissive than higher threshold.
79
+ assert not (lenient and not strict)