Spaces:
Sleeping
Filter: detect instructor frames the frontal-face cascade misses
Browse filesFrontal-face Haar misses faces at angles, with hands raised, partly
turned, or partly off-frame — so the threshold check never even fired
for those frames and they slipped through regardless of how strict
the slider was set. Now we run three cascades in sequence and take
the largest detected region across all of them:
- frontal face (existing — head-on shots)
- profile face (instructor turned to one side; runs on both the
image and its horizontal flip so either side is
covered without doubling cascade count)
- upper body (head + shoulders silhouette — robust when the
face cascade fails because the angle, lighting,
or partial occlusion blocks face detection but
the silhouette is still obvious)
The three cascades are parameterised individually (different
min_neighbors / min_size) since upper-body benefits from stricter
clustering and a larger min size to avoid false positives on
diagrams. Existing threshold semantics unchanged: lower → drop more.
Five new tests in tests/test_filter.py:
- cascades load and include the new profile/upperbody detectors;
- blank uniform frames don't trigger (no false positives);
- missing file path doesn't crash;
- filter_instructor_frames keeps non-person frames untouched and
leaves their files on disk;
- threshold monotonicity invariant: a stricter threshold can never
keep MORE frames than a lenient one.
- app/pipeline/filter.py +69 -19
- tests/test_filter.py +79 -0
|
@@ -1,34 +1,86 @@
|
|
| 1 |
-
"""Filter out instructor-only (talking-head) frames using face detection."""
|
| 2 |
from __future__ import annotations
|
| 3 |
|
| 4 |
from pathlib import Path
|
|
|
|
| 5 |
|
| 6 |
import cv2
|
| 7 |
|
| 8 |
from .types import SceneFrame
|
| 9 |
|
| 10 |
-
_CASCADE_PATH = Path(cv2.data.haarcascades) / "haarcascade_frontalface_default.xml"
|
| 11 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
|
| 19 |
|
| 20 |
def is_instructor_frame(
|
| 21 |
image_path: Path,
|
| 22 |
face_area_threshold: float = 0.12,
|
| 23 |
-
|
| 24 |
) -> bool:
|
| 25 |
"""Return True if the frame appears to be a talking-head shot.
|
| 26 |
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
|
|
|
|
|
|
| 30 |
"""
|
| 31 |
-
|
| 32 |
img = cv2.imread(str(image_path))
|
| 33 |
if img is None:
|
| 34 |
return False
|
|
@@ -36,12 +88,10 @@ def is_instructor_frame(
|
|
| 36 |
if h == 0 or w == 0:
|
| 37 |
return False
|
| 38 |
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
|
| 39 |
-
|
| 40 |
-
if
|
| 41 |
return False
|
| 42 |
-
|
| 43 |
-
largest = max((fw * fh) for (_, _, fw, fh) in faces)
|
| 44 |
-
return (largest / frame_area) >= face_area_threshold
|
| 45 |
|
| 46 |
|
| 47 |
def filter_instructor_frames(
|
|
@@ -49,11 +99,11 @@ def filter_instructor_frames(
|
|
| 49 |
face_area_threshold: float = 0.12,
|
| 50 |
) -> tuple[list[SceneFrame], list[SceneFrame]]:
|
| 51 |
"""Split frames into (kept, dropped). Dropped images are deleted from disk."""
|
| 52 |
-
|
| 53 |
kept: list[SceneFrame] = []
|
| 54 |
dropped: list[SceneFrame] = []
|
| 55 |
for frame in frames:
|
| 56 |
-
if is_instructor_frame(frame.image_path, face_area_threshold,
|
| 57 |
dropped.append(frame)
|
| 58 |
try:
|
| 59 |
frame.image_path.unlink(missing_ok=True)
|
|
|
|
| 1 |
+
"""Filter out instructor-only (talking-head) frames using face/upper-body detection."""
|
| 2 |
from __future__ import annotations
|
| 3 |
|
| 4 |
from pathlib import Path
|
| 5 |
+
from typing import NamedTuple
|
| 6 |
|
| 7 |
import cv2
|
| 8 |
|
| 9 |
from .types import SceneFrame
|
| 10 |
|
|
|
|
| 11 |
|
| 12 |
+
class _CascadeSpec(NamedTuple):
|
| 13 |
+
name: str
|
| 14 |
+
filename: str
|
| 15 |
+
min_neighbors: int
|
| 16 |
+
min_size: tuple[int, int]
|
| 17 |
+
flip: bool # also run on horizontally-flipped image (profile faces face one way)
|
| 18 |
|
| 19 |
+
|
| 20 |
+
# Multiple cascades catch person-frames the frontal-face detector misses:
|
| 21 |
+
# - profileface: instructor turned to the side (flipped also covers the other side)
|
| 22 |
+
# - upperbody: head+shoulders silhouette, robust when face is angled or partly off-frame
|
| 23 |
+
_CASCADE_SPECS: list[_CascadeSpec] = [
|
| 24 |
+
_CascadeSpec("frontal", "haarcascade_frontalface_default.xml", 5, (50, 50), flip=False),
|
| 25 |
+
_CascadeSpec("profile", "haarcascade_profileface.xml", 4, (50, 50), flip=True),
|
| 26 |
+
_CascadeSpec("upperbody", "haarcascade_upperbody.xml", 6, (90, 90), flip=False),
|
| 27 |
+
]
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _load_cascades() -> list[tuple[_CascadeSpec, cv2.CascadeClassifier]]:
|
| 31 |
+
out: list[tuple[_CascadeSpec, cv2.CascadeClassifier]] = []
|
| 32 |
+
for spec in _CASCADE_SPECS:
|
| 33 |
+
path = Path(cv2.data.haarcascades) / spec.filename
|
| 34 |
+
cascade = cv2.CascadeClassifier(str(path))
|
| 35 |
+
if not cascade.empty():
|
| 36 |
+
out.append((spec, cascade))
|
| 37 |
+
if not out:
|
| 38 |
+
raise RuntimeError(f"No Haar cascades loaded under {cv2.data.haarcascades}")
|
| 39 |
+
return out
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def _largest_region_area(
|
| 43 |
+
gray, cascades: list[tuple[_CascadeSpec, cv2.CascadeClassifier]]
|
| 44 |
+
) -> int:
|
| 45 |
+
largest = 0
|
| 46 |
+
flipped = None
|
| 47 |
+
for spec, cascade in cascades:
|
| 48 |
+
regions = cascade.detectMultiScale(
|
| 49 |
+
gray,
|
| 50 |
+
scaleFactor=1.15,
|
| 51 |
+
minNeighbors=spec.min_neighbors,
|
| 52 |
+
minSize=spec.min_size,
|
| 53 |
+
)
|
| 54 |
+
for (_, _, fw, fh) in regions:
|
| 55 |
+
largest = max(largest, fw * fh)
|
| 56 |
+
if spec.flip:
|
| 57 |
+
if flipped is None:
|
| 58 |
+
flipped = cv2.flip(gray, 1)
|
| 59 |
+
regions = cascade.detectMultiScale(
|
| 60 |
+
flipped,
|
| 61 |
+
scaleFactor=1.15,
|
| 62 |
+
minNeighbors=spec.min_neighbors,
|
| 63 |
+
minSize=spec.min_size,
|
| 64 |
+
)
|
| 65 |
+
for (_, _, fw, fh) in regions:
|
| 66 |
+
largest = max(largest, fw * fh)
|
| 67 |
+
return largest
|
| 68 |
|
| 69 |
|
| 70 |
def is_instructor_frame(
|
| 71 |
image_path: Path,
|
| 72 |
face_area_threshold: float = 0.12,
|
| 73 |
+
cascades: list[tuple[_CascadeSpec, cv2.CascadeClassifier]] | None = None,
|
| 74 |
) -> bool:
|
| 75 |
"""Return True if the frame appears to be a talking-head shot.
|
| 76 |
|
| 77 |
+
A frame is treated as instructor-only when the largest detected
|
| 78 |
+
face / upper-body region occupies more than `face_area_threshold` of
|
| 79 |
+
the image area. Multiple cascades run in sequence so an angled face
|
| 80 |
+
or hands-raised pose still registers; the largest region across all
|
| 81 |
+
of them is compared to the threshold.
|
| 82 |
"""
|
| 83 |
+
cascades = cascades or _load_cascades()
|
| 84 |
img = cv2.imread(str(image_path))
|
| 85 |
if img is None:
|
| 86 |
return False
|
|
|
|
| 88 |
if h == 0 or w == 0:
|
| 89 |
return False
|
| 90 |
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
|
| 91 |
+
largest = _largest_region_area(gray, cascades)
|
| 92 |
+
if largest == 0:
|
| 93 |
return False
|
| 94 |
+
return (largest / float(h * w)) >= face_area_threshold
|
|
|
|
|
|
|
| 95 |
|
| 96 |
|
| 97 |
def filter_instructor_frames(
|
|
|
|
| 99 |
face_area_threshold: float = 0.12,
|
| 100 |
) -> tuple[list[SceneFrame], list[SceneFrame]]:
|
| 101 |
"""Split frames into (kept, dropped). Dropped images are deleted from disk."""
|
| 102 |
+
cascades = _load_cascades()
|
| 103 |
kept: list[SceneFrame] = []
|
| 104 |
dropped: list[SceneFrame] = []
|
| 105 |
for frame in frames:
|
| 106 |
+
if is_instructor_frame(frame.image_path, face_area_threshold, cascades):
|
| 107 |
dropped.append(frame)
|
| 108 |
try:
|
| 109 |
frame.image_path.unlink(missing_ok=True)
|
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests for the instructor-frame filter."""
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
|
| 4 |
+
import cv2
|
| 5 |
+
import numpy as np
|
| 6 |
+
import pytest
|
| 7 |
+
from PIL import Image
|
| 8 |
+
|
| 9 |
+
from app.pipeline.filter import (
|
| 10 |
+
_CASCADE_SPECS,
|
| 11 |
+
_load_cascades,
|
| 12 |
+
is_instructor_frame,
|
| 13 |
+
filter_instructor_frames,
|
| 14 |
+
)
|
| 15 |
+
from app.pipeline.types import SceneFrame
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def _save(path: Path, color) -> None:
|
| 19 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 20 |
+
Image.new("RGB", (320, 180), color).save(path, "JPEG")
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def test_load_cascades_returns_multiple_detectors():
|
| 24 |
+
cascades = _load_cascades()
|
| 25 |
+
names = {spec.name for spec, _ in cascades}
|
| 26 |
+
# Frontal face is the baseline; the other two are the additions that
|
| 27 |
+
# actually catch instructor frames in the wild.
|
| 28 |
+
assert "frontal" in names
|
| 29 |
+
assert names & {"profile", "upperbody"}, (
|
| 30 |
+
f"need profile or upperbody for angled-face / hands-up coverage; got {names}"
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def test_is_instructor_frame_returns_false_on_uniform_image(tmp_path):
|
| 35 |
+
# A solid-color frame has no detectable face, no upper body.
|
| 36 |
+
p = tmp_path / "blank.jpg"
|
| 37 |
+
_save(p, (255, 255, 255))
|
| 38 |
+
assert is_instructor_frame(p, face_area_threshold=0.01) is False
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def test_is_instructor_frame_handles_unreadable_path(tmp_path):
|
| 42 |
+
# Missing file: don't crash, treat as non-instructor.
|
| 43 |
+
assert is_instructor_frame(tmp_path / "does-not-exist.jpg") is False
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def test_filter_keeps_blank_frames_and_unlinks_dropped(tmp_path):
|
| 47 |
+
# No real faces here; the filter should keep both frames and the
|
| 48 |
+
# filesystem should be untouched.
|
| 49 |
+
a = tmp_path / "a.jpg"
|
| 50 |
+
b = tmp_path / "b.jpg"
|
| 51 |
+
_save(a, (255, 255, 255))
|
| 52 |
+
_save(b, (10, 10, 10))
|
| 53 |
+
frames = [
|
| 54 |
+
SceneFrame(0, 0.0, 5.0, a),
|
| 55 |
+
SceneFrame(1, 5.0, 10.0, b),
|
| 56 |
+
]
|
| 57 |
+
kept, dropped = filter_instructor_frames(frames, face_area_threshold=0.05)
|
| 58 |
+
assert len(kept) == 2
|
| 59 |
+
assert dropped == []
|
| 60 |
+
assert a.exists() and b.exists()
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def test_threshold_changes_drop_decision(tmp_path):
|
| 64 |
+
# Build a frame with a synthetic large oval the upperbody cascade is
|
| 65 |
+
# at least a candidate to fire on — but this is only meant to verify
|
| 66 |
+
# the threshold knob still gates decisions, NOT the cascade quality.
|
| 67 |
+
# If detection happens to miss this synthetic shape on the test box,
|
| 68 |
+
# the test just confirms both calls return False (consistent), which
|
| 69 |
+
# is still a valid invariant: same input + lower threshold ≠ more
|
| 70 |
+
# frames kept.
|
| 71 |
+
p = tmp_path / "shape.jpg"
|
| 72 |
+
arr = np.full((360, 640, 3), 255, dtype=np.uint8)
|
| 73 |
+
cv2.ellipse(arr, (320, 220), (90, 130), 0, 0, 360, (60, 60, 60), -1)
|
| 74 |
+
cv2.imwrite(str(p), arr)
|
| 75 |
+
cascades = _load_cascades()
|
| 76 |
+
strict = is_instructor_frame(p, face_area_threshold=0.001, cascades=cascades)
|
| 77 |
+
lenient = is_instructor_frame(p, face_area_threshold=0.99, cascades=cascades)
|
| 78 |
+
# Lower threshold can never be MORE permissive than higher threshold.
|
| 79 |
+
assert not (lenient and not strict)
|