File size: 16,498 Bytes
fe8b9d3
 
e059eb2
fe8b9d3
 
 
fe75e0e
 
 
 
e059eb2
b3c6e2c
fe75e0e
fe8b9d3
 
 
e059eb2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
fe8b9d3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
fe75e0e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
06c75aa
 
fbb54cc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
06c75aa
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e7c2416
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
06c75aa
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b3c6e2c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
from pathlib import Path

import cv2
import numpy as np
from PIL import Image

from app.pipeline.scenes import (
    _cap_max_frames,
    _enforce_min_gap,
    _merge_duplicate_adjacent,
    _read_frame_at,
    _save_top_candidate_frames,
)
from app.pipeline.types import SceneFrame


class _FakeCapture:
    """Records the last seek property/value so we can assert which seek
    mode _read_frame_at chose without decoding a real video."""

    def __init__(self, frame_count=1000):
        self.frame_count = frame_count
        self.last_prop = None
        self.last_value = None

    def get(self, prop):
        if prop == cv2.CAP_PROP_FRAME_COUNT:
            return float(self.frame_count)
        return 0.0

    def set(self, prop, value):
        self.last_prop = prop
        self.last_value = value
        return True

    def read(self):
        return True, np.zeros((36, 64, 3), dtype=np.uint8)


def test_read_frame_at_seeks_by_frame_index_when_fps_known():
    """Deterministic path: with a valid FPS, the timestamp is converted to
    an integer frame index and seeked via CAP_PROP_POS_FRAMES. This is what
    makes representative-frame extraction reproducible run-to-run (and, in
    turn, stabilises the merge step's scene count)."""
    cap = _FakeCapture()
    ok, _ = _read_frame_at(cap, time_seconds=2.0, fps=30.0)
    assert ok
    assert cap.last_prop == cv2.CAP_PROP_POS_FRAMES
    assert cap.last_value == 60.0  # 2.0s * 30fps


def test_read_frame_at_clamps_to_last_frame():
    """A timestamp past the end clamps to the final frame index rather than
    seeking out of range."""
    cap = _FakeCapture(frame_count=100)
    _read_frame_at(cap, time_seconds=999.0, fps=30.0)
    assert cap.last_prop == cv2.CAP_PROP_POS_FRAMES
    assert cap.last_value == 99.0  # frame_count - 1


def test_read_frame_at_falls_back_to_msec_without_fps():
    """A broken/zero FPS header falls back to millisecond seeking so the
    frame still extracts (just non-deterministically) instead of hard-failing."""
    cap = _FakeCapture()
    _read_frame_at(cap, time_seconds=2.0, fps=0.0)
    assert cap.last_prop == cv2.CAP_PROP_POS_MSEC
    assert cap.last_value == 2000.0  # 2.0s * 1000


def _save(path: Path, color) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    Image.new("RGB", (64, 36), color).save(path, "JPEG")


def _save_text_like(path: Path, seed: int) -> None:
    rng = np.random.default_rng(seed)
    arr = rng.integers(0, 256, size=(36, 64, 3), dtype=np.uint8)
    path.parent.mkdir(parents=True, exist_ok=True)
    Image.fromarray(arr).save(path, "JPEG")


def test_merge_duplicate_adjacent_collapses_identical_neighbors(tmp_path):
    a = tmp_path / "a.jpg"
    b = tmp_path / "b.jpg"
    c = tmp_path / "c.jpg"
    _save(a, (255, 255, 255))
    _save(b, (255, 255, 255))
    _save_text_like(c, seed=1)

    frames = [
        SceneFrame(0, 0.0, 2.0, a),
        SceneFrame(1, 2.0, 10.0, b),
        SceneFrame(2, 10.0, 20.0, c),
    ]
    merged = _merge_duplicate_adjacent(frames, max_mean_abs_diff=5.0)

    assert len(merged) == 2
    assert merged[0].start_seconds == 0.0
    assert merged[0].end_seconds == 10.0
    assert merged[0].index == 0
    assert merged[1].index == 1
    assert not b.exists()
    assert a.exists() and c.exists()


def test_merge_duplicate_adjacent_preserves_distinct_frames(tmp_path):
    a = tmp_path / "a.jpg"
    b = tmp_path / "b.jpg"
    _save_text_like(a, seed=1)
    _save_text_like(b, seed=2)

    frames = [
        SceneFrame(0, 0.0, 5.0, a),
        SceneFrame(1, 5.0, 10.0, b),
    ]
    merged = _merge_duplicate_adjacent(frames, max_mean_abs_diff=5.0)

    assert len(merged) == 2
    assert a.exists() and b.exists()


def test_enforce_min_gap_drops_close_frames(tmp_path):
    paths = [tmp_path / f"f{i}.jpg" for i in range(4)]
    for i, p in enumerate(paths):
        _save_text_like(p, seed=i + 1)
    frames = [
        SceneFrame(0, 0.0, 1.0, paths[0]),
        SceneFrame(1, 1.5, 2.5, paths[1]),  # gap from prev = 1.5s, < 2s threshold
        SceneFrame(2, 4.0, 5.0, paths[2]),  # gap = 2.5s, kept
        SceneFrame(3, 4.5, 5.5, paths[3]),  # gap = 0.5s, dropped
    ]
    out = _enforce_min_gap(frames, min_gap_seconds=2.0)
    kept_paths = {f.image_path.name for f in out}
    assert len(out) == 2
    assert kept_paths == {"f0.jpg", "f2.jpg"}
    assert not paths[1].exists() and not paths[3].exists()
    assert paths[0].exists() and paths[2].exists()


def test_cap_max_frames_uniform_stride_preserves_first_and_last(tmp_path):
    paths = [tmp_path / f"f{i:02d}.jpg" for i in range(7)]
    for i, p in enumerate(paths):
        _save_text_like(p, seed=i + 1)
    frames = [SceneFrame(i, float(i), float(i) + 1, p) for i, p in enumerate(paths)]
    out = _cap_max_frames(frames, max_frames=3)
    kept = [f.image_path.name for f in out]
    assert len(kept) == 3
    assert kept[0] == "f00.jpg"
    assert kept[-1] == "f06.jpg"
    # The middle frame should be from the interior; not first or last.
    assert kept[1] not in ("f00.jpg", "f06.jpg")
    # Removed frames' files should be unlinked.
    surviving = set(kept)
    for p in paths:
        assert p.exists() == (p.name in surviving)


def test_cap_max_frames_no_op_when_under_limit(tmp_path):
    paths = [tmp_path / f"f{i}.jpg" for i in range(3)]
    for i, p in enumerate(paths):
        _save_text_like(p, seed=i + 1)
    frames = [SceneFrame(i, float(i), float(i) + 1, p) for i, p in enumerate(paths)]
    out = _cap_max_frames(frames, max_frames=10)
    assert len(out) == 3
    for p in paths:
        assert p.exists()


def test_cap_max_frames_one_returns_exactly_one(tmp_path):
    """max_frames=1 must return a single frame. The old {0, n-1} keep-set
    returned two (first + last), overshooting the requested cap."""
    paths = [tmp_path / f"f{i}.jpg" for i in range(4)]
    for i, p in enumerate(paths):
        _save_text_like(p, seed=i + 1)
    frames = [SceneFrame(i, float(i), float(i) + 1, p) for i, p in enumerate(paths)]
    out = _cap_max_frames(frames, max_frames=1)
    assert len(out) == 1
    assert out[0].index == 0
    # The end of the single kept frame extends to cover the whole timeline.
    assert out[0].end_seconds == 4.0
    # The other three frames' files are unlinked.
    assert paths[0].exists()
    for p in paths[1:]:
        assert not p.exists()


# -----------------------------------------------------------------------------
# Timing invariants. align_segments relies on these after frames are dropped:
#   * start_seconds strictly increases across kept frames,
#   * indices are contiguous and start at 0,
#   * each kept frame's end >= start,
#   * the last kept frame's end_seconds covers the source's true end so the
#     transcript-alignment code doesn't lose narration past the dropped tail.
# These invariants were previously only implicit. The tests below pin them.
# -----------------------------------------------------------------------------


def _assert_timing_invariants(out, total_end):
    assert [f.index for f in out] == list(range(len(out))), "indices not contiguous from 0"
    starts = [f.start_seconds for f in out]
    assert starts == sorted(starts), "start_seconds must be monotonically non-decreasing"
    assert len(starts) == len(set(starts)), "start_seconds must be strictly increasing"
    for f in out:
        assert f.end_seconds >= f.start_seconds, f"end < start at index {f.index}"
    assert out[-1].end_seconds >= total_end, (
        "last kept frame must cover the original total end so narration after "
        "dropped frames isn't lost during transcript alignment"
    )


def test_enforce_min_gap_preserves_timing_invariants(tmp_path):
    paths = [tmp_path / f"f{i}.jpg" for i in range(5)]
    for i, p in enumerate(paths):
        _save_text_like(p, seed=i + 1)
    # Five frames with the original tail at end_seconds=12.0.
    # Frames 1 and 3 are within 2s of the previous kept frame; both drop.
    frames = [
        SceneFrame(0, 0.0, 1.0, paths[0]),
        SceneFrame(1, 1.5, 2.0, paths[1]),    # dropped (gap=1.5)
        SceneFrame(2, 4.0, 6.0, paths[2]),    # kept
        SceneFrame(3, 6.5, 7.0, paths[3]),    # dropped (gap=2.5 from 4.0... wait, 2.5>=2 so kept actually)
        SceneFrame(4, 9.0, 12.0, paths[4]),   # kept
    ]
    out = _enforce_min_gap(frames, min_gap_seconds=2.0)
    _assert_timing_invariants(out, total_end=12.0)


def test_enforce_min_gap_extends_last_end_when_tail_drops(tmp_path):
    paths = [tmp_path / f"f{i}.jpg" for i in range(3)]
    for i, p in enumerate(paths):
        _save_text_like(p, seed=i + 1)
    # The final frame is within the gap and gets dropped β€” the kept second
    # frame must have its end_seconds extended so we don't lose the tail.
    frames = [
        SceneFrame(0, 0.0, 1.0, paths[0]),
        SceneFrame(1, 5.0, 8.0, paths[1]),
        SceneFrame(2, 8.5, 20.0, paths[2]),  # dropped, gap=3.5
    ]
    out = _enforce_min_gap(frames, min_gap_seconds=4.0)
    assert len(out) == 2
    assert out[-1].end_seconds == 20.0, "must absorb the tail of the dropped trailing frame"
    _assert_timing_invariants(out, total_end=20.0)


def test_cap_max_frames_preserves_timing_invariants(tmp_path):
    paths = [tmp_path / f"f{i:02d}.jpg" for i in range(8)]
    for i, p in enumerate(paths):
        _save_text_like(p, seed=i + 1)
    frames = [SceneFrame(i, float(i) * 2, float(i) * 2 + 2, p) for i, p in enumerate(paths)]
    out = _cap_max_frames(frames, max_frames=4)
    _assert_timing_invariants(out, total_end=frames[-1].end_seconds)


def test_min_gap_then_cap_max_frames_invariants(tmp_path):
    paths = [tmp_path / f"f{i:02d}.jpg" for i in range(10)]
    for i, p in enumerate(paths):
        _save_text_like(p, seed=i + 1)
    # Mix close + spread frames to exercise both helpers in sequence.
    starts = [0.0, 0.6, 3.0, 6.0, 6.4, 9.0, 12.0, 14.0, 14.3, 17.0]
    frames = [SceneFrame(i, s, s + 1.0, paths[i]) for i, s in enumerate(starts)]
    after_gap = _enforce_min_gap(frames, min_gap_seconds=1.5)
    out = _cap_max_frames(after_gap, max_frames=3)
    _assert_timing_invariants(out, total_end=18.0)


def test_min_gap_extends_intermediate_ends_to_cover_dropped_intervals(tmp_path):
    # Regression: when min-gap drops scenes between two kept frames, the
    # earlier kept frame's end_seconds was left at its original value (e.g.
    # 0:05) while the next kept frame's start was much later (0:30). The
    # transcript-alignment step uses kept starts as boundaries, so narration
    # 0:05 β†’ 0:30 was attached to the first kept frame even though its
    # displayed range read "0:00 β€” 0:05". Each kept frame's end must now
    # extend to the next kept frame's start.
    paths = [tmp_path / f"f{i}.jpg" for i in range(5)]
    for i, p in enumerate(paths):
        _save_text_like(p, seed=i + 1)
    frames = [
        SceneFrame(0, 0.0, 5.0, paths[0]),
        SceneFrame(1, 6.0, 7.0, paths[1]),    # gap=6 < 15 β†’ dropped
        SceneFrame(2, 8.5, 9.0, paths[2]),    # gap=8.5 < 15 β†’ dropped
        SceneFrame(3, 30.0, 35.0, paths[3]),  # gap=30 β‰₯ 15 β†’ kept
        SceneFrame(4, 40.0, 60.0, paths[4]),  # gap=10 < 15 β†’ dropped
    ]
    out = _enforce_min_gap(frames, min_gap_seconds=15.0)
    assert len(out) == 2
    # First kept frame's end now reaches the next kept frame's start.
    assert out[0].end_seconds == 30.0
    # Last kept frame absorbs the original tail.
    assert out[1].end_seconds == 60.0


def test_cap_max_frames_extends_intermediate_ends_to_next_kept_start(tmp_path):
    paths = [tmp_path / f"f{i:02d}.jpg" for i in range(10)]
    for i, p in enumerate(paths):
        _save_text_like(p, seed=i + 1)
    # 10 contiguous 1-second frames; cap to 3.
    frames = [SceneFrame(i, float(i), float(i) + 1, paths[i]) for i in range(10)]
    out = _cap_max_frames(frames, max_frames=3)
    assert len(out) == 3
    # Each kept frame's end must reach the next kept frame's start so the
    # rendered range reflects the narration window the transcript actually
    # attached to it.
    for i in range(len(out) - 1):
        assert out[i].end_seconds == out[i + 1].start_seconds, (
            f"frame {i} end {out[i].end_seconds} != next start {out[i+1].start_seconds}"
        )
    assert out[-1].end_seconds == frames[-1].end_seconds


def test_cap_max_frames_unlinks_alts_of_dropped(tmp_path):
    # Regression guard: when capping, alternate frame files for dropped
    # primaries must also be cleaned up β€” otherwise we leak disk on long
    # lectures. The helper routes through _unlink_frame_files.
    primaries = [tmp_path / f"p{i}.jpg" for i in range(5)]
    alts = [tmp_path / f"p{i}_alt1.jpg" for i in range(5)]
    for p in primaries + alts:
        _save_text_like(p, seed=hash(p.name) & 0xFF)
    frames = [
        SceneFrame(i, float(i), float(i) + 1, primaries[i], alt_image_paths=[alts[i]])
        for i in range(5)
    ]
    out = _cap_max_frames(frames, max_frames=2)
    kept_alts = {p.name for f in out for p in f.alt_image_paths}
    for f in frames:
        for ap in f.alt_image_paths:
            if ap.name in kept_alts:
                assert ap.exists()
            else:
                assert not ap.exists(), f"orphaned alt: {ap.name}"


def _write_brightness_video(path: Path, duration_s: float, fps: int, brightness_at):
    """Write a tiny video where each frame is a uniform gray of the value
    returned by `brightness_at(t_seconds)` (0..255). Used to control which
    sampling timestamps inside _save_top_candidate_frames map to which
    brightness so we can verify candidate ordering deterministically.
    """
    import cv2 as _cv2
    fourcc = _cv2.VideoWriter_fourcc(*"mp4v")
    w, h = 64, 36
    writer = _cv2.VideoWriter(str(path), fourcc, fps, (w, h))
    try:
        n = int(duration_s * fps)
        for i in range(n):
            t = i / fps
            v = int(max(0, min(255, brightness_at(t))))
            frame = np.full((h, w, 3), v, dtype=np.uint8)
            writer.write(frame)
    finally:
        writer.release()


def test_save_top_candidate_frames_promotes_latest_among_brightness_survivors(tmp_path):
    """The reviewer almost always wants the fully-populated final state of
    a progressive-reveal slide. The candidate picker filters dark
    transitions by brightness rank, but among the survivors the primary
    must be the latest-in-time, not the brightest. Without that, a
    mid-scene peak (e.g. a brief animation flash) becomes the default.
    """
    video_path = tmp_path / "scene.mp4"
    # Sampling timestamps for start=0,end=10 are 5.0, 6.125, 7.25, 8.375, 9.5.
    # Brightness profile:
    #   t=5.0   -> 60   (transition; drops out of top 3)
    #   t=6.125 -> 220  (brightest peak β€” would be the old default primary)
    #   t=7.25  -> 200
    #   t=8.375 -> 195
    #   t=9.5   -> 190
    # Top 3 by brightness: 220, 200, 195 -> timestamps 6.125, 7.25, 8.375.
    # Time-sort descending: 8.375, 7.25, 6.125 -> primary = t=8.375.
    def brightness_at(t):
        if t < 5.5:
            return 60
        if t < 6.5:
            return 220
        if t < 7.75:
            return 200
        if t < 9.0:
            return 195
        return 190

    _write_brightness_video(video_path, duration_s=10.0, fps=30, brightness_at=brightness_at)

    out_dir = tmp_path / "frames"
    paths = _save_top_candidate_frames(
        video_path, start_s=0.0, end_s=10.0, out_dir=out_dir, scene_index=0
    )
    assert len(paths) == 3, paths

    import cv2 as _cv2

    def mean_brightness(p):
        img = _cv2.imread(str(p))
        return float(_cv2.cvtColor(img, _cv2.COLOR_BGR2GRAY).mean())

    primary_b = mean_brightness(paths[0])
    # Primary must be the latest-time survivor (~195), NOT the brightness
    # peak (~220) that the old logic would have promoted.
    assert primary_b < 210, (
        f"primary should be the latest-in-time survivor (~195), not the "
        f"mid-scene brightness peak (~220); got {primary_b:.1f}"
    )
    assert primary_b > 180, (
        f"primary should still be a brightness-survivor (>180), not the "
        f"dark transition frame (~60); got {primary_b:.1f}"
    )
    # The peak should be retained as one of the alternates so the reviewer
    # can still pick it if they prefer that frame.
    alt_brights = sorted(mean_brightness(p) for p in paths[1:])
    assert alt_brights[-1] >= 215, (
        f"brightness peak should be kept as an alternate; got {alt_brights}"
    )