joemartis commited on
Commit
9eb7bf7
·
2 Parent(s): 419357c5d9159c

Merge pull request #60 from JoeMartis/claude/fix-zip-path-traversal-9kA4Q

Browse files
app/pipeline/orchestrator.py CHANGED
@@ -89,25 +89,36 @@ class PipelineError(RuntimeError):
89
  def _apply_topic_segmentation(
90
  kept: list[SceneFrame],
91
  topics: list[Topic],
92
- ) -> tuple[list[SceneFrame], list[str], list[str]]:
 
 
93
  """Collapse the per-slide kept-frame list into one entry per topic.
94
 
95
  Each scene is assigned to the topic whose [start, end) window
96
  contains its start_seconds; scenes that fall outside every range
97
  (rare; usually only the head/tail when the LLM trimmed too tight)
98
  are bucketed into the closest topic by edge distance. Within each
99
- topic, the longest-on-screen scene becomes the primary; that
100
- primary's existing within-scene candidate alts are kept, and the
101
- other scenes' primary frames are appended as additional alternates.
102
- Topics with no scenes are dropped — a topic-segment with no image
103
- has no place in the editor.
104
-
105
- Returns (new_kept, titles, summaries) where the three lists are
106
- parallel and the titles/summaries carry the LLM's labels for
107
- downstream use as section_title / segment summary.
 
 
 
 
 
 
 
 
108
  """
109
  if not topics or not kept:
110
- return kept, [""] * len(kept), [""] * len(kept)
 
111
  scenes_by_topic: list[list[SceneFrame]] = [[] for _ in topics]
112
  for sf in kept:
113
  placed = False
@@ -126,22 +137,46 @@ def _apply_topic_segmentation(
126
  )
127
  scenes_by_topic[best_i].append(sf)
128
 
 
 
 
 
 
 
 
 
 
129
  new_kept: list[SceneFrame] = []
130
  titles: list[str] = []
131
  summaries: list[str] = []
 
132
  for topic, scenes in zip(topics, scenes_by_topic):
133
  if not scenes:
134
  log.info("Dropping topic '%s' — no scenes fell inside its range.", topic.title)
135
  continue
136
  scenes.sort(key=lambda s: s.start_seconds)
137
- # Prefer scenes whose primary image is clearly slide content
138
- # (no face detected at all, or a tiny one) over scenes that
139
- # squeaked past the instructor filter with a borderline face
140
- # area. Only when every scene in the topic is borderline do we
141
- # fall back to picking from that pool — better some shot than
142
- # none. Within each pool, longest-on-screen wins.
143
- slide_scenes = [s for s in scenes if s.face_area_ratio < _SLIDE_CONTENT_FACE_RATIO]
144
- pool = slide_scenes or scenes
 
 
 
 
 
 
 
 
 
 
 
 
 
 
145
  primary = max(pool, key=lambda s: s.end_seconds - s.start_seconds)
146
  alt_paths: list[Path] = list(primary.alt_image_paths)
147
  for sf in scenes:
@@ -159,7 +194,8 @@ def _apply_topic_segmentation(
159
  )
160
  titles.append(topic.title)
161
  summaries.append(topic.summary)
162
- return new_kept, titles, summaries
 
163
 
164
 
165
  # A scene whose primary frame has more than this fraction of face is
@@ -169,6 +205,12 @@ def _apply_topic_segmentation(
169
  # instructor close-ups that snuck past.
170
  _SLIDE_CONTENT_FACE_RATIO = 0.03
171
 
 
 
 
 
 
 
172
 
173
  def _load_cues(inputs: PipelineInputs) -> tuple[list[Cue], bool]:
174
  """Return (cues, used_auto_transcribe)."""
@@ -246,17 +288,40 @@ def run_pipeline(
246
  if kept[-1].end_seconds < full_duration:
247
  kept[-1].end_seconds = full_duration
248
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
249
  # Optional: replace scene-based one-segment-per-slide with LLM topic
250
  # segmentation. A topic groups all the scenes it covers — the
251
- # longest-on-screen one becomes the primary frame, the rest become
252
- # switchable alternates in the editor. Surfaces failures rather than
253
- # silently falling back, per the toggle's contract.
 
254
  topic_titles: list[str] = []
255
  topic_summaries: list[str] = []
256
  if inputs.llm_toggles.topic_segmentation:
257
  with_images = inputs.llm_toggles.topic_segmentation_with_images
258
  _emit(
259
- 62,
260
  "topics",
261
  "Segmenting transcript into topics"
262
  + (" (with slide images)..." if with_images else "..."),
@@ -275,7 +340,9 @@ def run_pipeline(
275
  _emit(100, "error", str(exc))
276
  raise PipelineError(f"Topic segmentation failed: {exc}") from exc
277
  log.info("LLM produced %d topics", len(topics))
278
- kept, topic_titles, topic_summaries = _apply_topic_segmentation(kept, topics)
 
 
279
  if not kept:
280
  _emit(100, "error", "Topic segmentation left no usable segments.")
281
  raise PipelineError(
@@ -284,22 +351,6 @@ def run_pipeline(
284
 
285
  boundaries = [f.start_seconds for f in kept]
286
  narrations = align_segments(cues, boundaries, full_duration)
287
-
288
- if inputs.skip_ocr:
289
- log.info("Skipping OCR (--skip-ocr)")
290
- _emit(95, "ocr", "Skipping OCR.")
291
- ocr_texts = ["" for _ in kept]
292
- else:
293
- n = len(kept)
294
- log.info("Running OCR on %d frames", n)
295
- _emit(65, "ocr", f"Running OCR on {n} frames...")
296
- ocr_texts: list[str] = []
297
- for i, f in enumerate(kept):
298
- ocr_texts.append(
299
- ocr_frame(f.image_path, skip_inverted=inputs.skip_inverted_ocr)
300
- )
301
- pct = 65 + ((i + 1) / max(n, 1)) * 30
302
- _emit(pct, "ocr", f"OCR frame {i + 1}/{n}")
303
  # Alt-frame OCR is computed lazily by the /ocr endpoint when the
304
  # reviewer actually picks one — running it for every candidate up
305
  # front tripled OCR time on the HF free CPU tier.
 
89
  def _apply_topic_segmentation(
90
  kept: list[SceneFrame],
91
  topics: list[Topic],
92
+ *,
93
+ scene_ocr_texts: list[str] | None = None,
94
+ ) -> tuple[list[SceneFrame], list[str], list[str], list[str]]:
95
  """Collapse the per-slide kept-frame list into one entry per topic.
96
 
97
  Each scene is assigned to the topic whose [start, end) window
98
  contains its start_seconds; scenes that fall outside every range
99
  (rare; usually only the head/tail when the LLM trimmed too tight)
100
  are bucketed into the closest topic by edge distance. Within each
101
+ topic, pick a primary that's actually slide content: prefer scenes
102
+ with substantial OCR text, then scenes with very low face area,
103
+ then anything left. Within whichever pool we picked from, the
104
+ longest-on-screen scene wins. That primary's existing within-scene
105
+ candidate alts are kept, and the other scenes' primary frames are
106
+ appended as additional alternates. Topics with no scenes are
107
+ dropped — a topic-segment with no image has no place in the editor.
108
+
109
+ `scene_ocr_texts` is an optional parallel-to-`kept` list of OCR
110
+ text per scene. When provided, it's the strongest signal we have
111
+ that a scene is real slide content rather than a talking-head shot
112
+ that squeaked past the instructor filter.
113
+
114
+ Returns (new_kept, titles, summaries, primary_ocr_texts) where the
115
+ four lists are parallel; `primary_ocr_texts` carries the OCR text
116
+ of whichever scene became each topic's primary (or "" if no OCR
117
+ was provided), so the caller can avoid re-running OCR on it.
118
  """
119
  if not topics or not kept:
120
+ empty = [""] * len(kept)
121
+ return kept, empty, empty, list(scene_ocr_texts or empty)
122
  scenes_by_topic: list[list[SceneFrame]] = [[] for _ in topics]
123
  for sf in kept:
124
  placed = False
 
137
  )
138
  scenes_by_topic[best_i].append(sf)
139
 
140
+ # If the caller passed OCR text per scene, build a same-length lookup
141
+ # so we can route each pre-rebucket scene's OCR to its pool decision.
142
+ ocr_by_id: dict[int, str] = {}
143
+ if scene_ocr_texts is not None:
144
+ for i, sf in enumerate(kept):
145
+ ocr_by_id[id(sf)] = (
146
+ scene_ocr_texts[i] if i < len(scene_ocr_texts) else ""
147
+ )
148
+
149
  new_kept: list[SceneFrame] = []
150
  titles: list[str] = []
151
  summaries: list[str] = []
152
+ new_ocr_texts: list[str] = []
153
  for topic, scenes in zip(topics, scenes_by_topic):
154
  if not scenes:
155
  log.info("Dropping topic '%s' — no scenes fell inside its range.", topic.title)
156
  continue
157
  scenes.sort(key=lambda s: s.start_seconds)
158
+ # Pool the scenes by "is this clearly a slide?":
159
+ # slide_text — substantial OCR text on the frame (the strongest
160
+ # signal we have that it's actual slide content, not a
161
+ # talking-head shot the face filter let through).
162
+ # clean_visual — no slide text but very small face area, so
163
+ # probably a decorative/photo slide (still useful).
164
+ # borderline — everything else (talking heads that squeaked
165
+ # past the instructor filter).
166
+ # Always pick from the highest non-empty pool, then take the
167
+ # longest-on-screen scene within it.
168
+ slide_text: list[SceneFrame] = []
169
+ clean_visual: list[SceneFrame] = []
170
+ borderline: list[SceneFrame] = []
171
+ for s in scenes:
172
+ text = ocr_by_id.get(id(s), "")
173
+ if len(text.strip()) >= _SLIDE_CONTENT_OCR_CHARS:
174
+ slide_text.append(s)
175
+ elif s.face_area_ratio < _SLIDE_CONTENT_FACE_RATIO:
176
+ clean_visual.append(s)
177
+ else:
178
+ borderline.append(s)
179
+ pool = slide_text or clean_visual or borderline
180
  primary = max(pool, key=lambda s: s.end_seconds - s.start_seconds)
181
  alt_paths: list[Path] = list(primary.alt_image_paths)
182
  for sf in scenes:
 
194
  )
195
  titles.append(topic.title)
196
  summaries.append(topic.summary)
197
+ new_ocr_texts.append(ocr_by_id.get(id(primary), ""))
198
+ return new_kept, titles, summaries, new_ocr_texts
199
 
200
 
201
  # A scene whose primary frame has more than this fraction of face is
 
205
  # instructor close-ups that snuck past.
206
  _SLIDE_CONTENT_FACE_RATIO = 0.03
207
 
208
+ # Minimum OCR character count for a scene to be treated as "clearly
209
+ # slide content" when picking a topic's primary frame. Empirically
210
+ # anything with ≥30 chars of recognised text is at least a title slide
211
+ # or a short bullet — pure talking-head frames OCR to ≤ a few stray chars.
212
+ _SLIDE_CONTENT_OCR_CHARS = 30
213
+
214
 
215
  def _load_cues(inputs: PipelineInputs) -> tuple[list[Cue], bool]:
216
  """Return (cues, used_auto_transcribe)."""
 
288
  if kept[-1].end_seconds < full_duration:
289
  kept[-1].end_seconds = full_duration
290
 
291
+ # OCR runs here — before topic segmentation — because the topic
292
+ # primary picker uses OCR text as the strongest signal that a
293
+ # scene is real slide content (rather than a talking-head that
294
+ # squeaked past the instructor filter). For the non-topic path
295
+ # the placement makes no difference; this just consolidates the
296
+ # OCR pass into one location.
297
+ if inputs.skip_ocr:
298
+ log.info("Skipping OCR (--skip-ocr)")
299
+ _emit(63, "ocr", "Skipping OCR.")
300
+ ocr_texts = ["" for _ in kept]
301
+ else:
302
+ n = len(kept)
303
+ log.info("Running OCR on %d frames", n)
304
+ _emit(63, "ocr", f"Running OCR on {n} frames...")
305
+ ocr_texts = []
306
+ for i, f in enumerate(kept):
307
+ ocr_texts.append(
308
+ ocr_frame(f.image_path, skip_inverted=inputs.skip_inverted_ocr)
309
+ )
310
+ pct = 63 + ((i + 1) / max(n, 1)) * 12
311
+ _emit(pct, "ocr", f"OCR frame {i + 1}/{n}")
312
+
313
  # Optional: replace scene-based one-segment-per-slide with LLM topic
314
  # segmentation. A topic groups all the scenes it covers — the
315
+ # OCR-richest, slide-content scene becomes the primary frame, the
316
+ # rest become switchable alternates in the editor. Surfaces
317
+ # failures rather than silently falling back, per the toggle's
318
+ # contract.
319
  topic_titles: list[str] = []
320
  topic_summaries: list[str] = []
321
  if inputs.llm_toggles.topic_segmentation:
322
  with_images = inputs.llm_toggles.topic_segmentation_with_images
323
  _emit(
324
+ 76,
325
  "topics",
326
  "Segmenting transcript into topics"
327
  + (" (with slide images)..." if with_images else "..."),
 
340
  _emit(100, "error", str(exc))
341
  raise PipelineError(f"Topic segmentation failed: {exc}") from exc
342
  log.info("LLM produced %d topics", len(topics))
343
+ kept, topic_titles, topic_summaries, ocr_texts = _apply_topic_segmentation(
344
+ kept, topics, scene_ocr_texts=ocr_texts
345
+ )
346
  if not kept:
347
  _emit(100, "error", "Topic segmentation left no usable segments.")
348
  raise PipelineError(
 
351
 
352
  boundaries = [f.start_seconds for f in kept]
353
  narrations = align_segments(cues, boundaries, full_duration)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
354
  # Alt-frame OCR is computed lazily by the /ocr endpoint when the
355
  # reviewer actually picks one — running it for every candidate up
356
  # front tripled OCR time on the HF free CPU tier.
tests/test_topic_segmentation.py CHANGED
@@ -141,7 +141,7 @@ def test_apply_topic_segmentation_collapses_scenes_under_one_topic():
141
  Topic(start_seconds=0.0, end_seconds=30.0, title="Setup"),
142
  Topic(start_seconds=30.0, end_seconds=60.0, title="Heat pumps"),
143
  ]
144
- new_kept, titles, summaries = _apply_topic_segmentation(scenes, topics)
145
 
146
  assert len(new_kept) == 2
147
  assert titles == ["Setup", "Heat pumps"]
@@ -176,7 +176,7 @@ def test_apply_topic_segmentation_drops_empty_topics():
176
  Topic(start_seconds=0.0, end_seconds=30.0, title="Real"),
177
  Topic(start_seconds=30.0, end_seconds=60.0, title="Empty"),
178
  ]
179
- new_kept, titles, summaries = _apply_topic_segmentation(scenes, topics)
180
  assert len(new_kept) == 1
181
  assert titles == ["Real"]
182
 
@@ -195,7 +195,7 @@ def test_apply_topic_segmentation_buckets_outlier_scenes_into_nearest_topic():
195
  Topic(start_seconds=8.0, end_seconds=30.0, title="Mid"),
196
  Topic(start_seconds=30.0, end_seconds=70.0, title="End"),
197
  ]
198
- new_kept, titles, _ = _apply_topic_segmentation(scenes, topics)
199
  assert len(new_kept) == 2
200
  # Scene 0 (start=0) is closer to topic[0].start=8 than to either
201
  # edge of topic[1]; ends up in topic 0's alts.
@@ -223,7 +223,7 @@ def test_apply_topic_segmentation_prefers_slide_content_over_borderline_face():
223
  _sf(1, 40.0, 55.0, face_area_ratio=0.0),
224
  ]
225
  topics = [Topic(start_seconds=0.0, end_seconds=60.0, title="Heat pumps")]
226
- new_kept, _, _ = _apply_topic_segmentation(scenes, topics)
227
  assert len(new_kept) == 1
228
  # Scene 1 (clean) wins despite being shorter.
229
  assert new_kept[0].image_path.name == "scene_0001.jpg"
@@ -244,17 +244,95 @@ def test_apply_topic_segmentation_falls_back_when_all_scenes_are_borderline():
244
  _sf(2, 30.0, 40.0, face_area_ratio=0.06),
245
  ]
246
  topics = [Topic(start_seconds=0.0, end_seconds=40.0, title="All borderline")]
247
- new_kept, _, _ = _apply_topic_segmentation(scenes, topics)
248
  assert len(new_kept) == 1
249
  # Scene 1 wins on duration in the borderline pool.
250
  assert new_kept[0].image_path.name == "scene_0001.jpg"
251
 
252
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
253
  def test_apply_topic_segmentation_handles_empty_input():
254
  """Defensive: no scenes or no topics → return inputs unchanged."""
255
- assert _apply_topic_segmentation([], []) == ([], [], [])
256
  scenes = [_sf(0, 0.0, 5.0)]
257
- out_kept, titles, summaries = _apply_topic_segmentation(scenes, [])
258
  assert out_kept == scenes
259
  assert titles == [""]
260
  assert summaries == [""]
 
141
  Topic(start_seconds=0.0, end_seconds=30.0, title="Setup"),
142
  Topic(start_seconds=30.0, end_seconds=60.0, title="Heat pumps"),
143
  ]
144
+ new_kept, titles, summaries, _ = _apply_topic_segmentation(scenes, topics)
145
 
146
  assert len(new_kept) == 2
147
  assert titles == ["Setup", "Heat pumps"]
 
176
  Topic(start_seconds=0.0, end_seconds=30.0, title="Real"),
177
  Topic(start_seconds=30.0, end_seconds=60.0, title="Empty"),
178
  ]
179
+ new_kept, titles, summaries, _ = _apply_topic_segmentation(scenes, topics)
180
  assert len(new_kept) == 1
181
  assert titles == ["Real"]
182
 
 
195
  Topic(start_seconds=8.0, end_seconds=30.0, title="Mid"),
196
  Topic(start_seconds=30.0, end_seconds=70.0, title="End"),
197
  ]
198
+ new_kept, titles, _, _ = _apply_topic_segmentation(scenes, topics)
199
  assert len(new_kept) == 2
200
  # Scene 0 (start=0) is closer to topic[0].start=8 than to either
201
  # edge of topic[1]; ends up in topic 0's alts.
 
223
  _sf(1, 40.0, 55.0, face_area_ratio=0.0),
224
  ]
225
  topics = [Topic(start_seconds=0.0, end_seconds=60.0, title="Heat pumps")]
226
+ new_kept, _, _, _ = _apply_topic_segmentation(scenes, topics)
227
  assert len(new_kept) == 1
228
  # Scene 1 (clean) wins despite being shorter.
229
  assert new_kept[0].image_path.name == "scene_0001.jpg"
 
244
  _sf(2, 30.0, 40.0, face_area_ratio=0.06),
245
  ]
246
  topics = [Topic(start_seconds=0.0, end_seconds=40.0, title="All borderline")]
247
+ new_kept, _, _, _ = _apply_topic_segmentation(scenes, topics)
248
  assert len(new_kept) == 1
249
  # Scene 1 wins on duration in the borderline pool.
250
  assert new_kept[0].image_path.name == "scene_0001.jpg"
251
 
252
 
253
+ def test_apply_topic_segmentation_prefers_scene_with_slide_text_over_talking_head():
254
+ """The hard case from real lectures: a topic contains a long
255
+ talking-head scene (no OCR text, face below the filter's drop
256
+ threshold so it survived) and a shorter scene with real slide
257
+ content. The OCR-text signal must dominate face-area and
258
+ duration so the slide wins as primary.
259
+ """
260
+ scenes = [
261
+ # Long talking-head shot, no OCR text but a small visible face.
262
+ _sf(0, 0.0, 40.0, face_area_ratio=0.06),
263
+ # Shorter clean slide with substantial OCR'd text.
264
+ _sf(1, 40.0, 50.0, face_area_ratio=0.0),
265
+ ]
266
+ topics = [Topic(start_seconds=0.0, end_seconds=60.0, title="Heat pumps")]
267
+ new_kept, _, _, primary_ocr = _apply_topic_segmentation(
268
+ scenes,
269
+ topics,
270
+ scene_ocr_texts=[
271
+ "", # talking head: no on-screen text
272
+ "Learning outcomes: identify climate education definitions and goals",
273
+ ],
274
+ )
275
+ assert len(new_kept) == 1
276
+ assert new_kept[0].image_path.name == "scene_0001.jpg"
277
+ assert "Learning outcomes" in primary_ocr[0]
278
+ # The talking head is still available as an alternate.
279
+ alt_names = [p.name for p in new_kept[0].alt_image_paths]
280
+ assert "scene_0000.jpg" in alt_names
281
+
282
+
283
+ def test_apply_topic_segmentation_falls_back_to_face_area_when_no_scene_has_slide_text():
284
+ """If no scene in a topic has substantial OCR text (intro stretch,
285
+ pure Q&A), the picker falls back to face-area: prefer the cleanest
286
+ face frame even though none of them are slides.
287
+ """
288
+ scenes = [
289
+ # Long but borderline face area.
290
+ _sf(0, 0.0, 30.0, face_area_ratio=0.08),
291
+ # Shorter and cleaner.
292
+ _sf(1, 30.0, 45.0, face_area_ratio=0.0),
293
+ ]
294
+ topics = [Topic(start_seconds=0.0, end_seconds=50.0, title="Intro")]
295
+ new_kept, _, _, _ = _apply_topic_segmentation(
296
+ scenes, topics, scene_ocr_texts=["", ""]
297
+ )
298
+ assert len(new_kept) == 1
299
+ # Scene 1 (no face) wins over the longer scene 0 (borderline face).
300
+ assert new_kept[0].image_path.name == "scene_0001.jpg"
301
+
302
+
303
+ def test_apply_topic_segmentation_returns_ocr_aligned_to_new_primaries():
304
+ """The fourth return value carries the OCR text of whichever scene
305
+ became each topic's primary. The orchestrator threads this back
306
+ into its ocr_texts list so downstream rendering doesn't re-OCR.
307
+ """
308
+ scenes = [
309
+ _sf(0, 0.0, 10.0),
310
+ _sf(1, 10.0, 25.0), # primary by duration
311
+ _sf(2, 30.0, 50.0), # primary by duration
312
+ ]
313
+ topics = [
314
+ Topic(start_seconds=0.0, end_seconds=30.0, title="A"),
315
+ Topic(start_seconds=30.0, end_seconds=60.0, title="B"),
316
+ ]
317
+ new_kept, _, _, primary_ocr = _apply_topic_segmentation(
318
+ scenes,
319
+ topics,
320
+ scene_ocr_texts=[
321
+ "Slide zero text " * 5, # ≥30 chars: a real slide
322
+ "Slide one text " * 5,
323
+ "Slide two content " * 5,
324
+ ],
325
+ )
326
+ assert len(new_kept) == 2
327
+ assert "Slide one" in primary_ocr[0]
328
+ assert "Slide two" in primary_ocr[1]
329
+
330
+
331
  def test_apply_topic_segmentation_handles_empty_input():
332
  """Defensive: no scenes or no topics → return inputs unchanged."""
333
+ assert _apply_topic_segmentation([], []) == ([], [], [], [])
334
  scenes = [_sf(0, 0.0, 5.0)]
335
+ out_kept, titles, summaries, _ = _apply_topic_segmentation(scenes, [])
336
  assert out_kept == scenes
337
  assert titles == [""]
338
  assert summaries == [""]