Claude commited on
Commit
7e20c47
·
1 Parent(s): 816dd45

Add LLM topic-based segmentation as an opt-in toggle

Browse files

Replaces the scene-based one-segment-per-slide bucketing with topical
sections generated by Claude from the transcript. One extra LLM call
per job (transcript-only, no images) returns 3–15 sequential topics
with titles and summaries; the orchestrator then re-buckets the kept
scene frames so each topic-segment carries the longest-on-screen
scene as its primary image and the other scenes as switchable
alternates in the editor.

Per the design discussion:
- Opt-in via a new LLMToggles.topic_segmentation field; existing
scene-based path stays the default.
- Multi-slide topics keep every slide accessible (primary + cross-
scene alts on top of the per-scene candidate alts).
- LLM failure raises PipelineError so the job surfaces a clear error
rather than silently falling back.

The /alts and /ocr endpoints now also accept the scene_NNNN.jpg
primary pattern because cross-scene alternates re-use those filenames.

app/main.py CHANGED
@@ -261,6 +261,7 @@ async def start_job(
261
  llm_math_equations: bool = Form(False),
262
  llm_section_title: bool = Form(False),
263
  llm_ocr_text: bool = Form(False),
 
264
  extract_audio: bool = Form(False),
265
  ) -> JSONResponse:
266
  _validate_format(format)
@@ -304,6 +305,7 @@ async def start_job(
304
  math_equations=llm_math_equations,
305
  section_title=llm_section_title,
306
  ocr_text=llm_ocr_text,
 
307
  ),
308
  extract_audio=extract_audio,
309
  )
@@ -453,7 +455,11 @@ async def fetch_audio(job_id: str, filename: str) -> Response:
453
  async def fetch_alt(job_id: str, filename: str) -> Response:
454
  # Filename comes from a URL path, so be strict about its shape — only
455
  # patterns the pipeline produces are accepted, and no traversal chars.
456
- if not _ALT_FILENAME_RE.match(filename):
 
 
 
 
457
  raise HTTPException(404, "Unknown alternate.")
458
  job = registry.get(job_id)
459
  if not job:
@@ -475,7 +481,9 @@ async def fetch_ocr(job_id: str, filename: str) -> Response:
475
  # tier, so we skip it during the main pipeline and run it here only
476
  # when the reviewer actually picks an alt. First call writes the
477
  # cached bytes to a temp file and OCRs; subsequent calls return cache.
478
- if not _ALT_FILENAME_RE.match(filename):
 
 
479
  raise HTTPException(404, "Unknown alternate.")
480
  job = registry.get(job_id)
481
  if not job:
@@ -649,6 +657,7 @@ async def process(
649
  llm_math_equations: bool = Form(False),
650
  llm_section_title: bool = Form(False),
651
  llm_ocr_text: bool = Form(False),
 
652
  extract_audio: bool = Form(False),
653
  ) -> Response:
654
  _validate_format(format)
@@ -691,6 +700,7 @@ async def process(
691
  math_equations=llm_math_equations,
692
  section_title=llm_section_title,
693
  ocr_text=llm_ocr_text,
 
694
  ),
695
  extract_audio=extract_audio,
696
  )
 
261
  llm_math_equations: bool = Form(False),
262
  llm_section_title: bool = Form(False),
263
  llm_ocr_text: bool = Form(False),
264
+ llm_topic_segmentation: bool = Form(False),
265
  extract_audio: bool = Form(False),
266
  ) -> JSONResponse:
267
  _validate_format(format)
 
305
  math_equations=llm_math_equations,
306
  section_title=llm_section_title,
307
  ocr_text=llm_ocr_text,
308
+ topic_segmentation=llm_topic_segmentation,
309
  ),
310
  extract_audio=extract_audio,
311
  )
 
455
  async def fetch_alt(job_id: str, filename: str) -> Response:
456
  # Filename comes from a URL path, so be strict about its shape — only
457
  # patterns the pipeline produces are accepted, and no traversal chars.
458
+ # Topic-based segmentation re-uses other scenes' primary frames as
459
+ # alternates, so we also accept the primary scene_NNNN.jpg shape.
460
+ if not (
461
+ _ALT_FILENAME_RE.match(filename) or _PRIMARY_FILENAME_RE.match(filename)
462
+ ):
463
  raise HTTPException(404, "Unknown alternate.")
464
  job = registry.get(job_id)
465
  if not job:
 
481
  # tier, so we skip it during the main pipeline and run it here only
482
  # when the reviewer actually picks an alt. First call writes the
483
  # cached bytes to a temp file and OCRs; subsequent calls return cache.
484
+ if not (
485
+ _ALT_FILENAME_RE.match(filename) or _PRIMARY_FILENAME_RE.match(filename)
486
+ ):
487
  raise HTTPException(404, "Unknown alternate.")
488
  job = registry.get(job_id)
489
  if not job:
 
657
  llm_math_equations: bool = Form(False),
658
  llm_section_title: bool = Form(False),
659
  llm_ocr_text: bool = Form(False),
660
+ llm_topic_segmentation: bool = Form(False),
661
  extract_audio: bool = Form(False),
662
  ) -> Response:
663
  _validate_format(format)
 
700
  math_equations=llm_math_equations,
701
  section_title=llm_section_title,
702
  ocr_text=llm_ocr_text,
703
+ topic_segmentation=llm_topic_segmentation,
704
  ),
705
  extract_audio=extract_audio,
706
  )
app/pipeline/llm_extract.py CHANGED
@@ -47,6 +47,13 @@ class LLMToggles:
47
  # primary frame (already paid for if any other toggle is on; almost
48
  # free in tokens since it's added to the same request).
49
  ocr_text: bool = False
 
 
 
 
 
 
 
50
 
51
  @property
52
  def any_enabled(self) -> bool:
@@ -341,3 +348,196 @@ def extract_all(
341
  if on_progress:
342
  on_progress(completed, n)
343
  return results
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47
  # primary frame (already paid for if any other toggle is on; almost
48
  # free in tokens since it's added to the same request).
49
  ocr_text: bool = False
50
+ # When True, Claude segments the full transcript into topical
51
+ # sections that replace the scene-based one-segment-per-slide
52
+ # bucketing. Pure-transcript call (no images) made once per job
53
+ # before per-segment extraction; orchestrator rebuilds the kept
54
+ # frames into topic groups. Reviewer still gets all the original
55
+ # slides in each topic as switchable alternates.
56
+ topic_segmentation: bool = False
57
 
58
  @property
59
  def any_enabled(self) -> bool:
 
348
  if on_progress:
349
  on_progress(completed, n)
350
  return results
351
+
352
+
353
+ # ---------------------------------------------------------------------------
354
+ # Topic segmentation (one extra LLM call per job; replaces scene-based buckets)
355
+ # ---------------------------------------------------------------------------
356
+
357
+
358
+ @dataclass
359
+ class Topic:
360
+ """A continuous time-range of the lecture covering one coherent subject.
361
+
362
+ Replaces the scene-based "one segment per slide" model when the
363
+ topic_segmentation toggle is on. All slides whose start_seconds fall
364
+ inside [start_seconds, end_seconds) get attached to the same topic
365
+ segment; the reviewer can switch between them in the editor.
366
+ """
367
+ start_seconds: float
368
+ end_seconds: float
369
+ title: str
370
+ summary: str = ""
371
+
372
+
373
+ class LLMTopicError(RuntimeError):
374
+ """Raised when topic segmentation fails (API error, malformed JSON,
375
+ invalid topic schedule). The orchestrator surfaces this to the user
376
+ rather than silently falling back to scene-based segmentation."""
377
+
378
+
379
+ class _ExtractedTopic(BaseModel):
380
+ start_seconds: float
381
+ end_seconds: float
382
+ title: str
383
+ summary: str = ""
384
+
385
+
386
+ class _ExtractedTopicList(BaseModel):
387
+ topics: list[_ExtractedTopic]
388
+
389
+
390
+ _TOPIC_SYSTEM_PROMPT = (
391
+ "You segment lecture transcripts into topical sections.\n\n"
392
+ "Given a transcript with per-cue timestamps, return 3–15 sequential, "
393
+ "non-overlapping topics that together cover the entire duration. Each "
394
+ "topic must capture a single coherent subject the lecturer discusses — "
395
+ "prefer fewer, meaningful sections over many short ones. Topic "
396
+ "boundaries should fall at natural conversational shifts (new subject, "
397
+ "transitions like \"now let's talk about\", a clear pause in flow).\n\n"
398
+ "For each topic, provide:\n"
399
+ " - start_seconds, end_seconds: float timestamps in seconds, matching "
400
+ "the transcript timestamps. The first topic must start at the transcript's "
401
+ "first cue and the last topic must end at the final cue's end.\n"
402
+ " - title: ≤8 words, sentence case, no trailing punctuation. Describe "
403
+ "the topic substantively (\"How heat pumps work in cold climates\"), "
404
+ "not the slide chrome (\"Slide 2\").\n"
405
+ " - summary: one-sentence (≤30 words) plain-prose recap of the topic.\n\n"
406
+ "Return ONLY the structured topics. Do not invent content; if the "
407
+ "transcript is too short or off-topic for clean segmentation, return a "
408
+ "single topic covering the full range."
409
+ )
410
+
411
+
412
+ def _format_cues_for_topic_prompt(cues: list) -> str:
413
+ """One line per cue: [mm:ss] cue text. Compact for token budget."""
414
+ lines: list[str] = []
415
+ for c in cues:
416
+ m = int(c.start // 60)
417
+ s = int(c.start % 60)
418
+ lines.append(f"[{m:02d}:{s:02d}] {c.text}")
419
+ return "\n".join(lines)
420
+
421
+
422
+ def _validate_topics(
423
+ raw_topics: list[_ExtractedTopic],
424
+ *,
425
+ total_duration: float,
426
+ transcript_start: float,
427
+ ) -> list[Topic]:
428
+ """Coerce LLM output into a clean, sequential schedule covering the
429
+ full duration. Raises LLMTopicError if the output is unsalvageable.
430
+
431
+ Tolerates small rounding/overlap (≤0.5s) by snapping each topic's
432
+ start to the previous topic's end. Tolerates a small gap at the
433
+ head/tail by extending the first/last topic. Rejects empty output,
434
+ backwards ranges, or gaps larger than ~10% of the video.
435
+ """
436
+ if not raw_topics:
437
+ raise LLMTopicError("LLM returned no topics.")
438
+ # Sort by start in case the model returned them slightly out of order.
439
+ items = sorted(
440
+ [
441
+ Topic(
442
+ start_seconds=float(t.start_seconds),
443
+ end_seconds=float(t.end_seconds),
444
+ title=t.title.strip(),
445
+ summary=(t.summary or "").strip(),
446
+ )
447
+ for t in raw_topics
448
+ ],
449
+ key=lambda t: t.start_seconds,
450
+ )
451
+ out: list[Topic] = []
452
+ prev_end = transcript_start
453
+ for t in items:
454
+ if not t.title:
455
+ raise LLMTopicError("A topic is missing a title.")
456
+ if t.end_seconds <= t.start_seconds:
457
+ raise LLMTopicError(
458
+ f"Topic '{t.title}' has end ({t.end_seconds}) <= start ({t.start_seconds})."
459
+ )
460
+ # Snap to previous end if the model produced a small overlap or gap.
461
+ snapped_start = max(prev_end, t.start_seconds)
462
+ if snapped_start >= t.end_seconds:
463
+ # Skip degenerate topics that collapse after snapping; the
464
+ # previous topic effectively swallowed this one.
465
+ continue
466
+ out.append(
467
+ Topic(
468
+ start_seconds=snapped_start,
469
+ end_seconds=t.end_seconds,
470
+ title=t.title,
471
+ summary=t.summary,
472
+ )
473
+ )
474
+ prev_end = t.end_seconds
475
+ if not out:
476
+ raise LLMTopicError("All topics collapsed after snapping; nothing usable.")
477
+ # Pin head/tail to the actual timeline so downstream alignment doesn't
478
+ # leave a head/tail gap.
479
+ out[0] = Topic(
480
+ start_seconds=min(out[0].start_seconds, transcript_start),
481
+ end_seconds=out[0].end_seconds,
482
+ title=out[0].title,
483
+ summary=out[0].summary,
484
+ )
485
+ out[-1] = Topic(
486
+ start_seconds=out[-1].start_seconds,
487
+ end_seconds=max(out[-1].end_seconds, total_duration),
488
+ title=out[-1].title,
489
+ summary=out[-1].summary,
490
+ )
491
+ return out
492
+
493
+
494
+ def extract_topics(
495
+ cues: list,
496
+ *,
497
+ total_duration: float,
498
+ model: str = DEFAULT_MODEL,
499
+ api_key: str | None = None,
500
+ ) -> list[Topic]:
501
+ """Ask Claude to segment the transcript into topical sections.
502
+
503
+ Returns a sequential, non-overlapping list of Topics covering
504
+ [0, total_duration). Raises LLMTopicError on API failure, malformed
505
+ response, or unsalvageable topic schedule — the orchestrator
506
+ surfaces this as a job error rather than silently falling back.
507
+ """
508
+ if not cues:
509
+ raise LLMTopicError("No transcript cues; cannot segment topics.")
510
+ resolved = _resolve_api_key(api_key)
511
+ if resolved is None:
512
+ raise LLMTopicError(
513
+ "Topic segmentation requires an Anthropic API key, but none was provided."
514
+ )
515
+ transcript_block = _format_cues_for_topic_prompt(cues)
516
+ user_text = (
517
+ f"Transcript (mm:ss timestamps refer to cue starts):\n\n{transcript_block}"
518
+ )
519
+ try:
520
+ client = _build_client(resolved)
521
+ response = client.messages.parse(
522
+ model=model,
523
+ max_tokens=4096,
524
+ system=[
525
+ {
526
+ "type": "text",
527
+ "text": _TOPIC_SYSTEM_PROMPT,
528
+ "cache_control": {"type": "ephemeral"},
529
+ }
530
+ ],
531
+ messages=[{"role": "user", "content": user_text}],
532
+ output_format=_ExtractedTopicList,
533
+ )
534
+ except Exception as exc: # pragma: no cover (network-dependent)
535
+ raise LLMTopicError(f"LLM topic-segmentation request failed: {exc}") from exc
536
+ parsed = response.parsed_output
537
+ if parsed is None:
538
+ raise LLMTopicError("LLM returned no parsed topic output.")
539
+ return _validate_topics(
540
+ parsed.topics,
541
+ total_duration=total_duration,
542
+ transcript_start=cues[0].start,
543
+ )
app/pipeline/orchestrator.py CHANGED
@@ -14,7 +14,14 @@ from typing import Callable
14
 
15
  from .audio import slice_all as slice_segment_audio, ffmpeg_available
16
  from .filter import filter_instructor_frames
17
- from .llm_extract import LLMToggles, extract_all, is_enabled as llm_enabled
 
 
 
 
 
 
 
18
  from .metadata import PageMetadata, SourceInfo, page_metadata_from_segments
19
  from .ocr import ocr_frame
20
  from .scenes import detect_and_extract, video_duration
@@ -79,6 +86,73 @@ class PipelineError(RuntimeError):
79
  """Raised when the pipeline cannot produce a usable study guide."""
80
 
81
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
82
  def _load_cues(inputs: PipelineInputs) -> tuple[list[Cue], bool]:
83
  """Return (cues, used_auto_transcribe)."""
84
  if inputs.transcript_path and inputs.transcript_path.exists():
@@ -137,7 +211,6 @@ def run_pipeline(
137
  cues, auto_transcribed = _load_cues(inputs)
138
  log.info("Loaded %d cues (auto_transcribed=%s)", len(cues), auto_transcribed)
139
 
140
- boundaries = [f.start_seconds for f in kept]
141
  # Use the actual video duration as the right edge so transcript cues
142
  # past the last kept visual frame still get bucketed into the final
143
  # segment. Without this, when trailing scenes are dropped (instructor-
@@ -146,6 +219,34 @@ def run_pipeline(
146
  full_duration = max(video_duration(inputs.video_path), kept[-1].end_seconds)
147
  if kept[-1].end_seconds < full_duration:
148
  kept[-1].end_seconds = full_duration
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
149
  narrations = align_segments(cues, boundaries, full_duration)
150
 
151
  if inputs.skip_ocr:
@@ -249,6 +350,14 @@ def run_pipeline(
249
  seg.alt_text_reviewed = True
250
  if inputs.llm_toggles.section_title and extracted.section_title:
251
  seg.section_title = extracted.section_title
 
 
 
 
 
 
 
 
252
  for i, seg in enumerate(segments):
253
  ap = audio_paths.get(i)
254
  if ap is not None:
 
14
 
15
  from .audio import slice_all as slice_segment_audio, ffmpeg_available
16
  from .filter import filter_instructor_frames
17
+ from .llm_extract import (
18
+ LLMToggles,
19
+ LLMTopicError,
20
+ Topic,
21
+ extract_all,
22
+ extract_topics,
23
+ is_enabled as llm_enabled,
24
+ )
25
  from .metadata import PageMetadata, SourceInfo, page_metadata_from_segments
26
  from .ocr import ocr_frame
27
  from .scenes import detect_and_extract, video_duration
 
86
  """Raised when the pipeline cannot produce a usable study guide."""
87
 
88
 
89
+ def _apply_topic_segmentation(
90
+ kept: list[SceneFrame],
91
+ topics: list[Topic],
92
+ ) -> tuple[list[SceneFrame], list[str], list[str]]:
93
+ """Collapse the per-slide kept-frame list into one entry per topic.
94
+
95
+ Each scene is assigned to the topic whose [start, end) window
96
+ contains its start_seconds; scenes that fall outside every range
97
+ (rare; usually only the head/tail when the LLM trimmed too tight)
98
+ are bucketed into the closest topic by edge distance. Within each
99
+ topic, the longest-on-screen scene becomes the primary; that
100
+ primary's existing within-scene candidate alts are kept, and the
101
+ other scenes' primary frames are appended as additional alternates.
102
+ Topics with no scenes are dropped — a topic-segment with no image
103
+ has no place in the editor.
104
+
105
+ Returns (new_kept, titles, summaries) where the three lists are
106
+ parallel and the titles/summaries carry the LLM's labels for
107
+ downstream use as section_title / segment summary.
108
+ """
109
+ if not topics or not kept:
110
+ return kept, [""] * len(kept), [""] * len(kept)
111
+ scenes_by_topic: list[list[SceneFrame]] = [[] for _ in topics]
112
+ for sf in kept:
113
+ placed = False
114
+ for i, t in enumerate(topics):
115
+ if t.start_seconds <= sf.start_seconds < t.end_seconds:
116
+ scenes_by_topic[i].append(sf)
117
+ placed = True
118
+ break
119
+ if not placed:
120
+ best_i = min(
121
+ range(len(topics)),
122
+ key=lambda i: min(
123
+ abs(sf.start_seconds - topics[i].start_seconds),
124
+ abs(sf.start_seconds - topics[i].end_seconds),
125
+ ),
126
+ )
127
+ scenes_by_topic[best_i].append(sf)
128
+
129
+ new_kept: list[SceneFrame] = []
130
+ titles: list[str] = []
131
+ summaries: list[str] = []
132
+ for topic, scenes in zip(topics, scenes_by_topic):
133
+ if not scenes:
134
+ log.info("Dropping topic '%s' — no scenes fell inside its range.", topic.title)
135
+ continue
136
+ scenes.sort(key=lambda s: s.start_seconds)
137
+ primary = max(scenes, key=lambda s: s.end_seconds - s.start_seconds)
138
+ alt_paths: list[Path] = list(primary.alt_image_paths)
139
+ for sf in scenes:
140
+ if sf is not primary:
141
+ alt_paths.append(sf.image_path)
142
+ new_kept.append(
143
+ SceneFrame(
144
+ index=len(new_kept),
145
+ start_seconds=topic.start_seconds,
146
+ end_seconds=topic.end_seconds,
147
+ image_path=primary.image_path,
148
+ alt_image_paths=alt_paths,
149
+ )
150
+ )
151
+ titles.append(topic.title)
152
+ summaries.append(topic.summary)
153
+ return new_kept, titles, summaries
154
+
155
+
156
  def _load_cues(inputs: PipelineInputs) -> tuple[list[Cue], bool]:
157
  """Return (cues, used_auto_transcribe)."""
158
  if inputs.transcript_path and inputs.transcript_path.exists():
 
211
  cues, auto_transcribed = _load_cues(inputs)
212
  log.info("Loaded %d cues (auto_transcribed=%s)", len(cues), auto_transcribed)
213
 
 
214
  # Use the actual video duration as the right edge so transcript cues
215
  # past the last kept visual frame still get bucketed into the final
216
  # segment. Without this, when trailing scenes are dropped (instructor-
 
219
  full_duration = max(video_duration(inputs.video_path), kept[-1].end_seconds)
220
  if kept[-1].end_seconds < full_duration:
221
  kept[-1].end_seconds = full_duration
222
+
223
+ # Optional: replace scene-based one-segment-per-slide with LLM topic
224
+ # segmentation. A topic groups all the scenes it covers — the
225
+ # longest-on-screen one becomes the primary frame, the rest become
226
+ # switchable alternates in the editor. Surfaces failures rather than
227
+ # silently falling back, per the toggle's contract.
228
+ topic_titles: list[str] = []
229
+ topic_summaries: list[str] = []
230
+ if inputs.llm_toggles.topic_segmentation:
231
+ _emit(62, "topics", "Segmenting transcript into topics...")
232
+ try:
233
+ topics = extract_topics(
234
+ cues,
235
+ total_duration=full_duration,
236
+ api_key=inputs.anthropic_api_key,
237
+ )
238
+ except LLMTopicError as exc:
239
+ _emit(100, "error", str(exc))
240
+ raise PipelineError(f"Topic segmentation failed: {exc}") from exc
241
+ log.info("LLM produced %d topics", len(topics))
242
+ kept, topic_titles, topic_summaries = _apply_topic_segmentation(kept, topics)
243
+ if not kept:
244
+ _emit(100, "error", "Topic segmentation left no usable segments.")
245
+ raise PipelineError(
246
+ "Topic segmentation produced no segments that contained any slides."
247
+ )
248
+
249
+ boundaries = [f.start_seconds for f in kept]
250
  narrations = align_segments(cues, boundaries, full_duration)
251
 
252
  if inputs.skip_ocr:
 
350
  seg.alt_text_reviewed = True
351
  if inputs.llm_toggles.section_title and extracted.section_title:
352
  seg.section_title = extracted.section_title
353
+ # Topic segmentation produced its own titles in the same LLM call
354
+ # that built the segments — override anything the per-segment
355
+ # section_title toggle wrote, since the topic-level title knows
356
+ # the broader context and is more accurate for a multi-slide span.
357
+ if topic_titles:
358
+ for seg, title in zip(segments, topic_titles):
359
+ if title:
360
+ seg.section_title = title
361
  for i, seg in enumerate(segments):
362
  ap = audio_paths.get(i)
363
  if ap is not None:
app/templates/index.html CHANGED
@@ -583,6 +583,13 @@
583
  <small>Replaces Tesseract OCR for primary frames; better on coloured callouts and decorative fonts</small>
584
  </span>
585
  </label>
 
 
 
 
 
 
 
586
  </div>
587
  </div>
588
 
@@ -977,7 +984,7 @@
977
  if (typeof v === 'string' && v.trim() === '') fd.delete(k);
978
  });
979
  // Unchecked LLM toggles must explicitly send "false" so the server doesn't apply its default-on.
980
- ['llm_alt_text', 'llm_key_terms', 'llm_math_equations', 'llm_section_title', 'llm_ocr_text', 'extract_audio'].forEach(k => {
981
  if (!fd.has(k)) fd.set(k, 'false');
982
  });
983
  if (apiKeyEl) apiKeyEl.value = '';
 
583
  <small>Replaces Tesseract OCR for primary frames; better on coloured callouts and decorative fonts</small>
584
  </span>
585
  </label>
586
+ <label class="vgm-llm-toggle">
587
+ <input type="checkbox" name="llm_topic_segmentation" value="true">
588
+ <span class="vgm-llm-toggle__body">
589
+ <strong>Topic-based segmentation</strong>
590
+ <small>Group slides into topical sections instead of one segment per slide. Claude segments the transcript; each topic gets the longest-on-screen slide as primary and the other slides as switchable alternates. Fewer, more coherent sections; narration won't break mid-sentence.</small>
591
+ </span>
592
+ </label>
593
  </div>
594
  </div>
595
 
 
984
  if (typeof v === 'string' && v.trim() === '') fd.delete(k);
985
  });
986
  // Unchecked LLM toggles must explicitly send "false" so the server doesn't apply its default-on.
987
+ ['llm_alt_text', 'llm_key_terms', 'llm_math_equations', 'llm_section_title', 'llm_ocr_text', 'llm_topic_segmentation', 'extract_audio'].forEach(k => {
988
  if (!fd.has(k)) fd.set(k, 'false');
989
  });
990
  if (apiKeyEl) apiKeyEl.value = '';
tests/test_topic_segmentation.py ADDED
@@ -0,0 +1,212 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Tests for the topic-segmentation pipeline branch.
2
+
3
+ Covers the two deterministic pieces — schedule validation and the
4
+ re-bucketing of scene frames into topic-segments. The LLM call itself
5
+ is network-bound and excluded from unit tests.
6
+ """
7
+ from pathlib import Path
8
+
9
+ import pytest
10
+
11
+ from app.pipeline.llm_extract import (
12
+ LLMTopicError,
13
+ Topic,
14
+ _ExtractedTopic,
15
+ _validate_topics,
16
+ )
17
+ from app.pipeline.orchestrator import _apply_topic_segmentation
18
+ from app.pipeline.types import SceneFrame
19
+
20
+
21
+ # ---------------------------------------------------------------------------
22
+ # _validate_topics
23
+ # ---------------------------------------------------------------------------
24
+
25
+
26
+ def _et(start: float, end: float, title: str = "T", summary: str = "") -> _ExtractedTopic:
27
+ return _ExtractedTopic(start_seconds=start, end_seconds=end, title=title, summary=summary)
28
+
29
+
30
+ def test_validate_topics_rejects_empty_list():
31
+ with pytest.raises(LLMTopicError, match="no topics"):
32
+ _validate_topics([], total_duration=60.0, transcript_start=0.0)
33
+
34
+
35
+ def test_validate_topics_rejects_missing_title():
36
+ with pytest.raises(LLMTopicError, match="missing a title"):
37
+ _validate_topics(
38
+ [_et(0.0, 30.0, title=""), _et(30.0, 60.0, title="ok")],
39
+ total_duration=60.0,
40
+ transcript_start=0.0,
41
+ )
42
+
43
+
44
+ def test_validate_topics_rejects_backwards_range():
45
+ with pytest.raises(LLMTopicError, match="end .* <= start"):
46
+ _validate_topics(
47
+ [_et(10.0, 5.0)],
48
+ total_duration=60.0,
49
+ transcript_start=0.0,
50
+ )
51
+
52
+
53
+ def test_validate_topics_snaps_small_overlap_to_previous_end():
54
+ """LLM sometimes returns end-times that slightly overlap the next
55
+ topic's start (rounding). We snap to enforce monotonic non-overlap
56
+ rather than reject — the boundary is approximate anyway.
57
+ """
58
+ out = _validate_topics(
59
+ [_et(0.0, 30.5, "Intro"), _et(30.0, 60.0, "Body")],
60
+ total_duration=60.0,
61
+ transcript_start=0.0,
62
+ )
63
+ assert len(out) == 2
64
+ assert out[0].end_seconds == 30.5
65
+ assert out[1].start_seconds == 30.5 # snapped forward
66
+ assert out[1].end_seconds == 60.0
67
+
68
+
69
+ def test_validate_topics_drops_topic_swallowed_by_snap():
70
+ """If snapping makes a topic's start >= its own end, the previous
71
+ topic effectively swallowed it; drop the collapsed one rather than
72
+ emit a zero-length segment.
73
+ """
74
+ out = _validate_topics(
75
+ [_et(0.0, 40.0, "Intro"), _et(10.0, 35.0, "Inside intro")],
76
+ total_duration=60.0,
77
+ transcript_start=0.0,
78
+ )
79
+ assert len(out) == 1
80
+ assert out[0].title == "Intro"
81
+
82
+
83
+ def test_validate_topics_pins_head_and_tail_to_full_timeline():
84
+ """The orchestrator passes the actual video duration; the LLM's
85
+ last topic should extend to cover any narration after the last cue
86
+ boundary it picked, and its first should reach back to the start.
87
+ """
88
+ out = _validate_topics(
89
+ [_et(5.0, 40.0, "Intro"), _et(40.0, 55.0, "Outro")],
90
+ total_duration=70.0,
91
+ transcript_start=0.0,
92
+ )
93
+ assert out[0].start_seconds == 0.0
94
+ assert out[-1].end_seconds == 70.0
95
+
96
+
97
+ def test_validate_topics_sorts_out_of_order_input():
98
+ out = _validate_topics(
99
+ [_et(40.0, 60.0, "Late"), _et(0.0, 40.0, "Early")],
100
+ total_duration=60.0,
101
+ transcript_start=0.0,
102
+ )
103
+ assert [t.title for t in out] == ["Early", "Late"]
104
+
105
+
106
+ # ---------------------------------------------------------------------------
107
+ # _apply_topic_segmentation
108
+ # ---------------------------------------------------------------------------
109
+
110
+
111
+ def _sf(idx: int, start: float, end: float, *, alts: int = 0) -> SceneFrame:
112
+ return SceneFrame(
113
+ index=idx,
114
+ start_seconds=start,
115
+ end_seconds=end,
116
+ image_path=Path(f"scene_{idx:04d}.jpg"),
117
+ alt_image_paths=[Path(f"scene_{idx:04d}_alt{j}.jpg") for j in range(1, alts + 1)],
118
+ )
119
+
120
+
121
+ def test_apply_topic_segmentation_collapses_scenes_under_one_topic():
122
+ """The canonical case: 4 scenes, 2 topics. Each topic-segment
123
+ carries the longest-on-screen scene as primary and the others as
124
+ additional alternates the reviewer can switch to in the editor.
125
+ """
126
+ scenes = [
127
+ _sf(0, 0.0, 5.0),
128
+ _sf(1, 5.0, 25.0, alts=2), # longest in topic 0 — should be primary
129
+ _sf(2, 25.0, 35.0),
130
+ _sf(3, 35.0, 55.0, alts=2), # longest in topic 1 — should be primary
131
+ ]
132
+ topics = [
133
+ Topic(start_seconds=0.0, end_seconds=30.0, title="Setup"),
134
+ Topic(start_seconds=30.0, end_seconds=60.0, title="Heat pumps"),
135
+ ]
136
+ new_kept, titles, summaries = _apply_topic_segmentation(scenes, topics)
137
+
138
+ assert len(new_kept) == 2
139
+ assert titles == ["Setup", "Heat pumps"]
140
+
141
+ # Topic 0: scene 1 wins (20s vs 5s + 10s); scenes 0 and 2 attached as alts.
142
+ assert new_kept[0].image_path.name == "scene_0001.jpg"
143
+ assert new_kept[0].start_seconds == 0.0
144
+ assert new_kept[0].end_seconds == 30.0
145
+ alt_names_0 = [p.name for p in new_kept[0].alt_image_paths]
146
+ # Scene 1's within-scene candidate alts come first…
147
+ assert "scene_0001_alt1.jpg" in alt_names_0
148
+ assert "scene_0001_alt2.jpg" in alt_names_0
149
+ # …then the other scenes' primary frames.
150
+ assert "scene_0000.jpg" in alt_names_0
151
+ assert "scene_0002.jpg" in alt_names_0
152
+
153
+ # Topic 1: scene 3 wins; scene 2 (assigned to topic 1 if its start fell
154
+ # inside) — actually scene 2 starts at 25.0 which is in topic 0, so
155
+ # topic 1 only has scene 3.
156
+ assert new_kept[1].image_path.name == "scene_0003.jpg"
157
+ assert new_kept[1].start_seconds == 30.0
158
+ assert new_kept[1].end_seconds == 60.0
159
+
160
+
161
+ def test_apply_topic_segmentation_drops_empty_topics():
162
+ """A topic with no scenes (e.g. LLM imagined a section in a
163
+ visually static stretch) is dropped — a segment with no image has
164
+ no place in the editor.
165
+ """
166
+ scenes = [_sf(0, 0.0, 5.0), _sf(1, 5.0, 25.0)]
167
+ topics = [
168
+ Topic(start_seconds=0.0, end_seconds=30.0, title="Real"),
169
+ Topic(start_seconds=30.0, end_seconds=60.0, title="Empty"),
170
+ ]
171
+ new_kept, titles, summaries = _apply_topic_segmentation(scenes, topics)
172
+ assert len(new_kept) == 1
173
+ assert titles == ["Real"]
174
+
175
+
176
+ def test_apply_topic_segmentation_buckets_outlier_scenes_into_nearest_topic():
177
+ """If the LLM trims the time range tight, a scene whose start lies
178
+ just outside any topic range falls back to the closest topic by
179
+ edge distance — better than dropping it.
180
+ """
181
+ scenes = [
182
+ _sf(0, 0.0, 5.0), # before any topic
183
+ _sf(1, 10.0, 25.0),
184
+ _sf(2, 80.0, 95.0), # after the last topic
185
+ ]
186
+ topics = [
187
+ Topic(start_seconds=8.0, end_seconds=30.0, title="Mid"),
188
+ Topic(start_seconds=30.0, end_seconds=70.0, title="End"),
189
+ ]
190
+ new_kept, titles, _ = _apply_topic_segmentation(scenes, topics)
191
+ assert len(new_kept) == 2
192
+ # Scene 0 (start=0) is closer to topic[0].start=8 than to either
193
+ # edge of topic[1]; ends up in topic 0's alts.
194
+ topic0_alts = [p.name for p in new_kept[0].alt_image_paths]
195
+ assert "scene_0000.jpg" in topic0_alts
196
+ # Scene 2 (start=80) is closer to topic[1].end=70 than any topic 0
197
+ # edge; ends up in topic 1.
198
+ topic1_paths = (
199
+ new_kept[1].image_path.name,
200
+ *[p.name for p in new_kept[1].alt_image_paths],
201
+ )
202
+ assert "scene_0002.jpg" in topic1_paths
203
+
204
+
205
+ def test_apply_topic_segmentation_handles_empty_input():
206
+ """Defensive: no scenes or no topics → return inputs unchanged."""
207
+ assert _apply_topic_segmentation([], []) == ([], [], [])
208
+ scenes = [_sf(0, 0.0, 5.0)]
209
+ out_kept, titles, summaries = _apply_topic_segmentation(scenes, [])
210
+ assert out_kept == scenes
211
+ assert titles == [""]
212
+ assert summaries == [""]