Make auto-frame extraction narration-accurate

- Gate scene-detected frames to the spoken time range (drops screen-recorder
intro/idle screens) when a transcript is available.
- Weight transcript/LLM step timestamps far more in per-step frame selection:
tighter window, exclude out-of-speech frames, and extract a fresh frame at the
exact step time when no close frame exists.
- Thread the transcript into auto-extract + build; hint users to transcribe first.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

app.py

@@ -171,17 +171,23 @@ def on_capture(session: str, frames: list[dict], data_url: str, current_time: fl
     )
 
 
-def on_auto(session: str, frames: list[dict], video_path: str, progress=gr.Progress()):
+def on_auto(session: str, frames: list[dict], video_path: str, transcript_obj, progress=gr.Progress()):
     if not video_path:
         return _gallery_value(frames), frames, "Upload a video first."
     progress(0.1, "Detecting scenes…")
-    recs = extract_auto_frames(video_path, config.session_dir(session))
+    spoken = (
+        [(s.start, s.end) for s in transcript_obj.segments]
+        if transcript_obj and transcript_obj.segments
+        else None
+    )
+    recs = extract_auto_frames(video_path, config.session_dir(session), spoken_intervals=spoken)
     merged = frames + [asdict(r) for r in recs]
     progress(1.0, "Done.")
+    tip = "" if spoken else "  ·  tip: transcribe first for narration-aligned frames"
     return (
         _gallery_value(merged),
         merged,
-        f"Auto-extracted {len(recs)} frames ({len(merged)} total).",
+        f"Auto-extracted {len(recs)} frames ({len(merged)} total).{tip}",
     )
 
 
@@ -254,12 +260,19 @@ def on_build(
     video_path: str,
     do_caption: bool,
     hf_token: str,
+    transcript_obj,
     progress=gr.Progress(),
 ):
     if draft is None or not getattr(draft, "steps", None):
         return None, "Generate the step-by-step guide first."
     token = config.apply_token(hf_token)
     recs = [FrameRecord(**d) for d in frames]
+    spoken_range = None
+    if transcript_obj and transcript_obj.segments:
+        spoken_range = (
+            min(s.start for s in transcript_obj.segments),
+            max(s.end for s in transcript_obj.segments),
+        )
     progress(0.1, "Matching images to steps…")
     g = guide_lib.assemble_guide(
         draft,
@@ -268,6 +281,7 @@ def on_build(
         session_dir=config.session_dir(session),
         do_caption=do_caption,
         token=token,
+        spoken_range=spoken_range,
         progress=progress,
     )
     out = config.session_dir(session) / "guide.docx"
@@ -326,7 +340,11 @@ def build_ui() -> gr.Blocks:
                     capture_btn = gr.Button("📸 Capture current frame", variant="primary")
                     auto_btn = gr.Button("✨ Auto-extract frames")
             with gr.Column(scale=2):
-                gr.Markdown("### Captured frames")
+                gr.Markdown(
+                    "### Captured frames\n"
+                    "_Tip: **Transcribe** (step 2) before **Auto-extract** — frames then "
+                    "snap to the narration and skip recorder intro/idle screens._"
+                )
                 gallery = gr.Gallery(
                     label="Frames pool — click an image to enlarge / select it",
                     elem_id="dm-gallery",
@@ -373,7 +391,11 @@ def build_ui() -> gr.Blocks:
             outputs=[gallery, frames_state, status],
             js=CAPTURE_JS,
         )
-        auto_btn.click(on_auto, [session_state, frames_state, video_state], [gallery, frames_state, status])
+        auto_btn.click(
+            on_auto,
+            [session_state, frames_state, video_state, transcript_state],
+            [gallery, frames_state, status],
+        )
         gallery.select(on_select_frame, None, [selected_state, status])
         delete_btn.click(
             on_delete_frame,
@@ -392,7 +414,7 @@ def build_ui() -> gr.Blocks:
         )
         build_btn.click(
             on_build,
-            [session_state, draft_state, frames_state, video_state, caption_chk, hf_token],
+            [session_state, draft_state, frames_state, video_state, caption_chk, hf_token, transcript_state],
             [download, status],
         )
 

scripts/smoke_test.py

@@ -54,8 +54,9 @@ def main() -> None:
     print(f"  device={tr.device} segments={len(tr.segments)} text={tr.text[:120]!r}")
     assert tr.text.strip(), "Transcript is empty"
 
-    print("[2/5] Auto-extract frames…")
-    recs = frames_lib.extract_auto_frames(sample, sdir)
+    print("[2/5] Auto-extract frames (narration-gated)…")
+    spoken = [(s.start, s.end) for s in tr.segments] if tr.segments else None
+    recs = frames_lib.extract_auto_frames(sample, sdir, spoken_intervals=spoken)
     print(f"  frames={len(recs)}")
     assert recs, "No frames were extracted"
 
@@ -71,8 +72,13 @@ def main() -> None:
     assert draft.steps, "No steps in draft"
 
     print("[4/5] Assemble (align + caption)…")
+    spoken_range = (
+        (min(s.start for s in tr.segments), max(s.end for s in tr.segments))
+        if tr.segments else None
+    )
     g = guide_lib.assemble_guide(
-        draft, recs, video_path=str(sample), session_dir=sdir, do_caption=True, token=token
+        draft, recs, video_path=str(sample), session_dir=sdir, do_caption=True,
+        token=token, spoken_range=spoken_range,
     )
 
     print("[5/5] Export DOCX…")

src/frames.py

@@ -45,7 +45,11 @@ def detect_scenes(video_path: str | Path) -> list[tuple[float, float]]:
     return [(start.get_seconds(), end.get_seconds()) for start, end in scenes]
 
 
-def _scene_timestamps(video_path: str | Path, max_frames: int) -> list[float]:
+def _scene_timestamps(
+    video_path: str | Path,
+    max_frames: int,
+    spoken_intervals: list[tuple[float, float]] | None = None,
+) -> list[float]:
     scenes = detect_scenes(video_path)
     if scenes:
         timestamps = [(start + end) / 2.0 for start, end in scenes]
@@ -56,6 +60,18 @@ def _scene_timestamps(video_path: str | Path, max_frames: int) -> list[float]:
         step = duration / (count + 1)
         timestamps = [step * (i + 1) for i in range(count)]
 
+    # Anchor to the narration: keep only frames within the spoken time range.
+    # This drops screen-recorder intro/outro and idle screens (no speech there),
+    # which is the main source of irrelevant auto-frames.
+    if spoken_intervals:
+        lo = min(s for s, _ in spoken_intervals)
+        hi = max(e for _, e in spoken_intervals)
+        pad = 1.5
+        gated = [t for t in timestamps if lo - pad <= t <= hi + pad]
+        # If gating removed everything, fall back to the segment start times.
+        timestamps = gated or [s for s, _ in spoken_intervals]
+
+    timestamps = sorted(timestamps)
     # Keep an evenly spaced subset if we overshoot the cap.
     if len(timestamps) > max_frames:
         last = len(timestamps) - 1
@@ -65,14 +81,21 @@ def _scene_timestamps(video_path: str | Path, max_frames: int) -> list[float]:
 
 
 def extract_auto_frames(
-    video_path: str | Path, session_dir: str | Path, max_frames: int = 40
+    video_path: str | Path,
+    session_dir: str | Path,
+    max_frames: int = 40,
+    spoken_intervals: list[tuple[float, float]] | None = None,
 ) -> list[FrameRecord]:
-    """Extract one representative frame per detected scene, then dedup."""
+    """Extract one representative frame per detected scene, then dedup.
+
+    When ``spoken_intervals`` (from the transcript) are given, frames are gated to
+    the narrated time range so recorder intro/idle screens are not captured.
+    """
     frames_dir = Path(session_dir) / "frames"
     frames_dir.mkdir(parents=True, exist_ok=True)
 
     records: list[FrameRecord] = []
-    for i, ts in enumerate(_scene_timestamps(video_path, max_frames)):
+    for i, ts in enumerate(_scene_timestamps(video_path, max_frames, spoken_intervals)):
         out = frames_dir / f"auto_{i:03d}_{int(ts * 1000):08d}.png"
         try:
             video.extract_frame(video_path, ts, out)

src/guide.py

@@ -35,10 +35,15 @@ class Guide:
     steps: list[GuideStep] = field(default_factory=list)
 
 
-# Relevance weights. Timestamp proximity is the most reliable signal for
-# tutorials; the BLIP-caption match and sharpness refine the choice, break ties,
-# and carry the decision when a step has no timestamp.
-_W_PROX, _W_SEM, _W_SHARP, _MANUAL_BONUS = 0.55, 0.30, 0.15, 0.25
+# Relevance weights. Timestamp proximity (transcript/LLM time) is by far the most
+# reliable signal for tutorials and is weighted heavily; the BLIP-caption match and
+# sharpness only break ties.
+_W_PROX, _W_SEM, _W_SHARP = 0.70, 0.18, 0.12
+# How far (seconds) a frame may sit from a step's LLM timestamp to be reused.
+# Manual frames get a wider window (the user captured them on purpose); scene/auto
+# frames must be close, otherwise we extract a fresh frame at the exact step time.
+_MANUAL_WINDOW = 20.0
+_AUTO_WINDOW = 12.0
 
 _STOPWORDS = {
     "the", "a", "an", "to", "of", "and", "or", "in", "on", "at", "for", "with",
@@ -82,39 +87,51 @@ def _pick_frame(
     timestamp: float | None,
     step_text: str,
     used: set[str],
-    window: float = 30.0,
+    spoken_range: tuple[float, float] | None = None,
 ) -> FrameRecord | None:
-    """Pick the most relevant frame for a step by combining accurate signals:
-    timestamp proximity + sharpness + BLIP-caption semantic match + manual bonus.
+    """Pick the best existing frame for a step, anchored to its LLM timestamp.
+
+    Returns ``None`` when no pool frame sits close enough in time — the caller then
+    extracts a fresh frame at the exact step timestamp, which keeps every step's
+    image aligned to the narration rather than to an unrelated visual scene change.
     """
-    pool = [f for f in candidates if f.path not in used] or candidates
-    if not pool:
+    avail = [f for f in candidates if f.path not in used] or candidates
+    if not avail:
         return None
 
+    # No LLM timestamp for this step: fall back to caption relevance, then sharpness.
     if timestamp is None:
-        within = pool
-    else:
-        within = [f for f in pool if abs(f.timestamp - timestamp) <= window] or pool
-
-    sharps = [_sharpness(f.path) for f in within]
+        return max(avail, key=lambda f: (_text_relevance(f.caption, step_text), _sharpness(f.path)))
+
+    # 1) A manual frame captured near this step wins — it's deliberate user intent.
+    manual_near = [
+        f for f in avail
+        if f.source == "manual" and abs(f.timestamp - timestamp) <= _MANUAL_WINDOW
+    ]
+    if manual_near:
+        return min(manual_near, key=lambda f: abs(f.timestamp - timestamp))
+
+    # 2) Scene/auto frames tightly around the step's LLM time, inside the spoken range.
+    near = [f for f in avail if abs(f.timestamp - timestamp) <= _AUTO_WINDOW]
+    if spoken_range:
+        lo, hi = spoken_range
+        in_speech = [f for f in near if lo - 2.0 <= f.timestamp <= hi + 2.0]
+        near = in_speech or near
+    if not near:
+        return None  # -> caller extracts a fresh frame at the exact step time
+
+    sharps = [_sharpness(f.path) for f in near]
     smin, smax = min(sharps), max(sharps)
 
     def norm_sharp(value: float) -> float:
         return (value - smin) / (smax - smin) if smax > smin else 1.0
 
     def score(frame: FrameRecord) -> float:
-        if timestamp is None:
-            prox = 0.0
-        else:
-            prox = 1.0 - min(abs(frame.timestamp - timestamp) / window, 1.0)
+        prox = 1.0 - min(abs(frame.timestamp - timestamp) / _AUTO_WINDOW, 1.0)
         sem = _text_relevance(frame.caption, step_text)
-        sharp = norm_sharp(_sharpness(frame.path))
-        total = _W_PROX * prox + _W_SEM * sem + _W_SHARP * sharp
-        if frame.source == "manual":
-            total += _MANUAL_BONUS
-        return total
+        return _W_PROX * prox + _W_SEM * sem + _W_SHARP * norm_sharp(_sharpness(frame.path))
 
-    return max(within, key=score)
+    return max(near, key=score)
 
 
 def assemble_guide(
@@ -125,13 +142,15 @@ def assemble_guide(
     session_dir: str | Path | None = None,
     do_caption: bool = True,
     token: str | None = None,
+    spoken_range: tuple[float, float] | None = None,
     progress: Callable[[float, str], None] | None = None,
 ) -> Guide:
     """Combine a guide draft with frames into a fully illustrated :class:`Guide`.
 
     When captioning is on, the whole (deduped) frame pool is captioned once so
     BLIP can both *suggest* the most relevant frame per step and supply the
-    figure captions.
+    figure captions. ``spoken_range`` (first/last narration time) keeps selection
+    inside the narrated portion of the video.
     """
     frames_sorted = sorted(frames, key=lambda f: f.timestamp)
 
@@ -156,7 +175,7 @@ def assemble_guide(
             progress(0.5 + 0.5 * (i / total), f"Matching image to step {i + 1}/{total}…")
 
         step_text = f"{sd.heading} {sd.text}".strip()
-        chosen = _pick_frame(frames_sorted, sd.approx_timestamp, step_text, used)
+        chosen = _pick_frame(frames_sorted, sd.approx_timestamp, step_text, used, spoken_range)
 
         # No suitable frame nearby — extract one at the step timestamp.
         if (