vivekchakraverty Claude Opus 4.8 commited on
Commit
915f7b8
·
1 Parent(s): 6e3fc02

Auto-extract transcribes first when no transcript exists

Browse files

Narration-gating now works regardless of button order: Auto-extract runs Whisper
first if needed, surfaces the transcript in the box + state (reused, not recomputed),
and never clobbers a transcript the user may have edited.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Files changed (1) hide show
  1. app.py +32 -11
app.py CHANGED
@@ -172,22 +172,38 @@ def on_capture(session: str, frames: list[dict], data_url: str, current_time: fl
172
 
173
 
174
  def on_auto(session: str, frames: list[dict], video_path: str, transcript_obj, progress=gr.Progress()):
 
175
  if not video_path:
176
- return _gallery_value(frames), frames, "Upload a video first."
177
- progress(0.1, "Detecting scenes…")
 
 
 
 
 
 
 
 
178
  spoken = (
179
  [(s.start, s.end) for s in transcript_obj.segments]
180
  if transcript_obj and transcript_obj.segments
181
  else None
182
  )
 
183
  recs = extract_auto_frames(video_path, config.session_dir(session), spoken_intervals=spoken)
184
  merged = frames + [asdict(r) for r in recs]
185
  progress(1.0, "Done.")
186
- tip = "" if spoken else " · tip: transcribe first for narration-aligned frames"
 
 
 
 
187
  return (
188
  _gallery_value(merged),
189
  merged,
190
- f"Auto-extracted {len(recs)} frames ({len(merged)} total).{tip}",
 
 
191
  )
192
 
193
 
@@ -213,14 +229,19 @@ def on_clear():
213
  return [], [], None, "Cleared all frames."
214
 
215
 
216
- def on_transcribe(session: str, video_path: str, progress=gr.Progress()):
217
- if not video_path:
218
- return "", None, "Upload a video first."
219
  sdir = config.session_dir(session)
220
  progress(0.05, "Extracting audio…")
221
  wav = video.extract_audio(video_path, sdir / "audio.wav")
222
  progress(0.1, "Loading Whisper…")
223
- tr = transcribe_lib.transcribe(wav, progress=progress)
 
 
 
 
 
 
224
  return (
225
  tr.to_timestamped_text(),
226
  tr,
@@ -342,8 +363,8 @@ def build_ui() -> gr.Blocks:
342
  with gr.Column(scale=2):
343
  gr.Markdown(
344
  "### Captured frames\n"
345
- "_Tip: **Transcribe** (step 2) before **Auto-extract** frames then "
346
- "snap to the narration and skip recorder intro/idle screens._"
347
  )
348
  gallery = gr.Gallery(
349
  label="Frames pool — click an image to enlarge / select it",
@@ -394,7 +415,7 @@ def build_ui() -> gr.Blocks:
394
  auto_btn.click(
395
  on_auto,
396
  [session_state, frames_state, video_state, transcript_state],
397
- [gallery, frames_state, status],
398
  )
399
  gallery.select(on_select_frame, None, [selected_state, status])
400
  delete_btn.click(
 
172
 
173
 
174
  def on_auto(session: str, frames: list[dict], video_path: str, transcript_obj, progress=gr.Progress()):
175
+ # Outputs: gallery, frames_state, transcript_box, transcript_state, status
176
  if not video_path:
177
+ return _gallery_value(frames), frames, gr.update(), gr.update(), "Upload a video first."
178
+
179
+ # Transcribe first if we don't have a transcript yet, so frames can be gated
180
+ # to the narration (otherwise recorder intro/idle screens get captured).
181
+ auto_transcribed = False
182
+ if not (transcript_obj and getattr(transcript_obj, "segments", None)):
183
+ progress(0.0, "No transcript yet — transcribing first…")
184
+ transcript_obj = _run_transcription(session, video_path, progress)
185
+ auto_transcribed = True
186
+
187
  spoken = (
188
  [(s.start, s.end) for s in transcript_obj.segments]
189
  if transcript_obj and transcript_obj.segments
190
  else None
191
  )
192
+ progress(0.85, "Detecting scenes…")
193
  recs = extract_auto_frames(video_path, config.session_dir(session), spoken_intervals=spoken)
194
  merged = frames + [asdict(r) for r in recs]
195
  progress(1.0, "Done.")
196
+
197
+ # If we just transcribed, surface it in the box + state so it's reused (and
198
+ # don't clobber a transcript the user may have already edited).
199
+ box_out = transcript_obj.to_timestamped_text() if auto_transcribed else gr.update()
200
+ note = " (auto-transcribed first)" if auto_transcribed else ""
201
  return (
202
  _gallery_value(merged),
203
  merged,
204
+ box_out,
205
+ transcript_obj,
206
+ f"Auto-extracted {len(recs)} frames{note} ({len(merged)} total).",
207
  )
208
 
209
 
 
229
  return [], [], None, "Cleared all frames."
230
 
231
 
232
+ def _run_transcription(session: str, video_path: str, progress):
233
+ """Extract audio and transcribe — shared by Transcribe and Auto-extract."""
 
234
  sdir = config.session_dir(session)
235
  progress(0.05, "Extracting audio…")
236
  wav = video.extract_audio(video_path, sdir / "audio.wav")
237
  progress(0.1, "Loading Whisper…")
238
+ return transcribe_lib.transcribe(wav, progress=progress)
239
+
240
+
241
+ def on_transcribe(session: str, video_path: str, progress=gr.Progress()):
242
+ if not video_path:
243
+ return "", None, "Upload a video first."
244
+ tr = _run_transcription(session, video_path, progress)
245
  return (
246
  tr.to_timestamped_text(),
247
  tr,
 
363
  with gr.Column(scale=2):
364
  gr.Markdown(
365
  "### Captured frames\n"
366
+ "_**Auto-extract** transcribes first automatically, so frames snap to "
367
+ "the narration and skip recorder intro/idle screens._"
368
  )
369
  gallery = gr.Gallery(
370
  label="Frames pool — click an image to enlarge / select it",
 
415
  auto_btn.click(
416
  on_auto,
417
  [session_state, frames_state, video_state, transcript_state],
418
+ [gallery, frames_state, transcript_box, transcript_state, status],
419
  )
420
  gallery.select(on_select_frame, None, [selected_state, status])
421
  delete_btn.click(