Spaces:
Sleeping
Sleeping
Commit ·
915f7b8
1
Parent(s): 6e3fc02
Auto-extract transcribes first when no transcript exists
Browse filesNarration-gating now works regardless of button order: Auto-extract runs Whisper
first if needed, surfaces the transcript in the box + state (reused, not recomputed),
and never clobbers a transcript the user may have edited.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
app.py
CHANGED
|
@@ -172,22 +172,38 @@ def on_capture(session: str, frames: list[dict], data_url: str, current_time: fl
|
|
| 172 |
|
| 173 |
|
| 174 |
def on_auto(session: str, frames: list[dict], video_path: str, transcript_obj, progress=gr.Progress()):
|
|
|
|
| 175 |
if not video_path:
|
| 176 |
-
return _gallery_value(frames), frames, "Upload a video first."
|
| 177 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 178 |
spoken = (
|
| 179 |
[(s.start, s.end) for s in transcript_obj.segments]
|
| 180 |
if transcript_obj and transcript_obj.segments
|
| 181 |
else None
|
| 182 |
)
|
|
|
|
| 183 |
recs = extract_auto_frames(video_path, config.session_dir(session), spoken_intervals=spoken)
|
| 184 |
merged = frames + [asdict(r) for r in recs]
|
| 185 |
progress(1.0, "Done.")
|
| 186 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 187 |
return (
|
| 188 |
_gallery_value(merged),
|
| 189 |
merged,
|
| 190 |
-
|
|
|
|
|
|
|
| 191 |
)
|
| 192 |
|
| 193 |
|
|
@@ -213,14 +229,19 @@ def on_clear():
|
|
| 213 |
return [], [], None, "Cleared all frames."
|
| 214 |
|
| 215 |
|
| 216 |
-
def
|
| 217 |
-
|
| 218 |
-
return "", None, "Upload a video first."
|
| 219 |
sdir = config.session_dir(session)
|
| 220 |
progress(0.05, "Extracting audio…")
|
| 221 |
wav = video.extract_audio(video_path, sdir / "audio.wav")
|
| 222 |
progress(0.1, "Loading Whisper…")
|
| 223 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 224 |
return (
|
| 225 |
tr.to_timestamped_text(),
|
| 226 |
tr,
|
|
@@ -342,8 +363,8 @@ def build_ui() -> gr.Blocks:
|
|
| 342 |
with gr.Column(scale=2):
|
| 343 |
gr.Markdown(
|
| 344 |
"### Captured frames\n"
|
| 345 |
-
"
|
| 346 |
-
"
|
| 347 |
)
|
| 348 |
gallery = gr.Gallery(
|
| 349 |
label="Frames pool — click an image to enlarge / select it",
|
|
@@ -394,7 +415,7 @@ def build_ui() -> gr.Blocks:
|
|
| 394 |
auto_btn.click(
|
| 395 |
on_auto,
|
| 396 |
[session_state, frames_state, video_state, transcript_state],
|
| 397 |
-
[gallery, frames_state, status],
|
| 398 |
)
|
| 399 |
gallery.select(on_select_frame, None, [selected_state, status])
|
| 400 |
delete_btn.click(
|
|
|
|
| 172 |
|
| 173 |
|
| 174 |
def on_auto(session: str, frames: list[dict], video_path: str, transcript_obj, progress=gr.Progress()):
|
| 175 |
+
# Outputs: gallery, frames_state, transcript_box, transcript_state, status
|
| 176 |
if not video_path:
|
| 177 |
+
return _gallery_value(frames), frames, gr.update(), gr.update(), "Upload a video first."
|
| 178 |
+
|
| 179 |
+
# Transcribe first if we don't have a transcript yet, so frames can be gated
|
| 180 |
+
# to the narration (otherwise recorder intro/idle screens get captured).
|
| 181 |
+
auto_transcribed = False
|
| 182 |
+
if not (transcript_obj and getattr(transcript_obj, "segments", None)):
|
| 183 |
+
progress(0.0, "No transcript yet — transcribing first…")
|
| 184 |
+
transcript_obj = _run_transcription(session, video_path, progress)
|
| 185 |
+
auto_transcribed = True
|
| 186 |
+
|
| 187 |
spoken = (
|
| 188 |
[(s.start, s.end) for s in transcript_obj.segments]
|
| 189 |
if transcript_obj and transcript_obj.segments
|
| 190 |
else None
|
| 191 |
)
|
| 192 |
+
progress(0.85, "Detecting scenes…")
|
| 193 |
recs = extract_auto_frames(video_path, config.session_dir(session), spoken_intervals=spoken)
|
| 194 |
merged = frames + [asdict(r) for r in recs]
|
| 195 |
progress(1.0, "Done.")
|
| 196 |
+
|
| 197 |
+
# If we just transcribed, surface it in the box + state so it's reused (and
|
| 198 |
+
# don't clobber a transcript the user may have already edited).
|
| 199 |
+
box_out = transcript_obj.to_timestamped_text() if auto_transcribed else gr.update()
|
| 200 |
+
note = " (auto-transcribed first)" if auto_transcribed else ""
|
| 201 |
return (
|
| 202 |
_gallery_value(merged),
|
| 203 |
merged,
|
| 204 |
+
box_out,
|
| 205 |
+
transcript_obj,
|
| 206 |
+
f"Auto-extracted {len(recs)} frames{note} ({len(merged)} total).",
|
| 207 |
)
|
| 208 |
|
| 209 |
|
|
|
|
| 229 |
return [], [], None, "Cleared all frames."
|
| 230 |
|
| 231 |
|
| 232 |
+
def _run_transcription(session: str, video_path: str, progress):
|
| 233 |
+
"""Extract audio and transcribe — shared by Transcribe and Auto-extract."""
|
|
|
|
| 234 |
sdir = config.session_dir(session)
|
| 235 |
progress(0.05, "Extracting audio…")
|
| 236 |
wav = video.extract_audio(video_path, sdir / "audio.wav")
|
| 237 |
progress(0.1, "Loading Whisper…")
|
| 238 |
+
return transcribe_lib.transcribe(wav, progress=progress)
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
def on_transcribe(session: str, video_path: str, progress=gr.Progress()):
|
| 242 |
+
if not video_path:
|
| 243 |
+
return "", None, "Upload a video first."
|
| 244 |
+
tr = _run_transcription(session, video_path, progress)
|
| 245 |
return (
|
| 246 |
tr.to_timestamped_text(),
|
| 247 |
tr,
|
|
|
|
| 363 |
with gr.Column(scale=2):
|
| 364 |
gr.Markdown(
|
| 365 |
"### Captured frames\n"
|
| 366 |
+
"_**Auto-extract** transcribes first automatically, so frames snap to "
|
| 367 |
+
"the narration and skip recorder intro/idle screens._"
|
| 368 |
)
|
| 369 |
gallery = gr.Gallery(
|
| 370 |
label="Frames pool — click an image to enlarge / select it",
|
|
|
|
| 415 |
auto_btn.click(
|
| 416 |
on_auto,
|
| 417 |
[session_state, frames_state, video_state, transcript_state],
|
| 418 |
+
[gallery, frames_state, transcript_box, transcript_state, status],
|
| 419 |
)
|
| 420 |
gallery.select(on_select_frame, None, [selected_state, status])
|
| 421 |
delete_btn.click(
|