Spaces:
Sleeping
Three fixes: stale ends, /jobs cleanup window, audio toggle
Browse files1. _enforce_min_gap and _cap_max_frames previously only renumbered the
kept frames. Each kept frame retained its ORIGINAL end_seconds even
though transcript-alignment uses the next kept frame's start as the
narration boundary. So a guide could display 'Segment 1: 0:00-0:05'
while its narration body actually covered 0:00 through 0:30 (when
the next kept scene started). New _absorb_dropped_intervals helper
extends each kept frame's end_seconds to the next kept frame's
start (or to the original tail for the last). Both helpers call it
after renumbering. New test pins the behavior.
2. /jobs widens the cleanup window: previously the try/except only
wrapped file-save and max_frames parsing; PipelineInputs(),
registry.create(), and Thread.start() ran outside it. If anything
in those last three failed (dataclass validation, OS thread limits,
etc.) the workdir would leak. Now everything pre-handoff is inside
a try/finally with a handed_off flag; rmtree only fires when the
worker thread didn't actually take over, so we never double-clean.
3. extract_audio toggle on the upload form plumbed through both /jobs
and /process to PipelineInputs.extract_audio. Lives in a new
'Media extras' section beneath the LLM toggles, marked as local-
only (no API key needed). Default-on; unticking turns off ffmpeg
slicing and the <audio> elements entirely. The JS that defends
against unchecked-checkbox-omitted-from-form-data was extended to
include extract_audio so the server's default-True doesn't override
a deliberate uncheck.
- app/main.py +47 -42
- app/pipeline/scenes.py +22 -1
- app/templates/index.html +18 -2
- tests/test_scenes.py +44 -0
|
@@ -221,6 +221,7 @@ async def start_job(
|
|
| 221 |
llm_key_terms: bool = Form(True),
|
| 222 |
llm_math_equations: bool = Form(True),
|
| 223 |
llm_section_title: bool = Form(True),
|
|
|
|
| 224 |
) -> JSONResponse:
|
| 225 |
_validate_format(format)
|
| 226 |
video_suffix = Path(video.filename or "").suffix.lower()
|
|
@@ -231,6 +232,7 @@ async def start_job(
|
|
| 231 |
raise HTTPException(400, f"Unsupported transcript format: {transcript_suffix or '(none)'}")
|
| 232 |
|
| 233 |
workdir = Path(tempfile.mkdtemp(prefix="vgm_"))
|
|
|
|
| 234 |
try:
|
| 235 |
video_path = workdir / f"input{video_suffix}"
|
| 236 |
transcript_path = workdir / f"transcript{transcript_suffix}"
|
|
@@ -240,51 +242,52 @@ async def start_job(
|
|
| 240 |
with transcript_path.open("wb") as f:
|
| 241 |
shutil.copyfileobj(transcript.file, f)
|
| 242 |
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
# we'd otherwise orphan the dir we just mkdtemp'd. The worker has
|
| 248 |
-
# its own try/finally that handles the post-handoff path.
|
| 249 |
-
shutil.rmtree(workdir, ignore_errors=True)
|
| 250 |
-
raise
|
| 251 |
-
|
| 252 |
-
inputs = PipelineInputs(
|
| 253 |
-
video_path=video_path,
|
| 254 |
-
transcript_path=transcript_path,
|
| 255 |
-
frames_dir=frames_dir,
|
| 256 |
-
title=title,
|
| 257 |
-
subtitle=subtitle or None,
|
| 258 |
-
module=module or None,
|
| 259 |
-
lang=lang,
|
| 260 |
-
threshold=threshold,
|
| 261 |
-
min_gap_seconds=min_gap,
|
| 262 |
-
max_frames=max_frames_value,
|
| 263 |
-
skip_ocr=skip_ocr,
|
| 264 |
-
face_threshold=face_threshold,
|
| 265 |
-
inline_images=(format in ("single", "review")),
|
| 266 |
-
anthropic_api_key=(anthropic_api_key or None),
|
| 267 |
-
llm_toggles=LLMToggles(
|
| 268 |
-
alt_text=llm_alt_text,
|
| 269 |
-
key_terms=llm_key_terms,
|
| 270 |
-
math_equations=llm_math_equations,
|
| 271 |
-
section_title=llm_section_title,
|
| 272 |
-
),
|
| 273 |
-
)
|
| 274 |
-
|
| 275 |
-
job = registry.create()
|
| 276 |
-
threading.Thread(
|
| 277 |
-
target=_run_job_in_thread,
|
| 278 |
-
args=(job, inputs, workdir),
|
| 279 |
-
kwargs=dict(
|
| 280 |
-
format=format,
|
| 281 |
title=title,
|
| 282 |
-
lang=lang,
|
| 283 |
subtitle=subtitle or None,
|
| 284 |
module=module or None,
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 288 |
|
| 289 |
return JSONResponse(
|
| 290 |
{
|
|
@@ -410,6 +413,7 @@ async def process(
|
|
| 410 |
llm_key_terms: bool = Form(True),
|
| 411 |
llm_math_equations: bool = Form(True),
|
| 412 |
llm_section_title: bool = Form(True),
|
|
|
|
| 413 |
) -> Response:
|
| 414 |
_validate_format(format)
|
| 415 |
video_suffix = Path(video.filename or "").suffix.lower()
|
|
@@ -450,6 +454,7 @@ async def process(
|
|
| 450 |
math_equations=llm_math_equations,
|
| 451 |
section_title=llm_section_title,
|
| 452 |
),
|
|
|
|
| 453 |
)
|
| 454 |
try:
|
| 455 |
result = run_pipeline(inputs)
|
|
|
|
| 221 |
llm_key_terms: bool = Form(True),
|
| 222 |
llm_math_equations: bool = Form(True),
|
| 223 |
llm_section_title: bool = Form(True),
|
| 224 |
+
extract_audio: bool = Form(True),
|
| 225 |
) -> JSONResponse:
|
| 226 |
_validate_format(format)
|
| 227 |
video_suffix = Path(video.filename or "").suffix.lower()
|
|
|
|
| 232 |
raise HTTPException(400, f"Unsupported transcript format: {transcript_suffix or '(none)'}")
|
| 233 |
|
| 234 |
workdir = Path(tempfile.mkdtemp(prefix="vgm_"))
|
| 235 |
+
handed_off = False
|
| 236 |
try:
|
| 237 |
video_path = workdir / f"input{video_suffix}"
|
| 238 |
transcript_path = workdir / f"transcript{transcript_suffix}"
|
|
|
|
| 242 |
with transcript_path.open("wb") as f:
|
| 243 |
shutil.copyfileobj(transcript.file, f)
|
| 244 |
|
| 245 |
+
inputs = PipelineInputs(
|
| 246 |
+
video_path=video_path,
|
| 247 |
+
transcript_path=transcript_path,
|
| 248 |
+
frames_dir=frames_dir,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 249 |
title=title,
|
|
|
|
| 250 |
subtitle=subtitle or None,
|
| 251 |
module=module or None,
|
| 252 |
+
lang=lang,
|
| 253 |
+
threshold=threshold,
|
| 254 |
+
min_gap_seconds=min_gap,
|
| 255 |
+
max_frames=_parse_optional_int(max_frames),
|
| 256 |
+
skip_ocr=skip_ocr,
|
| 257 |
+
face_threshold=face_threshold,
|
| 258 |
+
inline_images=(format in ("single", "review")),
|
| 259 |
+
anthropic_api_key=(anthropic_api_key or None),
|
| 260 |
+
llm_toggles=LLMToggles(
|
| 261 |
+
alt_text=llm_alt_text,
|
| 262 |
+
key_terms=llm_key_terms,
|
| 263 |
+
math_equations=llm_math_equations,
|
| 264 |
+
section_title=llm_section_title,
|
| 265 |
+
),
|
| 266 |
+
extract_audio=extract_audio,
|
| 267 |
+
)
|
| 268 |
+
|
| 269 |
+
job = registry.create()
|
| 270 |
+
threading.Thread(
|
| 271 |
+
target=_run_job_in_thread,
|
| 272 |
+
args=(job, inputs, workdir),
|
| 273 |
+
kwargs=dict(
|
| 274 |
+
format=format,
|
| 275 |
+
title=title,
|
| 276 |
+
lang=lang,
|
| 277 |
+
subtitle=subtitle or None,
|
| 278 |
+
module=module or None,
|
| 279 |
+
),
|
| 280 |
+
daemon=True,
|
| 281 |
+
).start()
|
| 282 |
+
handed_off = True
|
| 283 |
+
finally:
|
| 284 |
+
# Anything that fails before the worker thread takes ownership of
|
| 285 |
+
# workdir β bad form field, copy error, dataclass validation, even
|
| 286 |
+
# an OS thread-start failure β must clean up the temp dir we just
|
| 287 |
+
# mkdtemp'd. Once Thread.start() returns successfully, the worker's
|
| 288 |
+
# own try/finally handles cleanup.
|
| 289 |
+
if not handed_off:
|
| 290 |
+
shutil.rmtree(workdir, ignore_errors=True)
|
| 291 |
|
| 292 |
return JSONResponse(
|
| 293 |
{
|
|
|
|
| 413 |
llm_key_terms: bool = Form(True),
|
| 414 |
llm_math_equations: bool = Form(True),
|
| 415 |
llm_section_title: bool = Form(True),
|
| 416 |
+
extract_audio: bool = Form(True),
|
| 417 |
) -> Response:
|
| 418 |
_validate_format(format)
|
| 419 |
video_suffix = Path(video.filename or "").suffix.lower()
|
|
|
|
| 454 |
math_equations=llm_math_equations,
|
| 455 |
section_title=llm_section_title,
|
| 456 |
),
|
| 457 |
+
extract_audio=extract_audio,
|
| 458 |
)
|
| 459 |
try:
|
| 460 |
result = run_pipeline(inputs)
|
|
@@ -182,6 +182,25 @@ def _merge_duplicate_adjacent(
|
|
| 182 |
return merged
|
| 183 |
|
| 184 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 185 |
def _enforce_min_gap(frames: list[SceneFrame], min_gap_seconds: float) -> list[SceneFrame]:
|
| 186 |
"""Drop frames whose start is closer than min_gap_seconds to the previous kept frame.
|
| 187 |
|
|
@@ -201,7 +220,7 @@ def _enforce_min_gap(frames: list[SceneFrame], min_gap_seconds: float) -> list[S
|
|
| 201 |
if kept:
|
| 202 |
for new_index, frame in enumerate(kept):
|
| 203 |
frame.index = new_index
|
| 204 |
-
|
| 205 |
return kept
|
| 206 |
|
| 207 |
|
|
@@ -227,6 +246,8 @@ def _cap_max_frames(frames: list[SceneFrame], max_frames: int) -> list[SceneFram
|
|
| 227 |
_unlink_frame_files(frame)
|
| 228 |
for new_index, frame in enumerate(kept):
|
| 229 |
frame.index = new_index
|
|
|
|
|
|
|
| 230 |
return kept
|
| 231 |
|
| 232 |
|
|
|
|
| 182 |
return merged
|
| 183 |
|
| 184 |
|
| 185 |
+
def _absorb_dropped_intervals(
|
| 186 |
+
kept: list[SceneFrame], original_tail_end: float
|
| 187 |
+
) -> None:
|
| 188 |
+
"""After frames are dropped, extend each kept frame's end_seconds so it
|
| 189 |
+
covers up to the next kept frame's start (or the original tail for the
|
| 190 |
+
last). Without this, intermediate kept frames keep their old end and the
|
| 191 |
+
rendered guide displays a misleading range like "0:00 β 0:05" while the
|
| 192 |
+
transcript-alignment step has already attached narration through 0:30.
|
| 193 |
+
"""
|
| 194 |
+
for i, frame in enumerate(kept):
|
| 195 |
+
if i + 1 < len(kept):
|
| 196 |
+
next_start = kept[i + 1].start_seconds
|
| 197 |
+
if frame.end_seconds < next_start:
|
| 198 |
+
frame.end_seconds = next_start
|
| 199 |
+
else:
|
| 200 |
+
if frame.end_seconds < original_tail_end:
|
| 201 |
+
frame.end_seconds = original_tail_end
|
| 202 |
+
|
| 203 |
+
|
| 204 |
def _enforce_min_gap(frames: list[SceneFrame], min_gap_seconds: float) -> list[SceneFrame]:
|
| 205 |
"""Drop frames whose start is closer than min_gap_seconds to the previous kept frame.
|
| 206 |
|
|
|
|
| 220 |
if kept:
|
| 221 |
for new_index, frame in enumerate(kept):
|
| 222 |
frame.index = new_index
|
| 223 |
+
_absorb_dropped_intervals(kept, original_tail_end=frames[-1].end_seconds)
|
| 224 |
return kept
|
| 225 |
|
| 226 |
|
|
|
|
| 246 |
_unlink_frame_files(frame)
|
| 247 |
for new_index, frame in enumerate(kept):
|
| 248 |
frame.index = new_index
|
| 249 |
+
if kept:
|
| 250 |
+
_absorb_dropped_intervals(kept, original_tail_end=frames[-1].end_seconds)
|
| 251 |
return kept
|
| 252 |
|
| 253 |
|
|
@@ -580,7 +580,23 @@
|
|
| 580 |
</div>
|
| 581 |
|
| 582 |
<div class="vgm-section">
|
| 583 |
-
<h3 class="vgm-section__title">3.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 584 |
<div class="vgm-formats">
|
| 585 |
<label class="vgm-fmt">
|
| 586 |
<span class="vgm-fmt__head"><input type="radio" name="format" value="review" checked>Review & edit</span>
|
|
@@ -944,7 +960,7 @@
|
|
| 944 |
if (typeof v === 'string' && v.trim() === '') fd.delete(k);
|
| 945 |
});
|
| 946 |
// Unchecked LLM toggles must explicitly send "false" so the server doesn't apply its default-on.
|
| 947 |
-
['llm_alt_text', 'llm_key_terms', 'llm_math_equations', 'llm_section_title'].forEach(k => {
|
| 948 |
if (!fd.has(k)) fd.set(k, 'false');
|
| 949 |
});
|
| 950 |
if (apiKeyEl) apiKeyEl.value = '';
|
|
|
|
| 580 |
</div>
|
| 581 |
|
| 582 |
<div class="vgm-section">
|
| 583 |
+
<h3 class="vgm-section__title">3. Media extras</h3>
|
| 584 |
+
<p class="vgm-section__intro">
|
| 585 |
+
Local-only enrichments β no API key needed.
|
| 586 |
+
</p>
|
| 587 |
+
<div class="vgm-llm-grid">
|
| 588 |
+
<label class="vgm-llm-toggle">
|
| 589 |
+
<input type="checkbox" name="extract_audio" value="true" checked>
|
| 590 |
+
<span class="vgm-llm-toggle__body">
|
| 591 |
+
<strong>Per-segment audio</strong>
|
| 592 |
+
<small>Slice the lecture's narration into one clip per segment so learners can re-listen. Adds ffmpeg processing time and ~500 KB per clip.</small>
|
| 593 |
+
</span>
|
| 594 |
+
</label>
|
| 595 |
+
</div>
|
| 596 |
+
</div>
|
| 597 |
+
|
| 598 |
+
<div class="vgm-section">
|
| 599 |
+
<h3 class="vgm-section__title">4. Output</h3>
|
| 600 |
<div class="vgm-formats">
|
| 601 |
<label class="vgm-fmt">
|
| 602 |
<span class="vgm-fmt__head"><input type="radio" name="format" value="review" checked>Review & edit</span>
|
|
|
|
| 960 |
if (typeof v === 'string' && v.trim() === '') fd.delete(k);
|
| 961 |
});
|
| 962 |
// Unchecked LLM toggles must explicitly send "false" so the server doesn't apply its default-on.
|
| 963 |
+
['llm_alt_text', 'llm_key_terms', 'llm_math_equations', 'llm_section_title', 'extract_audio'].forEach(k => {
|
| 964 |
if (!fd.has(k)) fd.set(k, 'false');
|
| 965 |
});
|
| 966 |
if (apiKeyEl) apiKeyEl.value = '';
|
|
@@ -189,6 +189,50 @@ def test_min_gap_then_cap_max_frames_invariants(tmp_path):
|
|
| 189 |
_assert_timing_invariants(out, total_end=18.0)
|
| 190 |
|
| 191 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 192 |
def test_cap_max_frames_unlinks_alts_of_dropped(tmp_path):
|
| 193 |
# Regression guard: when capping, alternate frame files for dropped
|
| 194 |
# primaries must also be cleaned up β otherwise we leak disk on long
|
|
|
|
| 189 |
_assert_timing_invariants(out, total_end=18.0)
|
| 190 |
|
| 191 |
|
| 192 |
+
def test_min_gap_extends_intermediate_ends_to_cover_dropped_intervals(tmp_path):
|
| 193 |
+
# Regression: when min-gap drops scenes between two kept frames, the
|
| 194 |
+
# earlier kept frame's end_seconds was left at its original value (e.g.
|
| 195 |
+
# 0:05) while the next kept frame's start was much later (0:30). The
|
| 196 |
+
# transcript-alignment step uses kept starts as boundaries, so narration
|
| 197 |
+
# 0:05 β 0:30 was attached to the first kept frame even though its
|
| 198 |
+
# displayed range read "0:00 β 0:05". Each kept frame's end must now
|
| 199 |
+
# extend to the next kept frame's start.
|
| 200 |
+
paths = [tmp_path / f"f{i}.jpg" for i in range(5)]
|
| 201 |
+
for i, p in enumerate(paths):
|
| 202 |
+
_save_text_like(p, seed=i + 1)
|
| 203 |
+
frames = [
|
| 204 |
+
SceneFrame(0, 0.0, 5.0, paths[0]),
|
| 205 |
+
SceneFrame(1, 6.0, 7.0, paths[1]), # gap=6 < 15 β dropped
|
| 206 |
+
SceneFrame(2, 8.5, 9.0, paths[2]), # gap=8.5 < 15 β dropped
|
| 207 |
+
SceneFrame(3, 30.0, 35.0, paths[3]), # gap=30 β₯ 15 β kept
|
| 208 |
+
SceneFrame(4, 40.0, 60.0, paths[4]), # gap=10 < 15 β dropped
|
| 209 |
+
]
|
| 210 |
+
out = _enforce_min_gap(frames, min_gap_seconds=15.0)
|
| 211 |
+
assert len(out) == 2
|
| 212 |
+
# First kept frame's end now reaches the next kept frame's start.
|
| 213 |
+
assert out[0].end_seconds == 30.0
|
| 214 |
+
# Last kept frame absorbs the original tail.
|
| 215 |
+
assert out[1].end_seconds == 60.0
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
def test_cap_max_frames_extends_intermediate_ends_to_next_kept_start(tmp_path):
|
| 219 |
+
paths = [tmp_path / f"f{i:02d}.jpg" for i in range(10)]
|
| 220 |
+
for i, p in enumerate(paths):
|
| 221 |
+
_save_text_like(p, seed=i + 1)
|
| 222 |
+
# 10 contiguous 1-second frames; cap to 3.
|
| 223 |
+
frames = [SceneFrame(i, float(i), float(i) + 1, paths[i]) for i in range(10)]
|
| 224 |
+
out = _cap_max_frames(frames, max_frames=3)
|
| 225 |
+
assert len(out) == 3
|
| 226 |
+
# Each kept frame's end must reach the next kept frame's start so the
|
| 227 |
+
# rendered range reflects the narration window the transcript actually
|
| 228 |
+
# attached to it.
|
| 229 |
+
for i in range(len(out) - 1):
|
| 230 |
+
assert out[i].end_seconds == out[i + 1].start_seconds, (
|
| 231 |
+
f"frame {i} end {out[i].end_seconds} != next start {out[i+1].start_seconds}"
|
| 232 |
+
)
|
| 233 |
+
assert out[-1].end_seconds == frames[-1].end_seconds
|
| 234 |
+
|
| 235 |
+
|
| 236 |
def test_cap_max_frames_unlinks_alts_of_dropped(tmp_path):
|
| 237 |
# Regression guard: when capping, alternate frame files for dropped
|
| 238 |
# primaries must also be cleaned up β otherwise we leak disk on long
|