Spaces:
Running
Per-segment narration audio (Feature 5)
Browse filesffmpeg slices each kept segment's [start, end) window from the source
video into a 32 kbps mono MP3 (~500 KB per 2-min segment). Below 1.5s
segments are skipped — too short to be useful. Failures are
non-fatal: a segment without audio just doesn't render an <audio>
element. ffmpeg-not-on-PATH skips the whole stage gracefully.
Plumbing:
app/pipeline/audio.py — slice_segment / slice_all helpers (subprocess
ffmpeg, libmp3lame, 22 kHz mono, bounded
timeout, stderr captured for warnings).
PipelineInputs.extract_audio (default True) gates the stage.
PipelineResult.audio_paths carries Path|None per segment.
Segment.audio_filename / audio_data_uri reach the templates.
Render paths:
- zip: audio/ folder added to the bundle; templates use 'audio/{fn}'.
- single: audio_data_uri inlined as base64 alongside images.
- review: lazy-fetched via /audio/{job_id}/{filename} (mirrors the
alts pattern). Editor download/preview fetches every URL,
converts to a base64 data URI, and rewrites <audio src>
so the published file plays offline after the server
sleeps. Status pill shows progress during conversion.
Endpoint /audio/{job_id}/{filename} validates filename against
^segment_\d{4}\.mp3$ and reads from Job.audio_bytes (snapshotted
in the worker before workdir cleanup, same lifecycle as alt frames).
The progress UI gets a new 'audio' stage pill between llm and render.
- app/jobs.py +2 -0
- app/main.py +52 -2
- app/pipeline/audio.py +102 -0
- app/pipeline/html_gen.py +11 -1
- app/pipeline/orchestrator.py +28 -1
- app/templates/_doc_styles.html +9 -0
- app/templates/guide.html +9 -0
- app/templates/index.html +2 -1
- app/templates/review.html +44 -2
|
@@ -41,6 +41,8 @@ class Job:
|
|
| 41 |
# before workdir cleanup so the editor can lazy-fetch them via /alts/...
|
| 42 |
# without us having to keep the whole workdir alive.
|
| 43 |
alt_image_bytes: dict[str, bytes] = field(default_factory=dict)
|
|
|
|
|
|
|
| 44 |
|
| 45 |
|
| 46 |
class JobRegistry:
|
|
|
|
| 41 |
# before workdir cleanup so the editor can lazy-fetch them via /alts/...
|
| 42 |
# without us having to keep the whole workdir alive.
|
| 43 |
alt_image_bytes: dict[str, bytes] = field(default_factory=dict)
|
| 44 |
+
# Per-segment narration audio MP3 bytes, served via /audio/{id}/{fn}.
|
| 45 |
+
audio_bytes: dict[str, bytes] = field(default_factory=dict)
|
| 46 |
|
| 47 |
|
| 48 |
class JobRegistry:
|
|
@@ -72,8 +72,11 @@ def _render_to_result(
|
|
| 72 |
meta_lines: list[str] | None,
|
| 73 |
segments: list,
|
| 74 |
image_paths: list[Path],
|
|
|
|
| 75 |
alts_url_template: str | None = None,
|
|
|
|
| 76 |
) -> JobResult:
|
|
|
|
| 77 |
common = dict(
|
| 78 |
title=title,
|
| 79 |
segments=segments,
|
|
@@ -89,10 +92,24 @@ def _render_to_result(
|
|
| 89 |
if alts_url_template:
|
| 90 |
for seg in segments:
|
| 91 |
seg.alt_image_data_uris = []
|
| 92 |
-
html = render_review(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 93 |
return JobResult(body=html.encode("utf-8"), media_type="text/html; charset=utf-8")
|
| 94 |
|
| 95 |
inline = format == "single"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 96 |
html = render_guide(inline_images=inline, **common)
|
| 97 |
|
| 98 |
if inline:
|
|
@@ -102,7 +119,8 @@ def _render_to_result(
|
|
| 102 |
media_type="text/html; charset=utf-8",
|
| 103 |
filename=filename,
|
| 104 |
)
|
| 105 |
-
|
|
|
|
| 106 |
filename = _safe_filename(title) + ".zip"
|
| 107 |
return JobResult(body=zip_bytes, media_type="application/zip", filename=filename)
|
| 108 |
|
|
@@ -137,6 +155,15 @@ def _run_job_in_thread(
|
|
| 137 |
job.alt_image_bytes[p.name] = p.read_bytes()
|
| 138 |
except OSError:
|
| 139 |
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 140 |
job_result = _render_to_result(
|
| 141 |
format=format,
|
| 142 |
title=title,
|
|
@@ -147,9 +174,13 @@ def _run_job_in_thread(
|
|
| 147 |
meta_lines=result.page.meta_lines,
|
| 148 |
segments=result.segments,
|
| 149 |
image_paths=[f.image_path for f in result.kept_frames],
|
|
|
|
| 150 |
alts_url_template=(
|
| 151 |
f"/alts/{job.id}/{{filename}}" if format == "review" else None
|
| 152 |
),
|
|
|
|
|
|
|
|
|
|
| 153 |
)
|
| 154 |
registry.finish_success(job, job_result)
|
| 155 |
except PipelineError as exc:
|
|
@@ -297,6 +328,24 @@ async def fetch_result(job_id: str) -> Response:
|
|
| 297 |
|
| 298 |
|
| 299 |
_ALT_FILENAME_RE = __import__("re").compile(r"^scene_\d{4}_alt\d+\.(jpg|jpeg|png)$")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 300 |
|
| 301 |
|
| 302 |
@app.get("/alts/{job_id}/{filename}")
|
|
@@ -395,6 +444,7 @@ async def process(
|
|
| 395 |
meta_lines=result.page.meta_lines,
|
| 396 |
segments=result.segments,
|
| 397 |
image_paths=[f.image_path for f in result.kept_frames],
|
|
|
|
| 398 |
)
|
| 399 |
headers: dict[str, str] = {}
|
| 400 |
if job_result.filename:
|
|
|
|
| 72 |
meta_lines: list[str] | None,
|
| 73 |
segments: list,
|
| 74 |
image_paths: list[Path],
|
| 75 |
+
audio_paths: list[Path | None] | None = None,
|
| 76 |
alts_url_template: str | None = None,
|
| 77 |
+
audio_url_template: str | None = None,
|
| 78 |
) -> JobResult:
|
| 79 |
+
audio_paths = audio_paths or [None] * len(segments)
|
| 80 |
common = dict(
|
| 81 |
title=title,
|
| 82 |
segments=segments,
|
|
|
|
| 92 |
if alts_url_template:
|
| 93 |
for seg in segments:
|
| 94 |
seg.alt_image_data_uris = []
|
| 95 |
+
html = render_review(
|
| 96 |
+
alts_url_template=alts_url_template,
|
| 97 |
+
audio_url_template=audio_url_template,
|
| 98 |
+
**common,
|
| 99 |
+
)
|
| 100 |
return JobResult(body=html.encode("utf-8"), media_type="text/html; charset=utf-8")
|
| 101 |
|
| 102 |
inline = format == "single"
|
| 103 |
+
if inline:
|
| 104 |
+
# Inline audio data URIs alongside images so the single HTML stays
|
| 105 |
+
# self-contained when downloaded.
|
| 106 |
+
for seg, ap in zip(segments, audio_paths):
|
| 107 |
+
if ap and ap.exists():
|
| 108 |
+
import base64
|
| 109 |
+
seg.audio_data_uri = (
|
| 110 |
+
"data:audio/mpeg;base64,"
|
| 111 |
+
+ base64.b64encode(ap.read_bytes()).decode("ascii")
|
| 112 |
+
)
|
| 113 |
html = render_guide(inline_images=inline, **common)
|
| 114 |
|
| 115 |
if inline:
|
|
|
|
| 119 |
media_type="text/html; charset=utf-8",
|
| 120 |
filename=filename,
|
| 121 |
)
|
| 122 |
+
audio_disk_paths = [p for p in audio_paths if p is not None]
|
| 123 |
+
zip_bytes = bundle_zip(html, image_paths, audio_paths=audio_disk_paths)
|
| 124 |
filename = _safe_filename(title) + ".zip"
|
| 125 |
return JobResult(body=zip_bytes, media_type="application/zip", filename=filename)
|
| 126 |
|
|
|
|
| 155 |
job.alt_image_bytes[p.name] = p.read_bytes()
|
| 156 |
except OSError:
|
| 157 |
pass
|
| 158 |
+
# Same pattern for sliced segment audio (served via /audio/{id}/{fn}).
|
| 159 |
+
for seg in result.segments:
|
| 160 |
+
if not seg.audio_filename:
|
| 161 |
+
continue
|
| 162 |
+
ap = workdir / "audio" / seg.audio_filename
|
| 163 |
+
try:
|
| 164 |
+
job.audio_bytes[seg.audio_filename] = ap.read_bytes()
|
| 165 |
+
except OSError:
|
| 166 |
+
pass
|
| 167 |
job_result = _render_to_result(
|
| 168 |
format=format,
|
| 169 |
title=title,
|
|
|
|
| 174 |
meta_lines=result.page.meta_lines,
|
| 175 |
segments=result.segments,
|
| 176 |
image_paths=[f.image_path for f in result.kept_frames],
|
| 177 |
+
audio_paths=result.audio_paths,
|
| 178 |
alts_url_template=(
|
| 179 |
f"/alts/{job.id}/{{filename}}" if format == "review" else None
|
| 180 |
),
|
| 181 |
+
audio_url_template=(
|
| 182 |
+
f"/audio/{job.id}/{{filename}}" if format == "review" else None
|
| 183 |
+
),
|
| 184 |
)
|
| 185 |
registry.finish_success(job, job_result)
|
| 186 |
except PipelineError as exc:
|
|
|
|
| 328 |
|
| 329 |
|
| 330 |
_ALT_FILENAME_RE = __import__("re").compile(r"^scene_\d{4}_alt\d+\.(jpg|jpeg|png)$")
|
| 331 |
+
_AUDIO_FILENAME_RE = __import__("re").compile(r"^segment_\d{4}\.mp3$")
|
| 332 |
+
|
| 333 |
+
|
| 334 |
+
@app.get("/audio/{job_id}/{filename}")
|
| 335 |
+
async def fetch_audio(job_id: str, filename: str) -> Response:
|
| 336 |
+
if not _AUDIO_FILENAME_RE.match(filename):
|
| 337 |
+
raise HTTPException(404, "Unknown audio.")
|
| 338 |
+
job = registry.get(job_id)
|
| 339 |
+
if not job:
|
| 340 |
+
raise HTTPException(404, "Unknown job id.")
|
| 341 |
+
data = job.audio_bytes.get(filename)
|
| 342 |
+
if data is None:
|
| 343 |
+
raise HTTPException(404, "Audio not found.")
|
| 344 |
+
return Response(
|
| 345 |
+
content=data,
|
| 346 |
+
media_type="audio/mpeg",
|
| 347 |
+
headers={"Cache-Control": "private, max-age=600"},
|
| 348 |
+
)
|
| 349 |
|
| 350 |
|
| 351 |
@app.get("/alts/{job_id}/{filename}")
|
|
|
|
| 444 |
meta_lines=result.page.meta_lines,
|
| 445 |
segments=result.segments,
|
| 446 |
image_paths=[f.image_path for f in result.kept_frames],
|
| 447 |
+
audio_paths=result.audio_paths,
|
| 448 |
)
|
| 449 |
headers: dict[str, str] = {}
|
| 450 |
if job_result.filename:
|
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Slice narration audio per segment via ffmpeg.
|
| 2 |
+
|
| 3 |
+
We extract a low-bitrate mono MP3 covering each kept segment's [start, end)
|
| 4 |
+
window. The clips give learners and accessibility-tool users a way to listen
|
| 5 |
+
to the lecturer for a specific slide without re-cueing the source video.
|
| 6 |
+
|
| 7 |
+
Single ffmpeg invocation per segment using stream copy is impossible (we need
|
| 8 |
+
a bounded slice that lands at requested timestamps), so we re-encode to MP3
|
| 9 |
+
at a low bitrate suited to speech. ~32 kbps mono keeps a 2-minute clip near
|
| 10 |
+
500 KB and is intelligible for narration.
|
| 11 |
+
|
| 12 |
+
Failures here are non-fatal — the pipeline ignores audio errors and the guide
|
| 13 |
+
just doesn't get an audio element for the affected segments.
|
| 14 |
+
"""
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
import logging
|
| 18 |
+
import shutil
|
| 19 |
+
import subprocess
|
| 20 |
+
from pathlib import Path
|
| 21 |
+
|
| 22 |
+
log = logging.getLogger("videoguidemaker.audio")
|
| 23 |
+
|
| 24 |
+
DEFAULT_BITRATE = "32k"
|
| 25 |
+
DEFAULT_SAMPLE_RATE = 22050
|
| 26 |
+
MIN_SEGMENT_SECONDS = 1.5 # below this, audio is rarely useful
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def ffmpeg_available() -> bool:
|
| 30 |
+
return shutil.which("ffmpeg") is not None
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def slice_segment(
|
| 34 |
+
video_path: Path,
|
| 35 |
+
start_seconds: float,
|
| 36 |
+
end_seconds: float,
|
| 37 |
+
dest: Path,
|
| 38 |
+
*,
|
| 39 |
+
bitrate: str = DEFAULT_BITRATE,
|
| 40 |
+
sample_rate: int = DEFAULT_SAMPLE_RATE,
|
| 41 |
+
) -> bool:
|
| 42 |
+
"""Extract [start, end) audio from `video_path` to `dest` as mono MP3.
|
| 43 |
+
|
| 44 |
+
Returns True on success. Logs and returns False on any ffmpeg failure;
|
| 45 |
+
callers must treat audio as best-effort.
|
| 46 |
+
"""
|
| 47 |
+
duration = max(end_seconds - start_seconds, 0.0)
|
| 48 |
+
if duration < MIN_SEGMENT_SECONDS:
|
| 49 |
+
return False
|
| 50 |
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
| 51 |
+
cmd = [
|
| 52 |
+
"ffmpeg",
|
| 53 |
+
"-loglevel", "error",
|
| 54 |
+
"-y",
|
| 55 |
+
"-ss", f"{max(start_seconds, 0):.3f}",
|
| 56 |
+
"-t", f"{duration:.3f}",
|
| 57 |
+
"-i", str(video_path),
|
| 58 |
+
"-vn",
|
| 59 |
+
"-ac", "1",
|
| 60 |
+
"-ar", str(sample_rate),
|
| 61 |
+
"-b:a", bitrate,
|
| 62 |
+
"-codec:a", "libmp3lame",
|
| 63 |
+
str(dest),
|
| 64 |
+
]
|
| 65 |
+
try:
|
| 66 |
+
result = subprocess.run(
|
| 67 |
+
cmd,
|
| 68 |
+
check=False,
|
| 69 |
+
capture_output=True,
|
| 70 |
+
timeout=60,
|
| 71 |
+
)
|
| 72 |
+
except (OSError, subprocess.TimeoutExpired) as exc:
|
| 73 |
+
log.warning("ffmpeg failed for %.1f-%.1f: %s", start_seconds, end_seconds, exc)
|
| 74 |
+
return False
|
| 75 |
+
if result.returncode != 0:
|
| 76 |
+
log.warning(
|
| 77 |
+
"ffmpeg returned %s for %.1f-%.1f: %s",
|
| 78 |
+
result.returncode, start_seconds, end_seconds,
|
| 79 |
+
(result.stderr or b"").decode("utf-8", "replace").strip()[:300],
|
| 80 |
+
)
|
| 81 |
+
return False
|
| 82 |
+
return dest.exists() and dest.stat().st_size > 0
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def slice_all(
|
| 86 |
+
video_path: Path,
|
| 87 |
+
segments: list[tuple[int, float, float]],
|
| 88 |
+
out_dir: Path,
|
| 89 |
+
) -> dict[int, Path]:
|
| 90 |
+
"""Slice audio for every (segment_index, start, end). Returns the index→path
|
| 91 |
+
map for segments where the slice actually wrote a non-empty file.
|
| 92 |
+
"""
|
| 93 |
+
results: dict[int, Path] = {}
|
| 94 |
+
if not ffmpeg_available():
|
| 95 |
+
log.info("ffmpeg not on PATH; skipping audio slicing.")
|
| 96 |
+
return results
|
| 97 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 98 |
+
for idx, start, end in segments:
|
| 99 |
+
dest = out_dir / f"segment_{idx:04d}.mp3"
|
| 100 |
+
if slice_segment(video_path, start, end, dest):
|
| 101 |
+
results[idx] = dest
|
| 102 |
+
return results
|
|
@@ -34,6 +34,9 @@ class Segment:
|
|
| 34 |
# scenes.detect_and_extract sampled multiple candidates.
|
| 35 |
alt_image_filenames: list[str] = field(default_factory=list)
|
| 36 |
alt_image_data_uris: list[str] = field(default_factory=list)
|
|
|
|
|
|
|
|
|
|
| 37 |
|
| 38 |
|
| 39 |
def _format_timestamp(seconds: float) -> str:
|
|
@@ -176,6 +179,7 @@ def render_review(
|
|
| 176 |
mathjax: bool = True,
|
| 177 |
mathjax_src: str = MATHJAX_CDN_URL,
|
| 178 |
alts_url_template: str | None = None,
|
|
|
|
| 179 |
) -> str:
|
| 180 |
"""Render an in-browser editor page. The primary image must be inlined
|
| 181 |
(so save/reopen still shows the picked frame), but alternates can be
|
|
@@ -196,6 +200,7 @@ def render_review(
|
|
| 196 |
mathjax=mathjax,
|
| 197 |
mathjax_src=mathjax_src,
|
| 198 |
alts_url_template=alts_url_template,
|
|
|
|
| 199 |
)
|
| 200 |
|
| 201 |
|
|
@@ -270,12 +275,17 @@ def bundle_zip(
|
|
| 270 |
*,
|
| 271 |
html_filename: str = "study-guide.html",
|
| 272 |
image_dir_name: str = "static",
|
|
|
|
|
|
|
| 273 |
) -> bytes:
|
| 274 |
-
"""Package HTML + images folder
|
| 275 |
buf = io.BytesIO()
|
| 276 |
with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
|
| 277 |
zf.writestr(html_filename, html)
|
| 278 |
for image_path in frame_image_paths:
|
| 279 |
if image_path.exists():
|
| 280 |
zf.write(image_path, arcname=f"{image_dir_name}/{image_path.name}")
|
|
|
|
|
|
|
|
|
|
| 281 |
return buf.getvalue()
|
|
|
|
| 34 |
# scenes.detect_and_extract sampled multiple candidates.
|
| 35 |
alt_image_filenames: list[str] = field(default_factory=list)
|
| 36 |
alt_image_data_uris: list[str] = field(default_factory=list)
|
| 37 |
+
# Optional sliced narration audio for this segment.
|
| 38 |
+
audio_filename: str = ""
|
| 39 |
+
audio_data_uri: str | None = None
|
| 40 |
|
| 41 |
|
| 42 |
def _format_timestamp(seconds: float) -> str:
|
|
|
|
| 179 |
mathjax: bool = True,
|
| 180 |
mathjax_src: str = MATHJAX_CDN_URL,
|
| 181 |
alts_url_template: str | None = None,
|
| 182 |
+
audio_url_template: str | None = None,
|
| 183 |
) -> str:
|
| 184 |
"""Render an in-browser editor page. The primary image must be inlined
|
| 185 |
(so save/reopen still shows the picked frame), but alternates can be
|
|
|
|
| 200 |
mathjax=mathjax,
|
| 201 |
mathjax_src=mathjax_src,
|
| 202 |
alts_url_template=alts_url_template,
|
| 203 |
+
audio_url_template=audio_url_template,
|
| 204 |
)
|
| 205 |
|
| 206 |
|
|
|
|
| 275 |
*,
|
| 276 |
html_filename: str = "study-guide.html",
|
| 277 |
image_dir_name: str = "static",
|
| 278 |
+
audio_paths: list[Path] | None = None,
|
| 279 |
+
audio_dir_name: str = "audio",
|
| 280 |
) -> bytes:
|
| 281 |
+
"""Package HTML + images folder (and optional audio folder) into a zip."""
|
| 282 |
buf = io.BytesIO()
|
| 283 |
with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
|
| 284 |
zf.writestr(html_filename, html)
|
| 285 |
for image_path in frame_image_paths:
|
| 286 |
if image_path.exists():
|
| 287 |
zf.write(image_path, arcname=f"{image_dir_name}/{image_path.name}")
|
| 288 |
+
for ap in audio_paths or []:
|
| 289 |
+
if ap.exists():
|
| 290 |
+
zf.write(ap, arcname=f"{audio_dir_name}/{ap.name}")
|
| 291 |
return buf.getvalue()
|
|
@@ -12,6 +12,7 @@ from dataclasses import dataclass, field
|
|
| 12 |
from pathlib import Path
|
| 13 |
from typing import Callable
|
| 14 |
|
|
|
|
| 15 |
from .filter import filter_instructor_frames
|
| 16 |
from .llm_extract import LLMToggles, extract_all, is_enabled as llm_enabled
|
| 17 |
from .metadata import PageMetadata, SourceInfo, page_metadata_from_segments
|
|
@@ -55,6 +56,8 @@ class PipelineInputs:
|
|
| 55 |
# Per-job credential for LLM extraction; never logged or persisted.
|
| 56 |
anthropic_api_key: str | None = field(default=None, repr=False)
|
| 57 |
llm_toggles: LLMToggles = field(default_factory=LLMToggles)
|
|
|
|
|
|
|
| 58 |
|
| 59 |
|
| 60 |
@dataclass
|
|
@@ -62,6 +65,8 @@ class PipelineResult:
|
|
| 62 |
page: PageMetadata
|
| 63 |
segments: list[Segment]
|
| 64 |
kept_frames: list[SceneFrame]
|
|
|
|
|
|
|
| 65 |
|
| 66 |
|
| 67 |
class PipelineError(RuntimeError):
|
|
@@ -172,6 +177,17 @@ def run_pipeline(
|
|
| 172 |
else:
|
| 173 |
extractions = None
|
| 174 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 175 |
_emit(96, "render", "Building segments...")
|
| 176 |
segments = build_segments(kept, narrations, ocr_texts, inline_images=inputs.inline_images)
|
| 177 |
if extractions is not None:
|
|
@@ -184,6 +200,10 @@ def run_pipeline(
|
|
| 184 |
seg.alt_draft = extracted.alt_text
|
| 185 |
if inputs.llm_toggles.section_title and extracted.section_title:
|
| 186 |
seg.section_title = extracted.section_title
|
|
|
|
|
|
|
|
|
|
|
|
|
| 187 |
|
| 188 |
page = page_metadata_from_segments(
|
| 189 |
title=inputs.title,
|
|
@@ -200,5 +220,12 @@ def run_pipeline(
|
|
| 200 |
),
|
| 201 |
frames_dir=str(inputs.frames_dir.name),
|
| 202 |
)
|
|
|
|
|
|
|
| 203 |
_emit(100, "done", f"Done — {len(segments)} segments.")
|
| 204 |
-
return PipelineResult(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
from pathlib import Path
|
| 13 |
from typing import Callable
|
| 14 |
|
| 15 |
+
from .audio import slice_all as slice_segment_audio, ffmpeg_available
|
| 16 |
from .filter import filter_instructor_frames
|
| 17 |
from .llm_extract import LLMToggles, extract_all, is_enabled as llm_enabled
|
| 18 |
from .metadata import PageMetadata, SourceInfo, page_metadata_from_segments
|
|
|
|
| 56 |
# Per-job credential for LLM extraction; never logged or persisted.
|
| 57 |
anthropic_api_key: str | None = field(default=None, repr=False)
|
| 58 |
llm_toggles: LLMToggles = field(default_factory=LLMToggles)
|
| 59 |
+
# Slice narration audio per segment (one mono MP3 per kept segment).
|
| 60 |
+
extract_audio: bool = True
|
| 61 |
|
| 62 |
|
| 63 |
@dataclass
|
|
|
|
| 65 |
page: PageMetadata
|
| 66 |
segments: list[Segment]
|
| 67 |
kept_frames: list[SceneFrame]
|
| 68 |
+
# Sliced narration audio paths, in segment order. None when missing.
|
| 69 |
+
audio_paths: list[Path | None] = field(default_factory=list)
|
| 70 |
|
| 71 |
|
| 72 |
class PipelineError(RuntimeError):
|
|
|
|
| 177 |
else:
|
| 178 |
extractions = None
|
| 179 |
|
| 180 |
+
audio_paths: dict[int, Path] = {}
|
| 181 |
+
if inputs.extract_audio and ffmpeg_available():
|
| 182 |
+
_emit(95, "audio", f"Slicing narration audio ({len(kept)} segments)...")
|
| 183 |
+
audio_dir = inputs.frames_dir.parent / "audio"
|
| 184 |
+
audio_paths = slice_segment_audio(
|
| 185 |
+
inputs.video_path,
|
| 186 |
+
[(i, f.start_seconds, f.end_seconds) for i, f in enumerate(kept)],
|
| 187 |
+
audio_dir,
|
| 188 |
+
)
|
| 189 |
+
log.info("Sliced %d/%d narration audio clips", len(audio_paths), len(kept))
|
| 190 |
+
|
| 191 |
_emit(96, "render", "Building segments...")
|
| 192 |
segments = build_segments(kept, narrations, ocr_texts, inline_images=inputs.inline_images)
|
| 193 |
if extractions is not None:
|
|
|
|
| 200 |
seg.alt_draft = extracted.alt_text
|
| 201 |
if inputs.llm_toggles.section_title and extracted.section_title:
|
| 202 |
seg.section_title = extracted.section_title
|
| 203 |
+
for i, seg in enumerate(segments):
|
| 204 |
+
ap = audio_paths.get(i)
|
| 205 |
+
if ap is not None:
|
| 206 |
+
seg.audio_filename = ap.name
|
| 207 |
|
| 208 |
page = page_metadata_from_segments(
|
| 209 |
title=inputs.title,
|
|
|
|
| 220 |
),
|
| 221 |
frames_dir=str(inputs.frames_dir.name),
|
| 222 |
)
|
| 223 |
+
audio_paths_ordered: list[Path | None] = [audio_paths.get(i) for i in range(len(kept))]
|
| 224 |
+
|
| 225 |
_emit(100, "done", f"Done — {len(segments)} segments.")
|
| 226 |
+
return PipelineResult(
|
| 227 |
+
page=page,
|
| 228 |
+
segments=segments,
|
| 229 |
+
kept_frames=kept,
|
| 230 |
+
audio_paths=audio_paths_ordered,
|
| 231 |
+
)
|
|
@@ -357,6 +357,15 @@
|
|
| 357 |
.toc { margin: var(--space-md); padding: var(--space-md); }
|
| 358 |
}
|
| 359 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 360 |
/* ========== VIEW MODE TOGGLE ========== */
|
| 361 |
.vgm-view-toggle {
|
| 362 |
display: flex;
|
|
|
|
| 357 |
.toc { margin: var(--space-md); padding: var(--space-md); }
|
| 358 |
}
|
| 359 |
|
| 360 |
+
/* ========== NARRATION AUDIO ========== */
|
| 361 |
+
.narration__audio {
|
| 362 |
+
display: block;
|
| 363 |
+
width: 100%;
|
| 364 |
+
max-width: 28rem;
|
| 365 |
+
margin-top: var(--space-sm);
|
| 366 |
+
height: 32px;
|
| 367 |
+
}
|
| 368 |
+
|
| 369 |
/* ========== VIEW MODE TOGGLE ========== */
|
| 370 |
.vgm-view-toggle {
|
| 371 |
display: flex;
|
|
@@ -88,6 +88,15 @@
|
|
| 88 |
<div class="narration__body">
|
| 89 |
{% for para in seg.narration %}<p>{{ para }}</p>{% endfor %}
|
| 90 |
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 91 |
</div>
|
| 92 |
{% endif %}
|
| 93 |
|
|
|
|
| 88 |
<div class="narration__body">
|
| 89 |
{% for para in seg.narration %}<p>{{ para }}</p>{% endfor %}
|
| 90 |
</div>
|
| 91 |
+
{% if seg.audio_data_uri %}
|
| 92 |
+
<audio class="narration__audio" controls preload="none"
|
| 93 |
+
src="{{ seg.audio_data_uri }}"
|
| 94 |
+
aria-label="Narration audio for this segment"></audio>
|
| 95 |
+
{% elif seg.audio_filename %}
|
| 96 |
+
<audio class="narration__audio" controls preload="none"
|
| 97 |
+
src="audio/{{ seg.audio_filename }}"
|
| 98 |
+
aria-label="Narration audio for this segment"></audio>
|
| 99 |
+
{% endif %}
|
| 100 |
</div>
|
| 101 |
{% endif %}
|
| 102 |
|
|
@@ -486,6 +486,7 @@
|
|
| 486 |
<span data-stage="transcript">transcript</span>
|
| 487 |
<span data-stage="ocr">ocr</span>
|
| 488 |
<span data-stage="llm_extract">llm</span>
|
|
|
|
| 489 |
<span data-stage="render">render</span>
|
| 490 |
</div>
|
| 491 |
<pre class="vgm-progress__details" aria-label="Error details"></pre>
|
|
@@ -746,7 +747,7 @@
|
|
| 746 |
const reviewOpenBtn = reviewEl.querySelector('.vgm-review__open');
|
| 747 |
let lastResultUrl = null;
|
| 748 |
|
| 749 |
-
const STAGE_ORDER = ['uploading', 'scene_detect', 'filter', 'transcript', 'ocr', 'llm_extract', 'render'];
|
| 750 |
function setStagePill(stage) {
|
| 751 |
const idx = STAGE_ORDER.indexOf(stage);
|
| 752 |
stagePillsEl.querySelectorAll('span').forEach(el => {
|
|
|
|
| 486 |
<span data-stage="transcript">transcript</span>
|
| 487 |
<span data-stage="ocr">ocr</span>
|
| 488 |
<span data-stage="llm_extract">llm</span>
|
| 489 |
+
<span data-stage="audio">audio</span>
|
| 490 |
<span data-stage="render">render</span>
|
| 491 |
</div>
|
| 492 |
<pre class="vgm-progress__details" aria-label="Error details"></pre>
|
|
|
|
| 747 |
const reviewOpenBtn = reviewEl.querySelector('.vgm-review__open');
|
| 748 |
let lastResultUrl = null;
|
| 749 |
|
| 750 |
+
const STAGE_ORDER = ['uploading', 'scene_detect', 'filter', 'transcript', 'ocr', 'llm_extract', 'audio', 'render'];
|
| 751 |
function setStagePill(stage) {
|
| 752 |
const idx = STAGE_ORDER.indexOf(stage);
|
| 753 |
stagePillsEl.querySelectorAll('span').forEach(el => {
|
|
@@ -339,6 +339,19 @@
|
|
| 339 |
<div class="narration__body" contenteditable="true" data-bind="narration" spellcheck="true">
|
| 340 |
{% for para in seg.narration %}<p>{{ para }}</p>{% endfor %}
|
| 341 |
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 342 |
</div>
|
| 343 |
|
| 344 |
<details class="onscreen-text">
|
|
@@ -929,7 +942,34 @@
|
|
| 929 |
return '<!DOCTYPE html>\n' + clone.outerHTML;
|
| 930 |
}
|
| 931 |
|
| 932 |
-
function
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 933 |
const html = buildFinalHTML();
|
| 934 |
const blob = new Blob([html], { type: 'text/html;charset=utf-8' });
|
| 935 |
const url = URL.createObjectURL(blob);
|
|
@@ -948,7 +988,9 @@
|
|
| 948 |
previewURL = null;
|
| 949 |
}
|
| 950 |
});
|
| 951 |
-
function previewHTML() {
|
|
|
|
|
|
|
| 952 |
const html = buildFinalHTML();
|
| 953 |
// Use a Blob URL — opening an empty window and document.write'ing into
|
| 954 |
// it from inside an iframe lands as a blank tab in some browsers
|
|
|
|
| 339 |
<div class="narration__body" contenteditable="true" data-bind="narration" spellcheck="true">
|
| 340 |
{% for para in seg.narration %}<p>{{ para }}</p>{% endfor %}
|
| 341 |
</div>
|
| 342 |
+
{% if seg.audio_filename and audio_url_template %}
|
| 343 |
+
<audio class="narration__audio" controls preload="none"
|
| 344 |
+
data-bind="audio"
|
| 345 |
+
data-audio-url="{{ audio_url_template.replace('{filename}', seg.audio_filename) }}"
|
| 346 |
+
data-audio-filename="{{ seg.audio_filename }}"
|
| 347 |
+
src="{{ audio_url_template.replace('{filename}', seg.audio_filename) }}"
|
| 348 |
+
aria-label="Narration audio for this segment"></audio>
|
| 349 |
+
{% elif seg.audio_data_uri %}
|
| 350 |
+
<audio class="narration__audio" controls preload="none"
|
| 351 |
+
data-bind="audio"
|
| 352 |
+
src="{{ seg.audio_data_uri }}"
|
| 353 |
+
aria-label="Narration audio for this segment"></audio>
|
| 354 |
+
{% endif %}
|
| 355 |
</div>
|
| 356 |
|
| 357 |
<details class="onscreen-text">
|
|
|
|
| 942 |
return '<!DOCTYPE html>\n' + clone.outerHTML;
|
| 943 |
}
|
| 944 |
|
| 945 |
+
async function inlineAllAudio() {
|
| 946 |
+
// Convert each lazy <audio data-audio-url=...> to a base64 data URI
|
| 947 |
+
// so the downloaded/previewed HTML keeps playing after the server
|
| 948 |
+
// sleeps. Done on the live DOM (so the data URI is captured by the
|
| 949 |
+
// next buildFinalHTML clone). Skips ones that already have a data URI.
|
| 950 |
+
const audios = Array.from(document.querySelectorAll('audio[data-audio-url]'));
|
| 951 |
+
const targets = audios.filter(a => !(a.getAttribute('src') || '').startsWith('data:'));
|
| 952 |
+
if (!targets.length) return;
|
| 953 |
+
let done = 0;
|
| 954 |
+
for (const audio of targets) {
|
| 955 |
+
const url = audio.getAttribute('data-audio-url');
|
| 956 |
+
try {
|
| 957 |
+
const resp = await fetch(url);
|
| 958 |
+
if (!resp.ok) throw new Error('HTTP ' + resp.status);
|
| 959 |
+
const blob = await resp.blob();
|
| 960 |
+
const dataUri = await blobToDataURI(blob);
|
| 961 |
+
audio.setAttribute('src', dataUri);
|
| 962 |
+
} catch (err) {
|
| 963 |
+
// Leave the URL as-is; downloaded file just won't have THIS clip.
|
| 964 |
+
}
|
| 965 |
+
done++;
|
| 966 |
+
setStatus(`Inlining audio… ${done}/${targets.length}`);
|
| 967 |
+
}
|
| 968 |
+
}
|
| 969 |
+
|
| 970 |
+
async function downloadHTML() {
|
| 971 |
+
setStatus('Preparing download…');
|
| 972 |
+
await inlineAllAudio();
|
| 973 |
const html = buildFinalHTML();
|
| 974 |
const blob = new Blob([html], { type: 'text/html;charset=utf-8' });
|
| 975 |
const url = URL.createObjectURL(blob);
|
|
|
|
| 988 |
previewURL = null;
|
| 989 |
}
|
| 990 |
});
|
| 991 |
+
async function previewHTML() {
|
| 992 |
+
setStatus('Preparing preview…');
|
| 993 |
+
await inlineAllAudio();
|
| 994 |
const html = buildFinalHTML();
|
| 995 |
// Use a Blob URL — opening an empty window and document.write'ing into
|
| 996 |
// it from inside an iframe lands as a blank tab in some browsers
|