Video2Guide / app /main.py
Claude
Claude Opus 4.8
Add AI slide-to-transcript alignment for untimestamped images
e646192
Raw History Blame Contribute Delete
69.9 kB
"""FastAPI app: upload UI + SSE-driven pipeline orchestration.""" from __future__ import annotations import asyncio import io import json import logging import os import queue import secrets import shutil import tempfile import threading import time from pathlib import Path from fastapi import FastAPI, File, Form, HTTPException, Request, UploadFile from fastapi.responses import HTMLResponse, JSONResponse, Response, StreamingResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates from .jobs import Job, JobQuotaExceeded, JobResult, registry from .pipeline import bundle_zip, render_from_metadata, render_guide, render_review from .pipeline.filter import filter_instructor_frames from .pipeline.llm_extract import ( LLMToggles, LLMTopicError, SlideAlignError, align_slides_to_transcript, extract_topics, is_valid_video_type, ) from .pipeline.metadata import CURRENT_SCHEMA_VERSION, PageMetadata, migrate_v1_to_v2 from .pipeline.orchestrator import ( PipelineError, PipelineInputs, ProgressEvent, _apply_topic_segmentation, run_pipeline, ) from .pipeline.image_input import ( ImageInputError, all_have_timestamps, build_frames_even_split, build_frames_from_images, build_frames_from_starts, build_frames_ordered_by_name, order_by_name, ) from .pipeline.scenes import detect_and_extract, video_duration from .pipeline.transcript import parse_transcript from .pipeline.types import SceneFrame logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") log = logging.getLogger("videoguidemaker") BASE_DIR = Path(__file__).resolve().parent templates = Jinja2Templates(directory=str(BASE_DIR / "templates")) app = FastAPI(title="The Describatron") app.mount("/static", StaticFiles(directory=str(BASE_DIR / "static")), name="static") _SLACK_BYTES = 8 * 1024 * 1024 # multipart framing + the small form fields def _request_body_ceiling(path: str) -> int | None: """Hard upper bound on an acceptable request body for `path`, or None to skip the early check. Sized per-endpoint from that endpoint's own limits so a legitimately large images-mode upload isn't wrongly rejected and /edit isn't given the (much larger) video ceiling. Returns None when the relevant cap is disabled (an explicit opt-out of size protection). NB: this only fires when the request declares a Content-Length; a chunked upload has none, so each upload endpoint additionally bounds its own read/stream (_save_upload for files, the chunked read in /edit). """ if path == "/jobs/from-images": if MAX_IMAGE_BYTES is None: return None return MAX_IMAGE_COUNT * MAX_IMAGE_BYTES + (MAX_TRANSCRIPT_BYTES or 0) + _SLACK_BYTES if path == "/edit": if MAX_EDIT_JSON_BYTES is None: return None return MAX_EDIT_JSON_BYTES + _SLACK_BYTES if MAX_VIDEO_BYTES is None: return None return MAX_VIDEO_BYTES + (MAX_TRANSCRIPT_BYTES or 0) + _SLACK_BYTES @app.middleware("http") async def _security_headers(request: Request, call_next): # Reject an over-cap upload from its declared Content-Length BEFORE # Starlette spools the multipart body to a temp file. The per-field # streaming check in _save_upload still runs (and catches an absent or # lying Content-Length), but this stops the obvious "declare a 10 GB # body" case from filling the Space's ephemeral disk first. if request.method in ("POST", "PUT", "PATCH"): ceiling = _request_body_ceiling(request.url.path) declared = request.headers.get("content-length") if ceiling is not None and declared is not None: try: size = int(declared) except ValueError: size = -1 if size > ceiling: return JSONResponse( {"detail": "Upload exceeds the maximum allowed size."}, status_code=413, headers={"X-Content-Type-Options": "nosniff"}, ) response = await call_next(request) # X-Content-Type-Options is safe everywhere — prevents MIME sniffing. response.headers["X-Content-Type-Options"] = "nosniff" # X-Frame-Options and CSP are skipped here: HF Spaces proxies the app # through its own infrastructure and setting these causes "refused to # connect" failures. Add them at the reverse-proxy / CDN layer instead. return response VIDEO_SUFFIXES = {".mp4", ".mov", ".mkv", ".webm", ".m4v", ".avi"} TRANSCRIPT_SUFFIXES = {".vtt", ".srt"} IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".webp"} # Upper bound on how many slide images one images-mode job accepts. Each # becomes a segment; a typical deck is well under this. Guards against an # accidental thousand-file drop. MAX_IMAGE_COUNT = 200 ALLOWED_FORMATS = {"review", "single", "zip"} # Cap upload sizes so a no-auth deployment can't be OOM'd / disk-filled # by an arbitrarily large body. 500 MB covers a ~30-min lecture at typical # bitrates; transcripts are tiny so 5 MB is plenty. # Override via env vars locally: MAX_VIDEO_MB=0 disables the cap entirely. def _size_limit(env_var: str, default_mb: int) -> int | None: mb = int(os.getenv(env_var, str(default_mb))) return None if mb == 0 else mb * 1024 * 1024 MAX_VIDEO_BYTES = _size_limit("MAX_VIDEO_MB", 500) MAX_TRANSCRIPT_BYTES = _size_limit("MAX_TRANSCRIPT_MB", 5) MAX_IMAGE_BYTES = _size_limit("MAX_IMAGE_MB", 25) # per slide image # In-memory cache of completed scene previews: token → {dir, video_path, frames_dir, frames, created} _preview_cache: dict[str, dict] = {} _preview_cache_lock = threading.Lock() _PREVIEW_TTL = 1800 # 30 minutes # Hard cap on cached previews so a flood of preview requests can't pin an # unbounded amount of video data on disk for the full 30-min TTL. When the # cap is hit, the oldest entries are evicted (their temp dirs removed). _MAX_PREVIEW_ENTRIES = 24 def _purge_stale_previews() -> None: cutoff = time.time() - _PREVIEW_TTL with _preview_cache_lock: stale = [t for t, v in list(_preview_cache.items()) if v["created"] < cutoff] evicted = [_preview_cache.pop(t) for t in stale] # Enforce the entry cap after the TTL sweep. Evict oldest-first # until we're back under the limit. if len(_preview_cache) > _MAX_PREVIEW_ENTRIES: by_age = sorted(_preview_cache.items(), key=lambda kv: kv[1].get("created", 0.0)) overflow = len(_preview_cache) - _MAX_PREVIEW_ENTRIES for tok, _ in by_age[:overflow]: evicted.append(_preview_cache.pop(tok)) for entry in evicted: shutil.rmtree(entry["dir"], ignore_errors=True) async def _save_upload( upload: UploadFile, dest: Path, *, max_bytes: int | None, label: str ) -> None: """Stream `upload` to `dest`, aborting (and unlinking) past `max_bytes`.""" size = 0 chunk_size = 1024 * 1024 with dest.open("wb") as f: while True: chunk = await upload.read(chunk_size) if not chunk: break size += len(chunk) if max_bytes is not None and size > max_bytes: try: dest.unlink(missing_ok=True) except OSError: pass limit_mb = max_bytes // (1024 * 1024) raise HTTPException(413, f"{label} exceeds the {limit_mb} MB upload limit.") f.write(chunk) def _validate_format(value: str) -> str: if value not in ALLOWED_FORMATS: raise HTTPException( 400, f"Unsupported format {value!r}. Must be one of: {sorted(ALLOWED_FORMATS)}.", ) return value def _parse_optional_int(value: str | None) -> int | None: if value is None: return None s = value.strip() if not s: return None try: return int(s) except ValueError as exc: raise HTTPException(400, f"Invalid integer: {value!r}") from exc @app.get("/", response_class=HTMLResponse) async def index(request: Request) -> HTMLResponse: return templates.TemplateResponse(request, "index.html", {}) @app.get("/images", response_class=HTMLResponse) async def images_page(request: Request) -> HTMLResponse: return templates.TemplateResponse(request, "images.html", {}) @app.get("/edit", response_class=HTMLResponse) async def edit_page(request: Request) -> HTMLResponse: return templates.TemplateResponse(request, "edit.html", {}) # A self-contained editor export embeds the slide images as base64, so the # JSON can be several MB. Cap generously; well above a normal deck. MAX_EDIT_JSON_BYTES = _size_limit("MAX_EDIT_MB", 150) @app.post("/edit", response_class=HTMLResponse) async def edit_from_json(edits: UploadFile = File(...)) -> HTMLResponse: """Rebuild a full editor from a single self-contained edits JSON. The editor's 'Save edits (JSON)' now embeds each slide as a data URI, so one file carries everything needed to reconstitute the editor — no job, no image files. This lets an author hand the JSON to someone else who opens it here and keeps editing. Returns the editor HTML directly so the browser navigates straight into it. """ # Bounded read: accumulate in ≤1 MB chunks and abort the moment the # body exceeds the cap, instead of buffering the whole (possibly # multi-GB, or Transfer-Encoding: chunked) upload into memory first. limit = MAX_EDIT_JSON_BYTES chunks: list[bytes] = [] total = 0 while True: chunk = await edits.read(1024 * 1024) if not chunk: break total += len(chunk) if limit is not None and total > limit: limit_mb = limit // (1024 * 1024) raise HTTPException(413, f"Edits file exceeds the {limit_mb} MB limit.") chunks.append(chunk) raw = b"".join(chunks) try: data = json.loads(raw.decode("utf-8")) except Exception: raise HTTPException(400, "That file isn't valid JSON. Upload an edits file saved from the editor.") if not isinstance(data, dict): raise HTTPException(400, "The JSON must be an editor export object.") if int(data.get("schema_version", 1) or 1) < CURRENT_SCHEMA_VERSION: data = migrate_v1_to_v2(data) try: page = PageMetadata.model_validate(data) except Exception as exc: raise HTTPException(422, f"Not a valid editor export: {exc}") if not page.segments: raise HTTPException(400, "This export has no segments to edit.") missing = [s.index for s in page.segments if not (s.image_data_uri or "").strip()] if missing: raise HTTPException( 400, "This edits file doesn't embed its slide images, so the editor " "can't be rebuilt from it alone. Re-save it from the editor's " "'Save edits (JSON)' button (which now embeds the slides) and " "upload that file.", ) # Render against a fresh EMPTY temp dir, not the app's working directory. # Every segment carries an inlined image_data_uri, so metadata_to_segment # never reads an image off disk; the empty dir plus basename sanitisation # in metadata_to_segment also means a crafted audio_filename can't be # read (no traversal, and nothing to read in an empty dir). scratch = Path(tempfile.mkdtemp(prefix="vgm_edit_")) try: html = render_from_metadata(page, scratch, mode="review") finally: shutil.rmtree(scratch, ignore_errors=True) return HTMLResponse(html) def _safe_filename(title: str) -> str: safe = "".join(c if c.isalnum() or c in "-_ " else "-" for c in title).strip() safe = safe.replace(" ", "-") return safe or "study-guide" def _content_disposition(filename: str) -> str: """Build a Content-Disposition header value that survives ASGI's latin-1 header encoding for any filename. `_safe_filename` keeps Unicode alphanumerics ("中".isalnum() is True), so a CJK / emoji / accented title reaches here as a non-latin-1 string. Emitting it raw as filename="…" makes Starlette raise UnicodeEncodeError when it encodes the response headers — a 500 that (for one-shot job results) also destroys the already-cleared body, so the retry gets nothing. RFC 6266 solves this: an ASCII-only `filename=` fallback for legacy clients plus a percent-encoded `filename*=UTF-8''…` that modern browsers prefer and that is pure ASCII on the wire. """ from urllib.parse import quote ascii_fallback = filename.encode("ascii", "replace").decode("ascii").replace('"', "'") encoded = quote(filename, safe="") return f"attachment; filename=\"{ascii_fallback}\"; filename*=UTF-8''{encoded}" def _render_to_result( *, format: str, title: str, lang: str, subtitle: str | None, module: str | None, eyebrow: str, meta_lines: list[str] | None, segments: list, image_paths: list[Path], audio_paths: list[Path | None] | None = None, alts_url_template: str | None = None, ocr_url_template: str | None = None, audio_url_template: str | None = None, zip_url: str | None = None, ) -> JobResult: audio_paths = audio_paths or [None] * len(segments) common = dict( title=title, segments=segments, lang=lang, subtitle=subtitle, module=module, meta_lines=meta_lines or None, eyebrow=eyebrow, ) if format == "review": # Don't inline alternates — the editor will lazy-fetch them via # alts_url_template. Keeps the initial review HTML small. if alts_url_template: for seg in segments: seg.alt_image_data_uris = [] html = render_review( alts_url_template=alts_url_template, ocr_url_template=ocr_url_template, audio_url_template=audio_url_template, zip_url=zip_url, **common, ) return JobResult(body=html.encode("utf-8"), media_type="text/html; charset=utf-8") inline = format == "single" if inline: # Inline audio data URIs alongside images so the single HTML stays # self-contained when downloaded. for seg, ap in zip(segments, audio_paths): if ap and ap.exists(): import base64 seg.audio_data_uri = ( "data:audio/mpeg;base64," + base64.b64encode(ap.read_bytes()).decode("ascii") ) html = render_guide(inline_images=inline, **common) if inline: filename = _safe_filename(title) + ".html" return JobResult( body=html.encode("utf-8"), media_type="text/html; charset=utf-8", filename=filename, ) audio_disk_paths = [p for p in audio_paths if p is not None] zip_bytes = bundle_zip(html, image_paths, audio_paths=audio_disk_paths) filename = _safe_filename(title) + ".zip" return JobResult(body=zip_bytes, media_type="application/zip", filename=filename) def _run_job_in_thread( job: Job, inputs: PipelineInputs, workdir: Path, *, format: str, title: str, lang: str, subtitle: str | None, module: str | None, preselected_frames: list[SceneFrame] | None = None, preselected_topic_titles: list[str] | None = None, preselected_topic_summaries: list[str] | None = None, preview_dir: Path | None = None, ) -> None: def progress_cb(ev: ProgressEvent) -> None: registry.emit( job, type="progress", percent=ev.percent, stage=ev.stage, message=ev.message, ) try: result = run_pipeline( inputs, progress=progress_cb, preselected_frames=preselected_frames, preselected_topic_titles=preselected_topic_titles, preselected_topic_summaries=preselected_topic_summaries, ) # Snapshot alt-frame bytes into the Job before workdir cleanup so the # editor can lazy-fetch them via /alts/{job_id}/{filename}. for f in result.kept_frames: for p in f.alt_image_paths: try: job.alt_image_bytes[p.name] = p.read_bytes() except OSError: pass # Same pattern for sliced segment audio (served via /audio/{id}/{fn}). # Read from the orchestrator's resolved paths rather than guessing # the on-disk layout, which has changed across versions. for seg, ap in zip(result.segments, result.audio_paths): if not seg.audio_filename or ap is None: continue try: job.audio_bytes[seg.audio_filename] = ap.read_bytes() except OSError: pass job_result = _render_to_result( format=format, title=title, lang=lang, subtitle=subtitle, module=module, eyebrow=result.page.eyebrow, meta_lines=result.page.meta_lines, segments=result.segments, image_paths=[f.image_path for f in result.kept_frames], audio_paths=result.audio_paths, alts_url_template=( f"/alts/{job.id}/{{filename}}" if format == "review" else None ), ocr_url_template=( f"/ocr/{job.id}/{{filename}}" if format == "review" else None ), audio_url_template=( f"/audio/{job.id}/{{filename}}" if format == "review" else None ), zip_url=f"/jobs/{job.id}/zip" if format == "review" else None, ) registry.finish_success(job, job_result) except PipelineError as exc: registry.finish_error(job, str(exc)) except Exception as exc: # pragma: no cover (defensive) log.exception("Job %s failed unexpectedly", job.id) registry.finish_error(job, f"Unexpected error: {exc}") finally: shutil.rmtree(workdir, ignore_errors=True) if preview_dir: shutil.rmtree(preview_dir, ignore_errors=True) @app.post("/scene-preview") async def scene_preview( video: UploadFile = File(...), threshold: float = Form(27.0), min_gap: float = Form(0.0), max_frames: str | None = Form(None), face_threshold: float = Form(0.12), ) -> JSONResponse: """Detect and filter scenes, return thumbnails for user selection.""" import base64 from PIL import Image video_suffix = Path(video.filename or "").suffix.lower() if video_suffix not in VIDEO_SUFFIXES: raise HTTPException(400, f"Unsupported video format: {video_suffix or '(none)'}") workdir = Path(tempfile.mkdtemp(prefix="vgm_prev_")) try: video_path = workdir / f"input{video_suffix}" frames_dir = workdir / "static" frames_dir.mkdir() await _save_upload(video, video_path, max_bytes=MAX_VIDEO_BYTES, label="Video") loop = asyncio.get_running_loop() max_frames_int = _parse_optional_int(max_frames) frames = await loop.run_in_executor( None, lambda: detect_and_extract( video_path, frames_dir, threshold=threshold, min_gap_seconds=min_gap, max_frames=max_frames_int, ) ) kept, _ = await loop.run_in_executor( None, lambda: filter_instructor_frames(frames, face_area_threshold=face_threshold) ) def _build_thumbnails(frames: list) -> list: out = [] for f in frames: img = Image.open(f.image_path) img.thumbnail((320, 180)) buf = io.BytesIO() img.save(buf, format="JPEG", quality=72) b64 = base64.b64encode(buf.getvalue()).decode() out.append({ "index": f.index, "start_seconds": f.start_seconds, "end_seconds": f.end_seconds, "thumbnail": f"data:image/jpeg;base64,{b64}", "image_filename": f.image_path.name, }) return out result_frames = await loop.run_in_executor(None, lambda: _build_thumbnails(kept)) _purge_stale_previews() token = secrets.token_hex(16) with _preview_cache_lock: _preview_cache[token] = { "dir": workdir, "video_path": video_path, "frames_dir": frames_dir, "frames": kept, "created": time.time(), } return JSONResponse({"preview_token": token, "frames": result_frames}) except Exception: shutil.rmtree(workdir, ignore_errors=True) raise @app.post("/topic-scene-preview") async def topic_scene_preview( video: UploadFile = File(...), transcript: UploadFile = File(...), threshold: float = Form(27.0), min_gap: float = Form(0.0), max_frames: str | None = Form(None), face_threshold: float = Form(0.12), skip_inverted_ocr: bool = Form(False), anthropic_api_key: str | None = Form(None), llm_topic_segmentation_with_images: bool = Form(False), llm_topic_video_type: str = Form("general"), ) -> JSONResponse: """Run the topic-segmentation pipeline up through primary-frame picking, then return one card per topic (title, summary, primary thumbnail) for the user to pick which to include before the expensive LLM extraction + audio + render stages. Caches the post-topic kept frames + parallel title/summary/ocr lists keyed by a token. /jobs accepts that token plus a list of selected topic indices and resumes the pipeline from there. """ import base64 from PIL import Image from .pipeline.ocr import ocr_frame as run_ocr if not anthropic_api_key or not anthropic_api_key.strip(): raise HTTPException( 400, "Topic preview requires an Anthropic API key — Claude segments the transcript.", ) video_suffix = Path(video.filename or "").suffix.lower() if video_suffix not in VIDEO_SUFFIXES: raise HTTPException(400, f"Unsupported video format: {video_suffix or '(none)'}") transcript_suffix = Path(transcript.filename or "").suffix.lower() if transcript_suffix not in TRANSCRIPT_SUFFIXES: raise HTTPException(400, f"Unsupported transcript format: {transcript_suffix or '(none)'}") if not is_valid_video_type(llm_topic_video_type): raise HTTPException( 400, f"Unknown video type '{llm_topic_video_type}'. " "Expected one of: general, lecture, lab, tutorial, interview, talk.", ) workdir = Path(tempfile.mkdtemp(prefix="vgm_topicprev_")) try: video_path = workdir / f"input{video_suffix}" transcript_path = workdir / f"transcript{transcript_suffix}" frames_dir = workdir / "static" frames_dir.mkdir() await _save_upload(video, video_path, max_bytes=MAX_VIDEO_BYTES, label="Video") await _save_upload( transcript, transcript_path, max_bytes=MAX_TRANSCRIPT_BYTES, label="Transcript" ) loop = asyncio.get_running_loop() max_frames_int = _parse_optional_int(max_frames) # Stage 1: scene detect + filter (CPU-heavy; off the event loop). frames = await loop.run_in_executor( None, lambda: detect_and_extract( video_path, frames_dir, threshold=threshold, min_gap_seconds=min_gap, max_frames=max_frames_int, ) ) kept, _ = await loop.run_in_executor( None, lambda: filter_instructor_frames(frames, face_area_threshold=face_threshold) ) if not kept: raise HTTPException( 400, "No usable visual segments were found. Try lowering scene-change sensitivity " "or raising the instructor face threshold in Advanced settings.", ) # Stage 2: transcript parse + full-duration computation. Match # the orchestrator's behaviour so the topic LLM's time range # matches what the eventual /jobs run will see. cues = await loop.run_in_executor(None, lambda: parse_transcript(transcript_path)) if not cues: raise HTTPException( 400, "Transcript parsed to zero cues. Topic segmentation needs spoken content.", ) full_duration = max( await loop.run_in_executor(None, lambda: video_duration(video_path)), kept[-1].end_seconds, ) if kept[-1].end_seconds < full_duration: kept[-1].end_seconds = full_duration # Stage 3: OCR per scene — the topic-primary picker uses it as # the "is this real slide content?" signal. def _ocr_all() -> list[str]: out: list[str] = [] for f in kept: out.append(run_ocr(f.image_path, skip_inverted=skip_inverted_ocr)) return out scene_ocr_texts = await loop.run_in_executor(None, _ocr_all) # Stage 4: topic LLM call. frames_for_topics = ( [(f.start_seconds, f.image_path) for f in kept] if llm_topic_segmentation_with_images else None ) try: topics = await loop.run_in_executor( None, lambda: extract_topics( cues, total_duration=full_duration, api_key=anthropic_api_key, frames=frames_for_topics, video_type=llm_topic_video_type, ), ) except LLMTopicError as exc: raise HTTPException(502, f"Topic segmentation failed: {exc}") # Stage 5: primary-frame picking per topic. topic_kept, topic_titles, topic_summaries, topic_ocr = _apply_topic_segmentation( kept, topics, scene_ocr_texts=scene_ocr_texts ) if not topic_kept: raise HTTPException(400, "Topic segmentation produced no segments containing slides.") # Stage 6: thumbnails for the picker UI. def _build_topic_cards() -> list[dict]: cards: list[dict] = [] for i, (sf, title, summary, ocr) in enumerate( zip(topic_kept, topic_titles, topic_summaries, topic_ocr) ): img = Image.open(sf.image_path) img.thumbnail((320, 180)) buf = io.BytesIO() img.save(buf, format="JPEG", quality=72) b64 = base64.b64encode(buf.getvalue()).decode() cards.append({ "index": i, "title": title, "summary": summary, "start_seconds": sf.start_seconds, "end_seconds": sf.end_seconds, "thumbnail": f"data:image/jpeg;base64,{b64}", "image_filename": sf.image_path.name, "ocr_text": ocr, "scene_count": 1 + len(sf.alt_image_paths), }) return cards result_topics = await loop.run_in_executor(None, _build_topic_cards) _purge_stale_previews() token = secrets.token_hex(16) with _preview_cache_lock: _preview_cache[token] = { "dir": workdir, "video_path": video_path, "frames_dir": frames_dir, # The "frames" field is what /jobs expects; we store the # post-topic kept list here so the existing preselect # path can pick a subset by index. "frames": topic_kept, # Topic-specific extras /jobs reads to override section # titles and skip the topic LLM call on resume. "topic_titles": topic_titles, "topic_summaries": topic_summaries, "topic_ocr_texts": topic_ocr, "is_topic_preview": True, "created": time.time(), } return JSONResponse({ "preview_token": token, "topics": result_topics, }) except HTTPException: shutil.rmtree(workdir, ignore_errors=True) raise except Exception: shutil.rmtree(workdir, ignore_errors=True) raise @app.post("/jobs") async def start_job( title: str = Form(...), video: UploadFile | None = File(None), transcript: UploadFile = File(...), format: str = Form("review"), threshold: float = Form(27.0), min_gap: float = Form(0.0), max_frames: str | None = Form(None), skip_ocr: bool = Form(False), skip_inverted_ocr: bool = Form(False), face_threshold: float = Form(0.12), lang: str = Form("en"), subtitle: str | None = Form(None), module: str | None = Form(None), anthropic_api_key: str | None = Form(None), llm_alt_text: bool = Form(False), llm_math_equations: bool = Form(False), llm_section_title: bool = Form(False), llm_ocr_text: bool = Form(False), llm_description: bool = Form(False), llm_topic_segmentation: bool = Form(False), llm_topic_segmentation_with_images: bool = Form(False), llm_topic_video_type: str = Form("general"), extract_audio: bool = Form(False), preview_token: str | None = Form(None), selected_indices: str | None = Form(None), ) -> JSONResponse: _validate_format(format) if not is_valid_video_type(llm_topic_video_type): raise HTTPException( 400, f"Unknown video type '{llm_topic_video_type}'. " "Expected one of: general, lecture, lab, tutorial, interview, talk.", ) # Preview cache TTL is normally driven by /scene-preview and # /topic-scene-preview hitting _purge_stale_previews on entry. A # deployment with sustained /jobs traffic but no fresh previews # would never trigger cleanup; do an opportunistic purge here so # stale workdirs can't accumulate beyond TTL. _purge_stale_previews() transcript_suffix = Path(transcript.filename or "").suffix.lower() if transcript_suffix not in TRANSCRIPT_SUFFIXES: raise HTTPException(400, f"Unsupported transcript format: {transcript_suffix or '(none)'}") # If the user ticked any AI feature but provided no API key, the run # would either silently no-op (per-segment toggles) or hard-fail # halfway through (topic segmentation). Catch it up-front with a # message that names the fix, so the form doesn't appear to "work" # while quietly dropping every LLM-driven feature they wanted. any_llm_requested = ( llm_alt_text or llm_math_equations or llm_section_title or llm_ocr_text or llm_topic_segmentation or llm_description ) if any_llm_requested and not (anthropic_api_key and anthropic_api_key.strip()): raise HTTPException( 400, "AI features were requested (e.g. topic segmentation, section titles) " "but no Anthropic API key was provided. Either add a key in the AI " "assist section or uncheck the AI features.", ) # Resolve preview cache entry (if the user went through either # scene picker — the basic Preview scenes or the topic-aware twin). preview_entry: dict | None = None preselected_frames: list[SceneFrame] | None = None preselected_topic_titles: list[str] | None = None preselected_topic_summaries: list[str] | None = None is_topic_preview = False if preview_token: with _preview_cache_lock: preview_entry = _preview_cache.pop(preview_token, None) if not preview_entry: raise HTTPException(400, "Scene preview has expired — please preview again.") # Past this point, preview_entry owns a temp dir on disk. Any # exception before the worker thread takes ownership (workdir # try/finally) would orphan that dir. Catch validation errors # here and clean it up before re-raising. try: all_frames: list[SceneFrame] = preview_entry["frames"] is_topic_preview = bool(preview_entry.get("is_topic_preview")) if is_topic_preview: valid_max = len(all_frames) - 1 else: # Scene preview tracks frames by their original .index # (post-filter ordering), not positional; valid range = # the set of those index values. valid_indices = {f.index for f in all_frames} if selected_indices: # Parse and validate up-front so an out-of-range index # is a 400 rather than a silently-dropped slide the # user thinks they kept. Negative numbers and non-ints # raise explicitly instead of being skipped. raw_indices: list[int] = [] for raw in selected_indices.split(","): token = raw.strip() if not token: continue try: n = int(token) except ValueError: raise HTTPException( 400, f"Invalid index '{token}' in selected_indices.", ) if n < 0: raise HTTPException( 400, f"Negative index {n} is not allowed in selected_indices.", ) raw_indices.append(n) if is_topic_preview: for n in raw_indices: if n > valid_max: raise HTTPException( 400, f"Topic index {n} is out of range " f"(have {len(all_frames)} topics).", ) else: bad = [n for n in raw_indices if n not in valid_indices] if bad: raise HTTPException( 400, f"Scene indices {bad} are not in the preview set.", ) idx_set = set(raw_indices) else: idx_set = ( set(range(len(all_frames))) if is_topic_preview else valid_indices ) if is_topic_preview: # Topic preview indices are positional (the order topics # came back in /topic-scene-preview), not the .index field. topic_titles = preview_entry.get("topic_titles") or [] topic_summaries = preview_entry.get("topic_summaries") or [] preselected_frames = [ f for i, f in enumerate(all_frames) if i in idx_set ] preselected_topic_titles = [ t for i, t in enumerate(topic_titles) if i in idx_set ] # Forward summaries too — without this the resumed # pipeline has no topic-segmentation pass (it was done # in the preview) so the "In brief:" advance organizer # would silently fall back to "" for every section. preselected_topic_summaries = [ s for i, s in enumerate(topic_summaries) if i in idx_set ] else: preselected_frames = [f for f in all_frames if f.index in idx_set] if not preselected_frames: raise HTTPException( 400, "No topics selected." if is_topic_preview else "No scenes selected.", ) except HTTPException as exc: # A 4xx here is the client's selection being invalid (bad index, # empty selection) — recoverable. Re-insert the popped preview so # the user can correct their input and retry WITHOUT paying for # another topic preview (which costs an LLM call). Only 4xx are # recoverable; a 5xx means something server-side broke, so the # preview is dropped like any other hard failure. if 400 <= exc.status_code < 500: with _preview_cache_lock: _preview_cache[preview_token] = preview_entry preview_entry = None # keep the finally from cleaning it up raise shutil.rmtree(preview_entry["dir"], ignore_errors=True) preview_entry = None raise except Exception: # Cleanup before re-raise so the popped preview workdir # doesn't leak when validation fails after the cache pop. # Catch Exception (not BaseException) so KeyboardInterrupt # and SystemExit still propagate without first running # cleanup that could mask the signal. shutil.rmtree(preview_entry["dir"], ignore_errors=True) preview_entry = None raise else: if not video or not video.filename: raise HTTPException(400, "A video file is required when not using scene preview.") video_suffix = Path(video.filename).suffix.lower() if video_suffix not in VIDEO_SUFFIXES: raise HTTPException(400, f"Unsupported video format: {video_suffix or '(none)'}") workdir = Path(tempfile.mkdtemp(prefix="vgm_")) handed_off = False try: transcript_path = workdir / f"transcript{transcript_suffix}" await _save_upload( transcript, transcript_path, max_bytes=MAX_TRANSCRIPT_BYTES, label="Transcript" ) if preview_entry: video_path = preview_entry["video_path"] frames_dir = preview_entry["frames_dir"] else: video_suffix = Path(video.filename).suffix.lower() # type: ignore[union-attr] video_path = workdir / f"input{video_suffix}" frames_dir = workdir / "static" frames_dir.mkdir(parents=True, exist_ok=True) await _save_upload(video, video_path, max_bytes=MAX_VIDEO_BYTES, label="Video") inputs = PipelineInputs( video_path=video_path, transcript_path=transcript_path, frames_dir=frames_dir, title=title, subtitle=subtitle or None, module=module or None, lang=lang, threshold=threshold, min_gap_seconds=min_gap, max_frames=_parse_optional_int(max_frames), skip_ocr=skip_ocr, skip_inverted_ocr=skip_inverted_ocr, face_threshold=face_threshold, inline_images=(format in ("single", "review")), anthropic_api_key=(anthropic_api_key or None), llm_toggles=LLMToggles( alt_text=llm_alt_text, math_equations=llm_math_equations, section_title=llm_section_title, ocr_text=llm_ocr_text, description=llm_description, # Topic segmentation already happened during the preview; # don't re-run it in the resume path or we'd pay for the # call twice and could pick different boundaries than the # user just curated. topic_segmentation=llm_topic_segmentation and not is_topic_preview, topic_segmentation_with_images=llm_topic_segmentation_with_images, topic_video_type=llm_topic_video_type, ), extract_audio=extract_audio, ) try: job = registry.create() except JobQuotaExceeded as exc: raise HTTPException(429, str(exc)) job.ocr_skip_inverted = skip_inverted_ocr try: threading.Thread( target=_run_job_in_thread, args=(job, inputs, workdir), kwargs=dict( format=format, title=title, lang=lang, subtitle=subtitle or None, module=module or None, preselected_frames=preselected_frames, preselected_topic_titles=preselected_topic_titles, preselected_topic_summaries=preselected_topic_summaries, preview_dir=preview_entry["dir"] if preview_entry else None, ), daemon=True, ).start() except BaseException: # If the OS refuses the thread, the job would otherwise sit in # the registry as "running" forever (finished_at stays None, so # the TTL purge never reclaims it) — a permanently-leaked quota # slot. Release it before propagating. registry.pop(job.id) raise handed_off = True finally: # Anything that fails before the worker thread takes ownership of # workdir — bad form field, copy error, dataclass validation, even # an OS thread-start failure — must clean up the temp dir we just # mkdtemp'd. Once Thread.start() returns successfully, the worker's # own try/finally handles cleanup. if not handed_off: shutil.rmtree(workdir, ignore_errors=True) if preview_entry: shutil.rmtree(preview_entry["dir"], ignore_errors=True) return JSONResponse( { "job_id": job.id, "events_url": f"/events/{job.id}", "result_url": f"/result/{job.id}", }, status_code=202, ) @app.post("/jobs/from-images") async def start_job_from_images( title: str = Form(...), images: list[UploadFile] = File(...), transcript: UploadFile | None = File(None), format: str = Form("review"), subtitle: str | None = Form(None), module: str | None = Form(None), lang: str = Form("en"), skip_ocr: bool = Form(False), skip_inverted_ocr: bool = Form(False), anthropic_api_key: str | None = Form(None), llm_alt_text: bool = Form(False), llm_math_equations: bool = Form(False), llm_section_title: bool = Form(False), llm_ocr_text: bool = Form(False), llm_description: bool = Form(False), llm_topic_segmentation: bool = Form(False), llm_topic_segmentation_with_images: bool = Form(False), llm_topic_video_type: str = Form("general"), ) -> JSONResponse: """Images-input mode: build a study guide from pre-exported slide images (each named with an HH-MM-SS timestamp) plus one transcript, with no source video and no scene detection. Each image becomes a segment anchored at its timestamp; the transcript is aligned to those anchors by the same downstream pipeline the video path uses. Deterministic, no large upload, author-curated slides. """ if format not in ALLOWED_FORMATS: raise HTTPException(400, f"Unknown output format: {format}") # The transcript is OPTIONAL in images mode: with none, the guide is # built from the slides alone (no narration). FastAPI can hand us an # empty UploadFile with a blank filename for an omitted field, so treat # "no filename" as "no transcript". has_transcript = bool(transcript is not None and (transcript.filename or "").strip()) transcript_suffix = Path(transcript.filename or "").suffix.lower() if has_transcript else "" if has_transcript and transcript_suffix not in TRANSCRIPT_SUFFIXES: raise HTTPException(400, f"Unsupported transcript format: {transcript_suffix or '(none)'}") if not images: raise HTTPException(400, "At least one slide image is required.") if len(images) > MAX_IMAGE_COUNT: raise HTTPException( 400, f"Too many images ({len(images)}); the limit is {MAX_IMAGE_COUNT}." ) # Same guard as /jobs: AI toggles without a key would silently no-op # or hard-fail mid-run. any_llm_requested = ( llm_alt_text or llm_math_equations or llm_section_title or llm_ocr_text or llm_topic_segmentation or llm_description ) if any_llm_requested and not (anthropic_api_key and anthropic_api_key.strip()): raise HTTPException( 400, "AI features were requested but no Anthropic API key was provided. " "Add a key or uncheck the AI features.", ) # Topic segmentation reads the transcript; it can't run without one. if llm_topic_segmentation and not has_transcript: raise HTTPException( 400, "Topic segmentation needs a transcript. Add one, or turn topic " "segmentation off.", ) if llm_topic_segmentation and not is_valid_video_type(llm_topic_video_type): raise HTTPException( 400, f"Unknown video type '{llm_topic_video_type}'. " "Expected one of: general, lecture, lab, tutorial, interview, talk.", ) workdir = Path(tempfile.mkdtemp(prefix="vgm_img_")) handed_off = False try: transcript_path: Path | None = None transcript_end: float | None = None transcript_start: float = 0.0 transcript_cues: list = [] if has_transcript: transcript_path = workdir / f"transcript{transcript_suffix}" await _save_upload( transcript, transcript_path, max_bytes=MAX_TRANSCRIPT_BYTES, label="Transcript" ) cues = parse_transcript(transcript_path) if not cues: raise HTTPException( 400, "Transcript parsed to zero cues. Provide a non-empty VTT/SRT " "file, or omit the transcript entirely.", ) transcript_cues = cues transcript_start = cues[0].start transcript_end = cues[-1].end frames_dir = workdir / "static" frames_dir.mkdir(parents=True, exist_ok=True) saved_paths: list[Path] = [] seen_names: set[str] = set() for up in images: base = Path(up.filename or "").name suffix = Path(base).suffix.lower() if suffix not in IMAGE_SUFFIXES: raise HTTPException( 400, f"Unsupported image format '{suffix or '(none)'}' for " f"'{base or '(unnamed)'}'. Use PNG or JPG.", ) if not base or base in seen_names: # Guarantee a unique, path-safe on-disk name; keep the # original (which carries the timestamp) as a prefix so the # parser still finds HH-MM-SS. base = f"{len(saved_paths):03d}_{base or 'image' + suffix}" seen_names.add(base) dest = frames_dir / base await _save_upload(up, dest, max_bytes=MAX_IMAGE_BYTES, label="Image") saved_paths.append(dest) # Three ways to place the slides on a timeline: # 1. No transcript — slides just need an order. Sort by filename, # no timestamps required, and mark the guide untimed so it # suppresses the (meaningless) time labels. # 2. Transcript + every filename carries HH-MM-SS — anchor each # slide at its own filename timestamp (deterministic). # 3. Transcript but filenames lack timestamps — read each slide's # on-screen text (local OCR) and ask Claude when the narration # first discusses it, giving real per-slide start times. If # that fails (no key, network, bad output), fall back to an # even split across the transcript's span. Both are "timed". untimed_flag = not has_transcript def _build_aligned_frames() -> list[SceneFrame]: """Sync (OCR + network) work, run off the event loop.""" from .pipeline.ocr import ocr_frame ordered = order_by_name(saved_paths) slide_texts = [ ocr_frame(p, skip_inverted=skip_inverted_ocr) for p in ordered ] try: starts = align_slides_to_transcript( transcript_cues, slide_texts, api_key=(anthropic_api_key or None), ) return build_frames_from_starts( ordered, starts, transcript_end=transcript_end ) except SlideAlignError as exc: log.info("Slide alignment fell back to even split: %s", exc) return build_frames_even_split( saved_paths, transcript_start=transcript_start, transcript_end=transcript_end, ) try: if not has_transcript: frames = build_frames_ordered_by_name(saved_paths) elif all_have_timestamps(saved_paths, transcript_end): frames = build_frames_from_images( saved_paths, transcript_end=transcript_end ) elif anthropic_api_key and anthropic_api_key.strip(): # AI alignment needs a key; without one, go straight to the # even split (no OCR/network to attempt). loop = asyncio.get_running_loop() frames = await loop.run_in_executor(None, _build_aligned_frames) else: frames = build_frames_even_split( saved_paths, transcript_start=transcript_start, transcript_end=transcript_end, ) except ImageInputError as exc: raise HTTPException(400, str(exc)) inputs = PipelineInputs( video_path=None, transcript_path=transcript_path, frames_dir=frames_dir, title=title, subtitle=subtitle or None, module=module or None, lang=lang, skip_ocr=skip_ocr, skip_inverted_ocr=skip_inverted_ocr, inline_images=(format in ("single", "review")), anthropic_api_key=(anthropic_api_key or None), llm_toggles=LLMToggles( alt_text=llm_alt_text, math_equations=llm_math_equations, section_title=llm_section_title, ocr_text=llm_ocr_text, description=llm_description, topic_segmentation=llm_topic_segmentation, topic_segmentation_with_images=llm_topic_segmentation_with_images, topic_video_type=llm_topic_video_type, ), extract_audio=False, # no source video to slice allow_empty_transcript=True, # transcript optional in images mode untimed=untimed_flag, # no transcript → order-only, no times ) try: job = registry.create() except JobQuotaExceeded as exc: raise HTTPException(429, str(exc)) job.ocr_skip_inverted = skip_inverted_ocr try: threading.Thread( target=_run_job_in_thread, args=(job, inputs, workdir), kwargs=dict( format=format, title=title, lang=lang, subtitle=subtitle or None, module=module or None, preselected_frames=frames, ), daemon=True, ).start() except BaseException: registry.pop(job.id) raise handed_off = True finally: if not handed_off: shutil.rmtree(workdir, ignore_errors=True) return JSONResponse( { "job_id": job.id, "events_url": f"/events/{job.id}", "result_url": f"/result/{job.id}", }, status_code=202, ) @app.get("/events/{job_id}") async def stream_events(job_id: str) -> StreamingResponse: job = registry.get(job_id) if not job: raise HTTPException(404, "Unknown job id.") # Hard cap on how long an SSE consumer will wait without ANY event # before declaring the job dead. Bounds the worst case where the # worker thread crashed before queuing a terminal event — without # this the SSE generator would ping forever. silence_seconds = 1800 async def gen(): last_event_time = time.time() while True: try: ev = await asyncio.to_thread(job.events.get, True, 15.0) except queue.Empty: # Status may have flipped without a queued event # (defensive); exit cleanly instead of pinging forever. if job.status in ("done", "error"): return # If we haven't seen an event in a long time AND the job # claims to still be running, treat it as wedged. Re-check # status right before finish_error to avoid racing a # finish_success that may have just fired. if ( time.time() - last_event_time > silence_seconds and job.status == "running" and job.finished_at is None ): registry.finish_error( job, "Job timed out (no progress in over 30 minutes)." ) # finish_error queued the terminal event; loop once # more to pick it up and exit through the normal path. continue yield ": ping\n\n" continue last_event_time = time.time() yield f"data: {json.dumps(ev)}\n\n" if ev.get("type") in ("done", "error"): return headers = { "Cache-Control": "no-cache, no-transform", "X-Accel-Buffering": "no", # disable proxy buffering for SSE } return StreamingResponse(gen(), media_type="text/event-stream", headers=headers) @app.get("/result/{job_id}") async def fetch_result(job_id: str) -> Response: # Keep the job alive after delivery so the editor can lazy-fetch alt # frames; the job-TTL purge handles eventual cleanup. job = registry.get(job_id) if not job: raise HTTPException(404, "Unknown job id.") if job.status == "error": raise HTTPException(422, job.error or "Pipeline error.") if job.status != "done" or job.result is None: raise HTTPException(409, "Job not finished.") if not job.result.body: # Body was already delivered and freed; alts are still fetchable # but the rendered HTML/zip itself is one-shot. raise HTTPException(410, "Result already delivered.") headers: dict[str, str] = {} if job.result.filename: headers["Content-Disposition"] = _content_disposition(job.result.filename) body = job.result.body media = job.result.media_type # Review-mode results have no attachment filename; the upload page # loads them into an iframe AND lets the user "Open in new tab", # which re-fetches the same URL. Keep the body for the job's TTL so # the second GET still works. Single/zip downloads stay one-shot — # the body for those can be MB-scale and isn't re-requested. if job.result.filename: job.result.body = b"" return Response(content=body, media_type=media, headers=headers) _ALT_FILENAME_RE = __import__("re").compile(r"^scene_\d{4}_alt\d+\.(jpg|jpeg|png)$") _PRIMARY_FILENAME_RE = __import__("re").compile(r"^scene_\d{4}\.(jpg|jpeg|png)$") _AUDIO_FILENAME_RE = __import__("re").compile(r"^segment_\d{4}\.mp3$") _IMAGE_ASSET_EXTS = {".jpg", ".jpeg", ".png", ".webp"} _AUDIO_ASSET_EXTS = {".mp3"} def _is_safe_asset_name(name: str, allowed_exts: set[str]) -> bool: """A bundle-safe filename: a bare basename (no directory components, no traversal) with an allowed extension. The ZIP builder joins these onto a temp dir and reads/writes them, so they must not be able to escape it. `Path(name).name == name` rejects anything with a separator or `..` component; the extension whitelist rejects the rest. This replaces the old scene_NNNN-only patterns, which wrongly excluded images-mode uploads that keep their original names (e.g. 'slide 1.jpg').""" if not isinstance(name, str) or not name: return False if Path(name).name != name: return False return Path(name).suffix.lower() in allowed_exts @app.get("/audio/{job_id}/{filename}") async def fetch_audio(job_id: str, filename: str) -> Response: if not _AUDIO_FILENAME_RE.match(filename): raise HTTPException(404, "Unknown audio.") job = registry.get(job_id) if not job: raise HTTPException(404, "Unknown job id.") data = job.audio_bytes.get(filename) if data is None: raise HTTPException(404, "Audio not found.") return Response( content=data, media_type="audio/mpeg", headers={"Cache-Control": "private, max-age=600"}, ) @app.get("/alts/{job_id}/{filename}") async def fetch_alt(job_id: str, filename: str) -> Response: # Filename comes from a URL path, so be strict about its shape — only # patterns the pipeline produces are accepted, and no traversal chars. # Topic-based segmentation re-uses other scenes' primary frames as # alternates, so we also accept the primary scene_NNNN.jpg shape. if not ( _ALT_FILENAME_RE.match(filename) or _PRIMARY_FILENAME_RE.match(filename) ): raise HTTPException(404, "Unknown alternate.") job = registry.get(job_id) if not job: raise HTTPException(404, "Unknown job id.") data = job.alt_image_bytes.get(filename) if data is None: raise HTTPException(404, "Alternate not found.") media = "image/jpeg" if filename.lower().endswith((".jpg", ".jpeg")) else "image/png" return Response( content=data, media_type=media, headers={"Cache-Control": "private, max-age=600"}, ) @app.get("/ocr/{job_id}/{filename}") async def fetch_ocr(job_id: str, filename: str) -> Response: # Lazy OCR for an alternate frame. Tesseract is slow on the HF CPU # tier, so we skip it during the main pipeline and run it here only # when the reviewer actually picks an alt. First call writes the # cached bytes to a temp file and OCRs; subsequent calls return cache. if not ( _ALT_FILENAME_RE.match(filename) or _PRIMARY_FILENAME_RE.match(filename) ): raise HTTPException(404, "Unknown alternate.") job = registry.get(job_id) if not job: raise HTTPException(404, "Unknown job id.") cached = job.alt_ocr_cache.get(filename) if cached is not None: return Response( content=cached, media_type="text/plain; charset=utf-8", headers={"Cache-Control": "private, max-age=600"}, ) data = job.alt_image_bytes.get(filename) if data is None: raise HTTPException(404, "Alternate not found.") import tempfile from pathlib import Path from fastapi.concurrency import run_in_threadpool from .pipeline.ocr import ocr_frame suffix = ".jpg" if filename.lower().endswith((".jpg", ".jpeg")) else ".png" with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp: tmp.write(data) tmp_path = Path(tmp.name) try: text = await run_in_threadpool( lambda: ocr_frame(tmp_path, skip_inverted=job.ocr_skip_inverted) ) finally: try: tmp_path.unlink(missing_ok=True) except OSError: pass # If a parallel /ocr request for the same filename beat us to the cache, # keep its result rather than overwriting; both ran OCR on identical # bytes so the texts should match anyway, but using setdefault makes the # winning text stable for any caller that already read it. text = job.alt_ocr_cache.setdefault(filename, text) return Response( content=text, media_type="text/plain; charset=utf-8", headers={"Cache-Control": "private, max-age=600"}, ) _DATA_URI_RE = __import__("re").compile(r"^data:[^;]+;base64,(.+)$", __import__("re").DOTALL) @app.post("/jobs/{job_id}/zip") async def fetch_job_zip(job_id: str, request: Request) -> Response: """Bundle the current editor state into a downloadable .zip. The editor POSTs its in-memory state plus the data URIs of every currently-picked primary frame (alt or original — the editor's main <img> src is the source of truth). The server decodes those bytes, pulls audio bytes from the in-memory job, calls the same render_from_metadata + bundle_zip path the CLI uses, and returns the bundle. No request body is persisted; the temp dir is deleted in `finally`. """ job = registry.get(job_id) if not job: raise HTTPException(404, "Unknown job id.") try: body = await request.json() except Exception: raise HTTPException(400, "Invalid JSON body.") state = body.get("state") if isinstance(body, dict) else None primary_uris = body.get("primary_image_data_uris") if isinstance(body, dict) else None if not isinstance(state, dict) or not isinstance(primary_uris, dict): raise HTTPException(400, "Body must be {state, primary_image_data_uris}.") try: page = PageMetadata.model_validate(state) except Exception as exc: raise HTTPException(422, f"Invalid metadata: {exc}") import base64 workdir = Path(tempfile.mkdtemp(prefix="vgm_zip_")) try: frames_dir = workdir / "static" frames_dir.mkdir() # Decode and write the picked primary frame bytes for each segment. # Filenames must match what the metadata references — only patterns # the pipeline produces are accepted. for filename, uri in primary_uris.items(): if not isinstance(filename, str) or not isinstance(uri, str): continue if not _is_safe_asset_name(filename, _IMAGE_ASSET_EXTS): continue m = _DATA_URI_RE.match(uri) if not m: continue try: data = base64.b64decode(m.group(1), validate=True) except Exception: continue (frames_dir / filename).write_bytes(data) # Pull audio bytes from the job (no client upload needed) and # write them next to the frames so the bundled HTML can link # via `static/<audio>`. for fn, data in job.audio_bytes.items(): if not _AUDIO_FILENAME_RE.match(fn): continue (frames_dir / fn).write_bytes(data) # Render with image_dir_name="static" and inline=False; the HTML # will link relatively to files we just wrote. try: html = render_from_metadata(page, frames_dir, mode="guide", image_dir_name="static") except Exception as exc: raise HTTPException(500, f"Render failed: {exc}") # Filenames come from client-supplied state — only accept the # exact patterns the pipeline produces so a crafted name with # path separators or `..` can't escape `frames_dir` and pull # arbitrary readable files into the bundle. frame_paths = [ frames_dir / s.image_filename for s in page.segments if _is_safe_asset_name(s.image_filename, _IMAGE_ASSET_EXTS) ] audio_paths = [ frames_dir / s.audio_filename for s in page.segments if _is_safe_asset_name(s.audio_filename, _AUDIO_ASSET_EXTS) ] zip_bytes = bundle_zip( html, frame_paths, audio_paths=audio_paths, ) filename = _safe_filename(page.title) + ".zip" return Response( content=zip_bytes, media_type="application/zip", headers={"Content-Disposition": _content_disposition(filename)}, ) finally: shutil.rmtree(workdir, ignore_errors=True) # Backwards-compat: keep a synchronous /process endpoint for non-browser clients. @app.post("/process") async def process( request: Request, title: str = Form(...), video: UploadFile = File(...), transcript: UploadFile = File(...), format: str = Form("review"), threshold: float = Form(27.0), min_gap: float = Form(0.0), max_frames: str | None = Form(None), skip_ocr: bool = Form(False), skip_inverted_ocr: bool = Form(False), face_threshold: float = Form(0.12), lang: str = Form("en"), subtitle: str | None = Form(None), module: str | None = Form(None), anthropic_api_key: str | None = Form(None), llm_alt_text: bool = Form(False), llm_math_equations: bool = Form(False), llm_section_title: bool = Form(False), llm_ocr_text: bool = Form(False), llm_description: bool = Form(False), llm_topic_segmentation: bool = Form(False), llm_topic_segmentation_with_images: bool = Form(False), llm_topic_video_type: str = Form("general"), extract_audio: bool = Form(False), ) -> Response: _validate_format(format) if not is_valid_video_type(llm_topic_video_type): raise HTTPException( 400, f"Unknown video type '{llm_topic_video_type}'. " "Expected one of: general, lecture, lab, tutorial, interview, talk.", ) # Same guard as /jobs: AI features ticked without a key would either # silently no-op (per-segment toggles) or hard-fail mid-run (topic # segmentation). Fail fast with a message that names the fix instead # of returning a guide that quietly dropped everything the caller asked # for. (/process is the legacy non-browser endpoint; it had no guard.) any_llm_requested = ( llm_alt_text or llm_math_equations or llm_section_title or llm_ocr_text or llm_topic_segmentation or llm_description ) if any_llm_requested and not (anthropic_api_key and anthropic_api_key.strip()): raise HTTPException( 400, "AI features were requested but no Anthropic API key was provided. " "Add a key or drop the AI options.", ) video_suffix = Path(video.filename or "").suffix.lower() transcript_suffix = Path(transcript.filename or "").suffix.lower() if video_suffix not in VIDEO_SUFFIXES: raise HTTPException(400, f"Unsupported video format: {video_suffix or '(none)'}") if transcript_suffix not in TRANSCRIPT_SUFFIXES: raise HTTPException(400, f"Unsupported transcript format: {transcript_suffix or '(none)'}") workdir = Path(tempfile.mkdtemp(prefix="vgm_")) try: video_path = workdir / f"input{video_suffix}" transcript_path = workdir / f"transcript{transcript_suffix}" frames_dir = workdir / "static" await _save_upload(video, video_path, max_bytes=MAX_VIDEO_BYTES, label="Video") await _save_upload( transcript, transcript_path, max_bytes=MAX_TRANSCRIPT_BYTES, label="Transcript" ) inputs = PipelineInputs( video_path=video_path, transcript_path=transcript_path, frames_dir=frames_dir, title=title, subtitle=subtitle or None, module=module or None, lang=lang, threshold=threshold, min_gap_seconds=min_gap, max_frames=_parse_optional_int(max_frames), skip_ocr=skip_ocr, skip_inverted_ocr=skip_inverted_ocr, face_threshold=face_threshold, inline_images=(format in ("single", "review")), anthropic_api_key=(anthropic_api_key or None), llm_toggles=LLMToggles( alt_text=llm_alt_text, math_equations=llm_math_equations, section_title=llm_section_title, ocr_text=llm_ocr_text, description=llm_description, topic_segmentation=llm_topic_segmentation, topic_segmentation_with_images=llm_topic_segmentation_with_images, topic_video_type=llm_topic_video_type, ), extract_audio=extract_audio, ) try: result = await asyncio.to_thread(run_pipeline, inputs) except PipelineError as exc: raise HTTPException(422, str(exc)) job_result = _render_to_result( format=format, title=title, lang=lang, subtitle=subtitle or None, module=module or None, eyebrow=result.page.eyebrow, meta_lines=result.page.meta_lines, segments=result.segments, image_paths=[f.image_path for f in result.kept_frames], audio_paths=result.audio_paths, ) headers: dict[str, str] = {} if job_result.filename: headers["Content-Disposition"] = _content_disposition(job_result.filename) return Response(content=job_result.body, media_type=job_result.media_type, headers=headers) finally: shutil.rmtree(workdir, ignore_errors=True)
"""FastAPI app: upload UI + SSE-driven pipeline orchestration."""
from __future__ import annotations
import asyncio
import io
import json
import logging
import os
import queue
import secrets
import shutil
import tempfile
import threading
import time
from pathlib import Path
from fastapi import FastAPI, File, Form, HTTPException, Request, UploadFile
from fastapi.responses import HTMLResponse, JSONResponse, Response, StreamingResponse
from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates
from .jobs import Job, JobQuotaExceeded, JobResult, registry
from .pipeline import bundle_zip, render_from_metadata, render_guide, render_review
from .pipeline.filter import filter_instructor_frames
from .pipeline.llm_extract import (
LLMToggles,
LLMTopicError,
SlideAlignError,
align_slides_to_transcript,
extract_topics,
is_valid_video_type,
)
from .pipeline.metadata import CURRENT_SCHEMA_VERSION, PageMetadata, migrate_v1_to_v2
from .pipeline.orchestrator import (
PipelineError,
PipelineInputs,
ProgressEvent,
_apply_topic_segmentation,
run_pipeline,
)
from .pipeline.image_input import (
ImageInputError,
all_have_timestamps,
build_frames_even_split,
build_frames_from_images,
build_frames_from_starts,
build_frames_ordered_by_name,
order_by_name,
)
from .pipeline.scenes import detect_and_extract, video_duration
from .pipeline.transcript import parse_transcript
from .pipeline.types import SceneFrame
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
log = logging.getLogger("videoguidemaker")
BASE_DIR = Path(__file__).resolve().parent
templates = Jinja2Templates(directory=str(BASE_DIR / "templates"))
app = FastAPI(title="The Describatron")
app.mount("/static", StaticFiles(directory=str(BASE_DIR / "static")), name="static")
_SLACK_BYTES = 8 * 1024 * 1024 # multipart framing + the small form fields
def _request_body_ceiling(path: str) -> int | None:
"""Hard upper bound on an acceptable request body for `path`, or None to
skip the early check. Sized per-endpoint from that endpoint's own limits
so a legitimately large images-mode upload isn't wrongly rejected and
/edit isn't given the (much larger) video ceiling. Returns None when the
relevant cap is disabled (an explicit opt-out of size protection).
NB: this only fires when the request declares a Content-Length; a
chunked upload has none, so each upload endpoint additionally bounds
its own read/stream (_save_upload for files, the chunked read in /edit).
"""
if path == "/jobs/from-images":
if MAX_IMAGE_BYTES is None:
return None
return MAX_IMAGE_COUNT * MAX_IMAGE_BYTES + (MAX_TRANSCRIPT_BYTES or 0) + _SLACK_BYTES
if path == "/edit":
if MAX_EDIT_JSON_BYTES is None:
return None
return MAX_EDIT_JSON_BYTES + _SLACK_BYTES
if MAX_VIDEO_BYTES is None:
return None
return MAX_VIDEO_BYTES + (MAX_TRANSCRIPT_BYTES or 0) + _SLACK_BYTES
@app.middleware("http")
async def _security_headers(request: Request, call_next):
# Reject an over-cap upload from its declared Content-Length BEFORE
# Starlette spools the multipart body to a temp file. The per-field
# streaming check in _save_upload still runs (and catches an absent or
# lying Content-Length), but this stops the obvious "declare a 10 GB
# body" case from filling the Space's ephemeral disk first.
if request.method in ("POST", "PUT", "PATCH"):
ceiling = _request_body_ceiling(request.url.path)
declared = request.headers.get("content-length")