Spaces:
Sleeping
Second-pass audit: SSE timing, OCR memory cap, filter robustness, CLI ordering
Browse files- SSE generator now resets its idle timeout on every event received
rather than measuring from job.created_at, so a healthy long-running
job emitting periodic progress events isn't killed at the 30-min mark.
Also re-checks job.status == "running" right before calling
finish_error to avoid racing a finish_success that landed during the
events.get timeout.
- ocr._preprocess_for_ocr caps the post-upscale max dimension at 2400 px
so an unusual input frame doesn't allocate huge intermediates (the
subsequent adaptiveThreshold doubles the footprint again, and we run
OCR concurrently across job + lazy-alt requests).
- filter_instructor_frames now skips candidate paths that don't exist
on disk. Previously is_instructor_frame returned False on a missing
file (cv2.imread is None), which let phantom paths slip through into
the editor's frame picker where /alts/ would then 404.
- cli build: dump_metadata runs BEFORE rendering, not after, so a
template / disk-write failure doesn't discard the LLM/OCR work the
user just paid for.
Verified along the way: the Anthropic SDK call signature
(client.messages.parse(output_format=...)) is correct against the
installed anthropic 0.100.0 — the audit subagent flagged it without
being able to check the SDK.
- app/cli.py +7 -4
- app/main.py +18 -16
- app/pipeline/filter.py +6 -0
- app/pipeline/ocr.py +9 -2
|
@@ -136,6 +136,13 @@ def build(
|
|
| 136 |
typer.secho(f"error: {exc}", fg=typer.colors.RED, err=True)
|
| 137 |
raise typer.Exit(2)
|
| 138 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 139 |
common = dict(
|
| 140 |
title=title,
|
| 141 |
segments=result.segments,
|
|
@@ -177,10 +184,6 @@ def build(
|
|
| 177 |
|
| 178 |
typer.echo(f"wrote {output}")
|
| 179 |
|
| 180 |
-
if export_metadata:
|
| 181 |
-
dump_metadata(export_metadata, result.page)
|
| 182 |
-
typer.echo(f"wrote {export_metadata}")
|
| 183 |
-
|
| 184 |
|
| 185 |
@app.command("export-metadata")
|
| 186 |
def export_metadata_cmd(
|
|
|
|
| 136 |
typer.secho(f"error: {exc}", fg=typer.colors.RED, err=True)
|
| 137 |
raise typer.Exit(2)
|
| 138 |
|
| 139 |
+
# Dump metadata BEFORE rendering: a render failure (template bug,
|
| 140 |
+
# disk full mid-write) would otherwise discard the LLM/OCR work
|
| 141 |
+
# the user just paid for.
|
| 142 |
+
if export_metadata:
|
| 143 |
+
dump_metadata(export_metadata, result.page)
|
| 144 |
+
typer.echo(f"wrote {export_metadata}")
|
| 145 |
+
|
| 146 |
common = dict(
|
| 147 |
title=title,
|
| 148 |
segments=result.segments,
|
|
|
|
| 184 |
|
| 185 |
typer.echo(f"wrote {output}")
|
| 186 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 187 |
|
| 188 |
@app.command("export-metadata")
|
| 189 |
def export_metadata_cmd(
|
|
@@ -342,25 +342,30 @@ async def stream_events(job_id: str) -> StreamingResponse:
|
|
| 342 |
if not job:
|
| 343 |
raise HTTPException(404, "Unknown job id.")
|
| 344 |
|
| 345 |
-
# Hard cap on how long an SSE consumer will wait
|
| 346 |
-
#
|
| 347 |
-
# worker thread crashed before
|
| 348 |
-
#
|
| 349 |
-
|
| 350 |
-
max_job_seconds = 1800
|
| 351 |
|
| 352 |
async def gen():
|
| 353 |
-
|
| 354 |
while True:
|
| 355 |
try:
|
| 356 |
ev = await asyncio.to_thread(job.events.get, True, 15.0)
|
| 357 |
except queue.Empty:
|
| 358 |
-
#
|
| 359 |
-
#
|
| 360 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 361 |
if (
|
| 362 |
-
|
| 363 |
-
and
|
|
|
|
| 364 |
):
|
| 365 |
registry.finish_error(
|
| 366 |
job, "Job timed out (no progress in over 30 minutes)."
|
|
@@ -368,12 +373,9 @@ async def stream_events(job_id: str) -> StreamingResponse:
|
|
| 368 |
# finish_error queued the terminal event; loop once
|
| 369 |
# more to pick it up and exit through the normal path.
|
| 370 |
continue
|
| 371 |
-
# If status flipped without a queued event (defensive),
|
| 372 |
-
# exit cleanly instead of pinging forever.
|
| 373 |
-
if job.status in ("done", "error"):
|
| 374 |
-
return
|
| 375 |
yield ": ping\n\n"
|
| 376 |
continue
|
|
|
|
| 377 |
yield f"data: {json.dumps(ev)}\n\n"
|
| 378 |
if ev.get("type") in ("done", "error"):
|
| 379 |
return
|
|
|
|
| 342 |
if not job:
|
| 343 |
raise HTTPException(404, "Unknown job id.")
|
| 344 |
|
| 345 |
+
# Hard cap on how long an SSE consumer will wait without ANY event
|
| 346 |
+
# before declaring the job dead. Bounds the worst case where the
|
| 347 |
+
# worker thread crashed before queuing a terminal event — without
|
| 348 |
+
# this the SSE generator would ping forever.
|
| 349 |
+
silence_seconds = 1800
|
|
|
|
| 350 |
|
| 351 |
async def gen():
|
| 352 |
+
last_event_time = time.time()
|
| 353 |
while True:
|
| 354 |
try:
|
| 355 |
ev = await asyncio.to_thread(job.events.get, True, 15.0)
|
| 356 |
except queue.Empty:
|
| 357 |
+
# Status may have flipped without a queued event
|
| 358 |
+
# (defensive); exit cleanly instead of pinging forever.
|
| 359 |
+
if job.status in ("done", "error"):
|
| 360 |
+
return
|
| 361 |
+
# If we haven't seen an event in a long time AND the job
|
| 362 |
+
# claims to still be running, treat it as wedged. Re-check
|
| 363 |
+
# status right before finish_error to avoid racing a
|
| 364 |
+
# finish_success that may have just fired.
|
| 365 |
if (
|
| 366 |
+
time.time() - last_event_time > silence_seconds
|
| 367 |
+
and job.status == "running"
|
| 368 |
+
and job.finished_at is None
|
| 369 |
):
|
| 370 |
registry.finish_error(
|
| 371 |
job, "Job timed out (no progress in over 30 minutes)."
|
|
|
|
| 373 |
# finish_error queued the terminal event; loop once
|
| 374 |
# more to pick it up and exit through the normal path.
|
| 375 |
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
| 376 |
yield ": ping\n\n"
|
| 377 |
continue
|
| 378 |
+
last_event_time = time.time()
|
| 379 |
yield f"data: {json.dumps(ev)}\n\n"
|
| 380 |
if ev.get("type") in ("done", "error"):
|
| 381 |
return
|
|
@@ -156,6 +156,12 @@ def filter_instructor_frames(
|
|
| 156 |
candidates = [frame.image_path, *frame.alt_image_paths]
|
| 157 |
survivors: list[Path] = []
|
| 158 |
for path in candidates:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 159 |
if is_instructor_frame(path, face_area_threshold, detector):
|
| 160 |
try:
|
| 161 |
path.unlink(missing_ok=True)
|
|
|
|
| 156 |
candidates = [frame.image_path, *frame.alt_image_paths]
|
| 157 |
survivors: list[Path] = []
|
| 158 |
for path in candidates:
|
| 159 |
+
# is_instructor_frame returns False on a missing/unreadable
|
| 160 |
+
# file (cv2.imread is None). That'd let a phantom path slip
|
| 161 |
+
# through into the editor's frame picker, where /alts/ would
|
| 162 |
+
# then 404. Skip non-existent paths up front.
|
| 163 |
+
if not path.exists():
|
| 164 |
+
continue
|
| 165 |
if is_instructor_frame(path, face_area_threshold, detector):
|
| 166 |
try:
|
| 167 |
path.unlink(missing_ok=True)
|
|
@@ -23,9 +23,16 @@ def _preprocess_for_ocr(image_path: Path) -> Image.Image | None:
|
|
| 23 |
return None
|
| 24 |
|
| 25 |
h, w = arr.shape[:2]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
if max(h, w) < 1600:
|
| 27 |
-
scale = 2
|
| 28 |
-
|
|
|
|
|
|
|
|
|
|
| 29 |
|
| 30 |
binarized = cv2.adaptiveThreshold(
|
| 31 |
arr,
|
|
|
|
| 23 |
return None
|
| 24 |
|
| 25 |
h, w = arr.shape[:2]
|
| 26 |
+
# Upscale small frames so Tesseract has more pixels to work with, but
|
| 27 |
+
# cap the post-resize dimension so we don't allocate huge intermediates
|
| 28 |
+
# (adaptiveThreshold then doubles the memory footprint again).
|
| 29 |
+
MAX_DIM = 2400
|
| 30 |
if max(h, w) < 1600:
|
| 31 |
+
scale = min(2, MAX_DIM / max(h, w)) if max(h, w) > 0 else 1
|
| 32 |
+
if scale > 1.0:
|
| 33 |
+
arr = cv2.resize(
|
| 34 |
+
arr, (int(w * scale), int(h * scale)), interpolation=cv2.INTER_CUBIC
|
| 35 |
+
)
|
| 36 |
|
| 37 |
binarized = cv2.adaptiveThreshold(
|
| 38 |
arr,
|