Claude commited on
Commit
9d2a95f
·
1 Parent(s): 7c2d809

Second-pass audit: SSE timing, OCR memory cap, filter robustness, CLI ordering

Browse files

- SSE generator now resets its idle timeout on every event received
rather than measuring from job.created_at, so a healthy long-running
job emitting periodic progress events isn't killed at the 30-min mark.
Also re-checks job.status == "running" right before calling
finish_error to avoid racing a finish_success that landed during the
events.get timeout.
- ocr._preprocess_for_ocr caps the post-upscale max dimension at 2400 px
so an unusual input frame doesn't allocate huge intermediates (the
subsequent adaptiveThreshold doubles the footprint again, and we run
OCR concurrently across job + lazy-alt requests).
- filter_instructor_frames now skips candidate paths that don't exist
on disk. Previously is_instructor_frame returned False on a missing
file (cv2.imread is None), which let phantom paths slip through into
the editor's frame picker where /alts/ would then 404.
- cli build: dump_metadata runs BEFORE rendering, not after, so a
template / disk-write failure doesn't discard the LLM/OCR work the
user just paid for.

Verified along the way: the Anthropic SDK call signature
(client.messages.parse(output_format=...)) is correct against the
installed anthropic 0.100.0 — the audit subagent flagged it without
being able to check the SDK.

Files changed (4) hide show
  1. app/cli.py +7 -4
  2. app/main.py +18 -16
  3. app/pipeline/filter.py +6 -0
  4. app/pipeline/ocr.py +9 -2
app/cli.py CHANGED
@@ -136,6 +136,13 @@ def build(
136
  typer.secho(f"error: {exc}", fg=typer.colors.RED, err=True)
137
  raise typer.Exit(2)
138
 
 
 
 
 
 
 
 
139
  common = dict(
140
  title=title,
141
  segments=result.segments,
@@ -177,10 +184,6 @@ def build(
177
 
178
  typer.echo(f"wrote {output}")
179
 
180
- if export_metadata:
181
- dump_metadata(export_metadata, result.page)
182
- typer.echo(f"wrote {export_metadata}")
183
-
184
 
185
  @app.command("export-metadata")
186
  def export_metadata_cmd(
 
136
  typer.secho(f"error: {exc}", fg=typer.colors.RED, err=True)
137
  raise typer.Exit(2)
138
 
139
+ # Dump metadata BEFORE rendering: a render failure (template bug,
140
+ # disk full mid-write) would otherwise discard the LLM/OCR work
141
+ # the user just paid for.
142
+ if export_metadata:
143
+ dump_metadata(export_metadata, result.page)
144
+ typer.echo(f"wrote {export_metadata}")
145
+
146
  common = dict(
147
  title=title,
148
  segments=result.segments,
 
184
 
185
  typer.echo(f"wrote {output}")
186
 
 
 
 
 
187
 
188
  @app.command("export-metadata")
189
  def export_metadata_cmd(
app/main.py CHANGED
@@ -342,25 +342,30 @@ async def stream_events(job_id: str) -> StreamingResponse:
342
  if not job:
343
  raise HTTPException(404, "Unknown job id.")
344
 
345
- # Hard cap on how long an SSE consumer will wait for an event-less
346
- # job before declaring it dead. Bounds the worst case where the
347
- # worker thread crashed before reaching its try/except (so neither
348
- # finish_success nor finish_error ever ran) — without this guard the
349
- # SSE generator would ping forever.
350
- max_job_seconds = 1800
351
 
352
  async def gen():
353
- # Catch up on already-completed jobs by emitting one final event.
354
  while True:
355
  try:
356
  ev = await asyncio.to_thread(job.events.get, True, 15.0)
357
  except queue.Empty:
358
- # If the worker died silently without ever queuing a
359
- # terminal event, time-bound the wait so the consumer
360
- # doesn't loop forever.
 
 
 
 
 
361
  if (
362
- job.status == "running"
363
- and time.time() - job.created_at > max_job_seconds
 
364
  ):
365
  registry.finish_error(
366
  job, "Job timed out (no progress in over 30 minutes)."
@@ -368,12 +373,9 @@ async def stream_events(job_id: str) -> StreamingResponse:
368
  # finish_error queued the terminal event; loop once
369
  # more to pick it up and exit through the normal path.
370
  continue
371
- # If status flipped without a queued event (defensive),
372
- # exit cleanly instead of pinging forever.
373
- if job.status in ("done", "error"):
374
- return
375
  yield ": ping\n\n"
376
  continue
 
377
  yield f"data: {json.dumps(ev)}\n\n"
378
  if ev.get("type") in ("done", "error"):
379
  return
 
342
  if not job:
343
  raise HTTPException(404, "Unknown job id.")
344
 
345
+ # Hard cap on how long an SSE consumer will wait without ANY event
346
+ # before declaring the job dead. Bounds the worst case where the
347
+ # worker thread crashed before queuing a terminal event — without
348
+ # this the SSE generator would ping forever.
349
+ silence_seconds = 1800
 
350
 
351
  async def gen():
352
+ last_event_time = time.time()
353
  while True:
354
  try:
355
  ev = await asyncio.to_thread(job.events.get, True, 15.0)
356
  except queue.Empty:
357
+ # Status may have flipped without a queued event
358
+ # (defensive); exit cleanly instead of pinging forever.
359
+ if job.status in ("done", "error"):
360
+ return
361
+ # If we haven't seen an event in a long time AND the job
362
+ # claims to still be running, treat it as wedged. Re-check
363
+ # status right before finish_error to avoid racing a
364
+ # finish_success that may have just fired.
365
  if (
366
+ time.time() - last_event_time > silence_seconds
367
+ and job.status == "running"
368
+ and job.finished_at is None
369
  ):
370
  registry.finish_error(
371
  job, "Job timed out (no progress in over 30 minutes)."
 
373
  # finish_error queued the terminal event; loop once
374
  # more to pick it up and exit through the normal path.
375
  continue
 
 
 
 
376
  yield ": ping\n\n"
377
  continue
378
+ last_event_time = time.time()
379
  yield f"data: {json.dumps(ev)}\n\n"
380
  if ev.get("type") in ("done", "error"):
381
  return
app/pipeline/filter.py CHANGED
@@ -156,6 +156,12 @@ def filter_instructor_frames(
156
  candidates = [frame.image_path, *frame.alt_image_paths]
157
  survivors: list[Path] = []
158
  for path in candidates:
 
 
 
 
 
 
159
  if is_instructor_frame(path, face_area_threshold, detector):
160
  try:
161
  path.unlink(missing_ok=True)
 
156
  candidates = [frame.image_path, *frame.alt_image_paths]
157
  survivors: list[Path] = []
158
  for path in candidates:
159
+ # is_instructor_frame returns False on a missing/unreadable
160
+ # file (cv2.imread is None). That'd let a phantom path slip
161
+ # through into the editor's frame picker, where /alts/ would
162
+ # then 404. Skip non-existent paths up front.
163
+ if not path.exists():
164
+ continue
165
  if is_instructor_frame(path, face_area_threshold, detector):
166
  try:
167
  path.unlink(missing_ok=True)
app/pipeline/ocr.py CHANGED
@@ -23,9 +23,16 @@ def _preprocess_for_ocr(image_path: Path) -> Image.Image | None:
23
  return None
24
 
25
  h, w = arr.shape[:2]
 
 
 
 
26
  if max(h, w) < 1600:
27
- scale = 2
28
- arr = cv2.resize(arr, (w * scale, h * scale), interpolation=cv2.INTER_CUBIC)
 
 
 
29
 
30
  binarized = cv2.adaptiveThreshold(
31
  arr,
 
23
  return None
24
 
25
  h, w = arr.shape[:2]
26
+ # Upscale small frames so Tesseract has more pixels to work with, but
27
+ # cap the post-resize dimension so we don't allocate huge intermediates
28
+ # (adaptiveThreshold then doubles the memory footprint again).
29
+ MAX_DIM = 2400
30
  if max(h, w) < 1600:
31
+ scale = min(2, MAX_DIM / max(h, w)) if max(h, w) > 0 else 1
32
+ if scale > 1.0:
33
+ arr = cv2.resize(
34
+ arr, (int(w * scale), int(h * scale)), interpolation=cv2.INTER_CUBIC
35
+ )
36
 
37
  binarized = cv2.adaptiveThreshold(
38
  arr,