Claude commited on
Commit
3602e87
·
1 Parent(s): 0b32c80

llm_extract: cut per-segment API cost and add transient-error retry

Browse files

Three small efficiency wins on the Claude calls that dominate job cost:

* Drop the Tesseract OCR block from the user message when the LLM is
itself producing OCR (toggles.ocr_text). The block was being sent
only to be ignored — ~50-300 wasted input tokens per segment, scaled
with slide text density.

* Build one Anthropic client per extract_all run and reuse it across
all worker threads, instead of constructing a fresh client (and TCP+
TLS handshake) per segment.

* Warm the prompt cache by firing the first segment synchronously
before parallelising the rest. Previously the first wave of 4 worker
threads raced for the cache write and only one won, so cache hits
started at segment 5; now they start at segment 2.

Also wraps both extract_one and extract_topics in a single-retry helper
that catches RateLimitError / APIConnectionError / 5xx-overloaded
statuses, so a transient blip no longer permanently drops a segment's
LLM metadata. 4xx are not retried.

https://claude.ai/code/session_01EM8jiYxdVFy1w3LeJ9GBJg

Files changed (2) hide show
  1. app/pipeline/llm_extract.py +113 -26
  2. tests/test_llm_extract.py +163 -0
app/pipeline/llm_extract.py CHANGED
@@ -13,10 +13,11 @@ from __future__ import annotations
13
  import base64
14
  import logging
15
  import os
 
16
  from concurrent.futures import ThreadPoolExecutor, as_completed
17
  from dataclasses import dataclass
18
  from pathlib import Path
19
- from typing import Callable
20
 
21
  from pydantic import BaseModel, create_model
22
 
@@ -239,11 +240,46 @@ def is_enabled(api_key: str | None = None) -> bool:
239
 
240
 
241
  def _build_client(api_key: str):
242
- """Per-call client, never cached. Caller-supplied keys must not outlive the request."""
 
 
 
 
243
  import anthropic
244
  return anthropic.Anthropic(api_key=api_key)
245
 
246
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
247
 
248
  _MAX_IMAGE_WIDTH = 800 # ~5-6× fewer image tokens vs 1080p; quality loss negligible for slides
249
 
@@ -262,10 +298,27 @@ def _encode_image(image_path: Path) -> tuple[str, str]:
262
  return base64.standard_b64encode(buf.getvalue()).decode("ascii"), "image/jpeg"
263
 
264
 
265
- def _build_user_content(image_path: Path, narration: list[str], ocr_text: str) -> list[dict]:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
266
  data, media_type = _encode_image(image_path)
267
  narration_text = "\n\n".join(p.strip() for p in narration if p.strip()) or "(no narration)"
268
- ocr_block = ocr_text.strip() or "(no OCR text)"
 
 
 
269
  return [
270
  {
271
  "type": "image",
@@ -277,10 +330,7 @@ def _build_user_content(image_path: Path, narration: list[str], ocr_text: str) -
277
  },
278
  {
279
  "type": "text",
280
- "text": (
281
- f"Spoken narration during this segment:\n{narration_text}\n\n"
282
- f"OCR text from the slide (may contain errors):\n{ocr_block}"
283
- ),
284
  },
285
  ]
286
 
@@ -293,21 +343,28 @@ def extract_one(
293
  model: str = DEFAULT_MODEL,
294
  api_key: str | None = None,
295
  toggles: LLMToggles = _DEFAULT_TOGGLES,
 
296
  ) -> ExtractionResult:
297
  """Extract enabled fields for a single segment. Empty on any failure.
298
 
299
  Disabled toggles are dropped from both the prompt and the response schema
300
  so we don't pay for tokens we'd discard. Any per-segment failure logs a
301
  warning and returns _EMPTY — never aborts the pipeline.
 
 
 
 
302
  """
303
  if not toggles.any_enabled:
304
  return _EMPTY
305
  resolved = _resolve_api_key(api_key)
306
- if resolved is None:
307
  return _EMPTY
308
  try:
309
- client = _build_client(resolved)
310
- response = client.messages.parse(
 
 
311
  model=model,
312
  max_tokens=2048,
313
  system=[
@@ -320,7 +377,15 @@ def extract_one(
320
  messages=[
321
  {
322
  "role": "user",
323
- "content": _build_user_content(image_path, narration, ocr_text),
 
 
 
 
 
 
 
 
324
  }
325
  ],
326
  output_format=_build_extraction_model(toggles),
@@ -377,21 +442,42 @@ def extract_all(
377
  on_progress(n, n)
378
  return results
379
 
380
- workers = max(1, min(max_workers, n))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
381
  completed = 0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
382
  with ThreadPoolExecutor(max_workers=workers) as pool:
383
- future_to_i = {
384
- pool.submit(
385
- extract_one,
386
- image_paths[i],
387
- narrations[i] if i < len(narrations) else [],
388
- ocr_texts[i] if i < len(ocr_texts) else "",
389
- model=model,
390
- api_key=api_key,
391
- toggles=toggles,
392
- ): i
393
- for i in range(n)
394
- }
395
  for fut in as_completed(future_to_i):
396
  i = future_to_i[fut]
397
  try:
@@ -790,7 +876,8 @@ def extract_topics(
790
  user_content = content
791
  try:
792
  client = _build_client(resolved)
793
- response = client.messages.parse(
 
794
  model=model,
795
  max_tokens=4096,
796
  system=[
 
13
  import base64
14
  import logging
15
  import os
16
+ import time
17
  from concurrent.futures import ThreadPoolExecutor, as_completed
18
  from dataclasses import dataclass
19
  from pathlib import Path
20
+ from typing import Any, Callable
21
 
22
  from pydantic import BaseModel, create_model
23
 
 
240
 
241
 
242
  def _build_client(api_key: str):
243
+ """Per-job client. Cached for the duration of a single extract_all
244
+ run so connection pools survive across segments; never cached
245
+ process-wide because caller-supplied keys must not outlive the
246
+ request that brought them in.
247
+ """
248
  import anthropic
249
  return anthropic.Anthropic(api_key=api_key)
250
 
251
 
252
+ # HTTP statuses we'll retry once with a short backoff. 529 is Anthropic's
253
+ # "overloaded — try again in a moment"; 502/503/504 are generic gateway
254
+ # blips. Other 4xx (auth, schema, content) are user errors — never retry.
255
+ _RETRY_STATUSES = frozenset({502, 503, 504, 529})
256
+ _RETRY_BACKOFF_SECONDS = 1.5
257
+
258
+
259
+ def _parse_with_retry(client, **kwargs):
260
+ """client.messages.parse(**kwargs) with one retry on transient errors.
261
+
262
+ Without this, a single 529 or a network blip permanently drops one
263
+ segment's LLM metadata (extract_one catches and returns _EMPTY). One
264
+ retry covers the typical "overloaded for a few hundred ms" case
265
+ without exposing the caller to multi-minute hangs.
266
+ """
267
+ import anthropic
268
+ transient = (
269
+ anthropic.RateLimitError,
270
+ anthropic.APIConnectionError,
271
+ )
272
+ try:
273
+ return client.messages.parse(**kwargs)
274
+ except transient:
275
+ time.sleep(_RETRY_BACKOFF_SECONDS)
276
+ return client.messages.parse(**kwargs)
277
+ except anthropic.APIStatusError as exc:
278
+ if exc.status_code not in _RETRY_STATUSES:
279
+ raise
280
+ time.sleep(_RETRY_BACKOFF_SECONDS)
281
+ return client.messages.parse(**kwargs)
282
+
283
 
284
  _MAX_IMAGE_WIDTH = 800 # ~5-6× fewer image tokens vs 1080p; quality loss negligible for slides
285
 
 
298
  return base64.standard_b64encode(buf.getvalue()).decode("ascii"), "image/jpeg"
299
 
300
 
301
+ def _build_user_content(
302
+ image_path: Path,
303
+ narration: list[str],
304
+ ocr_text: str,
305
+ *,
306
+ include_ocr: bool,
307
+ ) -> list[dict]:
308
+ """Build the user-message content for one segment.
309
+
310
+ When `include_ocr` is False, the Tesseract OCR block is omitted —
311
+ used when the caller is asking Claude to produce its own OCR for
312
+ this segment (toggles.ocr_text), so the Tesseract pass would be
313
+ sent only to be ignored. Saves ~50-300 input tokens per segment
314
+ depending on slide text density.
315
+ """
316
  data, media_type = _encode_image(image_path)
317
  narration_text = "\n\n".join(p.strip() for p in narration if p.strip()) or "(no narration)"
318
+ text_block = f"Spoken narration during this segment:\n{narration_text}"
319
+ if include_ocr:
320
+ ocr_block = ocr_text.strip() or "(no OCR text)"
321
+ text_block += f"\n\nOCR text from the slide (may contain errors):\n{ocr_block}"
322
  return [
323
  {
324
  "type": "image",
 
330
  },
331
  {
332
  "type": "text",
333
+ "text": text_block,
 
 
 
334
  },
335
  ]
336
 
 
343
  model: str = DEFAULT_MODEL,
344
  api_key: str | None = None,
345
  toggles: LLMToggles = _DEFAULT_TOGGLES,
346
+ client: Any | None = None,
347
  ) -> ExtractionResult:
348
  """Extract enabled fields for a single segment. Empty on any failure.
349
 
350
  Disabled toggles are dropped from both the prompt and the response schema
351
  so we don't pay for tokens we'd discard. Any per-segment failure logs a
352
  warning and returns _EMPTY — never aborts the pipeline.
353
+
354
+ `client` is an optional pre-built anthropic.Anthropic instance. Pass
355
+ one to share a connection pool across many segments; if omitted, a
356
+ one-shot client is constructed from `api_key`.
357
  """
358
  if not toggles.any_enabled:
359
  return _EMPTY
360
  resolved = _resolve_api_key(api_key)
361
+ if client is None and resolved is None:
362
  return _EMPTY
363
  try:
364
+ if client is None:
365
+ client = _build_client(resolved) # type: ignore[arg-type]
366
+ response = _parse_with_retry(
367
+ client,
368
  model=model,
369
  max_tokens=2048,
370
  system=[
 
377
  messages=[
378
  {
379
  "role": "user",
380
+ "content": _build_user_content(
381
+ image_path,
382
+ narration,
383
+ ocr_text,
384
+ # When Claude is producing OCR itself, don't also
385
+ # send Tesseract's (possibly garbled) version —
386
+ # it'd be ignored and we'd pay input tokens for it.
387
+ include_ocr=not toggles.ocr_text,
388
+ ),
389
  }
390
  ],
391
  output_format=_build_extraction_model(toggles),
 
442
  on_progress(n, n)
443
  return results
444
 
445
+ # One client, one connection pool, shared across all worker threads —
446
+ # avoids N TCP+TLS handshakes for an N-segment job.
447
+ resolved = _resolve_api_key(api_key)
448
+ client = _build_client(resolved) # type: ignore[arg-type]
449
+
450
+ def _call(i: int) -> ExtractionResult:
451
+ return extract_one(
452
+ image_paths[i],
453
+ narrations[i] if i < len(narrations) else [],
454
+ ocr_texts[i] if i < len(ocr_texts) else "",
455
+ model=model,
456
+ api_key=resolved,
457
+ toggles=toggles,
458
+ client=client,
459
+ )
460
+
461
  completed = 0
462
+ # Warm the prompt cache with the first call before parallelising.
463
+ # The first request writes the cache; if we let the whole first wave
464
+ # race in parallel, only one wins the write and the other workers-1
465
+ # calls pay the cache-write surcharge for no benefit. Serializing
466
+ # the first call alone makes every subsequent call a cache read.
467
+ try:
468
+ results[0] = _call(0)
469
+ except Exception as exc: # pragma: no cover
470
+ log.warning("LLM extraction worker raised: %s", exc)
471
+ results[0] = _EMPTY
472
+ completed += 1
473
+ if on_progress:
474
+ on_progress(completed, n)
475
+ if n == 1:
476
+ return results
477
+
478
+ workers = max(1, min(max_workers, n - 1))
479
  with ThreadPoolExecutor(max_workers=workers) as pool:
480
+ future_to_i = {pool.submit(_call, i): i for i in range(1, n)}
 
 
 
 
 
 
 
 
 
 
 
481
  for fut in as_completed(future_to_i):
482
  i = future_to_i[fut]
483
  try:
 
876
  user_content = content
877
  try:
878
  client = _build_client(resolved)
879
+ response = _parse_with_retry(
880
+ client,
881
  model=model,
882
  max_tokens=4096,
883
  system=[
tests/test_llm_extract.py ADDED
@@ -0,0 +1,163 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Tests for the small efficiency-critical helpers in llm_extract.
2
+
3
+ These pin the behaviour that:
4
+ - the Tesseract OCR block is dropped from the user message when the
5
+ LLM is the OCR source (saves input tokens on every per-segment call);
6
+ - a single transient error is retried once before giving up;
7
+ - permanent errors (4xx) are NOT retried.
8
+ """
9
+ from __future__ import annotations
10
+
11
+ import io
12
+ from pathlib import Path
13
+ from unittest.mock import MagicMock
14
+
15
+ import pytest
16
+ from PIL import Image
17
+
18
+ from app.pipeline import llm_extract
19
+ from app.pipeline.llm_extract import (
20
+ LLMToggles,
21
+ _build_user_content,
22
+ _parse_with_retry,
23
+ extract_all,
24
+ )
25
+
26
+
27
+ @pytest.fixture
28
+ def tiny_image(tmp_path: Path) -> Path:
29
+ """A 16×16 white JPEG — just enough that _encode_image succeeds."""
30
+ p = tmp_path / "frame.jpg"
31
+ Image.new("RGB", (16, 16), "white").save(p, "JPEG")
32
+ return p
33
+
34
+
35
+ def _text_block(content: list[dict]) -> str:
36
+ return next(b["text"] for b in content if b["type"] == "text")
37
+
38
+
39
+ def test_build_user_content_includes_ocr_by_default(tiny_image: Path):
40
+ content = _build_user_content(
41
+ tiny_image, ["narration here"], "OCR FROM SLIDE", include_ocr=True
42
+ )
43
+ text = _text_block(content)
44
+ assert "narration here" in text
45
+ assert "OCR FROM SLIDE" in text
46
+ assert "OCR text from the slide" in text
47
+
48
+
49
+ def test_build_user_content_omits_ocr_when_llm_is_doing_ocr(tiny_image: Path):
50
+ """When toggles.ocr_text is on, Claude reads the slide directly —
51
+ sending Tesseract's version too just wastes input tokens."""
52
+ content = _build_user_content(
53
+ tiny_image, ["narration here"], "OCR FROM SLIDE", include_ocr=False
54
+ )
55
+ text = _text_block(content)
56
+ assert "narration here" in text
57
+ assert "OCR FROM SLIDE" not in text
58
+ assert "OCR text from the slide" not in text
59
+
60
+
61
+ def test_build_user_content_uses_placeholder_when_no_narration(tiny_image: Path):
62
+ content = _build_user_content(tiny_image, [], "some ocr", include_ocr=True)
63
+ text = _text_block(content)
64
+ assert "(no narration)" in text
65
+
66
+
67
+ def _fake_anthropic_module(monkeypatch, *, transient_exc=None, status_exc=None):
68
+ """Install a fake `anthropic` module exposing just the exception
69
+ classes _parse_with_retry imports. Lets us drive its retry logic
70
+ without the real SDK installed.
71
+ """
72
+ import sys
73
+ import types
74
+
75
+ mod = types.ModuleType("anthropic")
76
+
77
+ class RateLimitError(Exception):
78
+ pass
79
+
80
+ class APIConnectionError(Exception):
81
+ pass
82
+
83
+ class APIStatusError(Exception):
84
+ def __init__(self, status_code: int, message: str = ""):
85
+ super().__init__(message)
86
+ self.status_code = status_code
87
+
88
+ mod.RateLimitError = RateLimitError
89
+ mod.APIConnectionError = APIConnectionError
90
+ mod.APIStatusError = APIStatusError
91
+ monkeypatch.setitem(sys.modules, "anthropic", mod)
92
+ # Speed: don't actually sleep during retry
93
+ monkeypatch.setattr(llm_extract, "_RETRY_BACKOFF_SECONDS", 0)
94
+ return mod
95
+
96
+
97
+ def test_parse_with_retry_retries_once_on_rate_limit(monkeypatch):
98
+ mod = _fake_anthropic_module(monkeypatch)
99
+ client = MagicMock()
100
+ client.messages.parse.side_effect = [mod.RateLimitError("slow down"), "ok"]
101
+ result = _parse_with_retry(client, model="m")
102
+ assert result == "ok"
103
+ assert client.messages.parse.call_count == 2
104
+
105
+
106
+ def test_parse_with_retry_retries_on_overloaded_status(monkeypatch):
107
+ mod = _fake_anthropic_module(monkeypatch)
108
+ client = MagicMock()
109
+ client.messages.parse.side_effect = [
110
+ mod.APIStatusError(529, "overloaded"),
111
+ "ok",
112
+ ]
113
+ assert _parse_with_retry(client, model="m") == "ok"
114
+ assert client.messages.parse.call_count == 2
115
+
116
+
117
+ def test_parse_with_retry_does_not_retry_4xx(monkeypatch):
118
+ mod = _fake_anthropic_module(monkeypatch)
119
+ client = MagicMock()
120
+ client.messages.parse.side_effect = mod.APIStatusError(400, "bad request")
121
+ with pytest.raises(mod.APIStatusError):
122
+ _parse_with_retry(client, model="m")
123
+ assert client.messages.parse.call_count == 1
124
+
125
+
126
+ def test_parse_with_retry_reraises_after_second_failure(monkeypatch):
127
+ mod = _fake_anthropic_module(monkeypatch)
128
+ client = MagicMock()
129
+ client.messages.parse.side_effect = [
130
+ mod.RateLimitError("slow down"),
131
+ mod.RateLimitError("still slow"),
132
+ ]
133
+ with pytest.raises(mod.RateLimitError):
134
+ _parse_with_retry(client, model="m")
135
+ assert client.messages.parse.call_count == 2
136
+
137
+
138
+ def test_extract_all_returns_empty_without_api_key():
139
+ """No key → no client built, no network attempted, every segment
140
+ gets an empty result. (Sanity: the warm-cache refactor must not
141
+ regress the early-return path.)"""
142
+ results = extract_all(
143
+ [Path("frame.jpg")],
144
+ [["narration"]],
145
+ ["ocr"],
146
+ api_key=None,
147
+ toggles=LLMToggles(alt_text=True),
148
+ )
149
+ assert len(results) == 1
150
+ assert results[0].alt_text == ""
151
+ assert results[0].key_terms == []
152
+
153
+
154
+ def test_extract_all_no_op_when_no_toggles_enabled():
155
+ results = extract_all(
156
+ [Path("frame.jpg")],
157
+ [["narration"]],
158
+ ["ocr"],
159
+ api_key="sk-stub",
160
+ toggles=LLMToggles(), # all off
161
+ )
162
+ assert len(results) == 1
163
+ assert results[0] is llm_extract._EMPTY