Video2Guide / tests /test_llm_extract.py
Claude
Claude Opus 4.8
Add AI slide-to-transcript alignment for untimestamped images
e646192
Raw History Blame Contribute Delete
14.3 kB
"""Tests for the small efficiency-critical helpers in llm_extract.
These pin the behaviour that:
- the Tesseract OCR block is dropped from the user message when the
LLM is the OCR source (saves input tokens on every per-segment call);
- a single transient error is retried once before giving up;
- permanent errors (4xx) are NOT retried.
"""
from __future__ import annotations
import io
import base64
from pathlib import Path
from unittest.mock import MagicMock
import pytest
from PIL import Image
from app.pipeline import llm_extract
from app.pipeline.llm_extract import (
LLMToggles,
_build_user_content,
_create_with_retry,
extract_all,
)
@pytest.fixture
def tiny_image(tmp_path: Path) -> Path:
"""A 16×16 white JPEG — just enough that _encode_image succeeds."""
p = tmp_path / "frame.jpg"
Image.new("RGB", (16, 16), "white").save(p, "JPEG")
return p
def _text_block(content: list[dict]) -> str:
return next(b["text"] for b in content if b["type"] == "text")
def test_resolve_base_url_routes_parley_keys(monkeypatch):
"""A Parley key goes to the Parley proxy; a normal Anthropic key uses
the SDK default (None) unless ANTHROPIC_BASE_URL is explicitly set."""
monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False)
assert llm_extract._resolve_base_url("sk-parley-v1-abc") == llm_extract._PARLEY_BASE_URL
assert llm_extract._resolve_base_url("sk-ant-abc") is None
def test_resolve_base_url_env_override_for_non_parley(monkeypatch):
"""An explicit ANTHROPIC_BASE_URL routes a normal key to a custom proxy,
but never overrides Parley-key routing."""
monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://proxy.example/ ")
assert llm_extract._resolve_base_url("sk-ant-abc") == "https://proxy.example/"
# Parley keys always go to Parley regardless of the env override.
assert llm_extract._resolve_base_url("sk-parley-v1-abc") == llm_extract._PARLEY_BASE_URL
def test_encode_image_handles_rgba_png(tmp_path: Path):
"""PNG slide exports are frequently RGBA; JPEG has no alpha channel,
so encoding must flatten rather than raise 'cannot write mode RGBA as
JPEG'. Regression for the images-input mode."""
p = tmp_path / "00-05-30.png"
# Semi-transparent red on a transparent canvas.
Image.new("RGBA", (24, 18), (255, 0, 0, 128)).save(p, "PNG")
data, media_type = llm_extract._encode_image(p)
assert media_type == "image/jpeg"
# Round-trips to a valid JPEG.
import base64 as _b64
decoded = _b64.b64decode(data)
img = Image.open(io.BytesIO(decoded))
assert img.format == "JPEG"
assert img.mode == "RGB"
def test_encode_image_handles_palette_png_with_transparency(tmp_path: Path):
"""Palette-mode ('P') PNGs with a transparency entry also can't go
straight to JPEG; they must flatten cleanly too."""
p = tmp_path / "00-01-00.png"
base = Image.new("RGBA", (20, 20), (0, 128, 0, 200))
base.convert("P", palette=Image.ADAPTIVE).save(p, "PNG", transparency=0)
data, media_type = llm_extract._encode_image(p)
assert media_type == "image/jpeg"
img = Image.open(io.BytesIO(base64.b64decode(data)))
assert img.mode == "RGB"
def test_extract_json_object_plain():
assert llm_extract._extract_json_object('{"a": 1, "b": "x"}') == {"a": 1, "b": "x"}
def test_extract_json_object_strips_markdown_fence():
"""The Parley proxy returns JSON wrapped in a ```json fence, which the
strict .parse path rejected. It must parse cleanly now."""
text = '```json\n{\n "section_title": "Intro",\n "math_equations": []\n}\n```'
assert llm_extract._extract_json_object(text) == {
"section_title": "Intro",
"math_equations": [],
}
def test_extract_json_object_ignores_surrounding_prose():
text = 'Here is the result:\n{"description": "A slide."}\nHope that helps!'
assert llm_extract._extract_json_object(text) == {"description": "A slide."}
def test_extract_one_retries_once_on_malformed_json(tmp_path, monkeypatch):
"""A malformed-JSON response (e.g. an unescaped quote) is retried once;
the model usually returns clean JSON the second time."""
_fake_anthropic_module(monkeypatch)
from unittest.mock import MagicMock
img = tmp_path / "slide 1.jpg"
Image.new("RGB", (16, 16), "white").save(img, "JPEG")
def resp(text):
blk = MagicMock(); blk.type = "text"; blk.text = text
r = MagicMock(); r.content = [blk]; return r
client = MagicMock()
client.messages.create.side_effect = [
resp('{"section_title": "The "big" idea"}'), # invalid JSON
resp('{"section_title": "Projecting Future Climate"}'), # valid
]
res = llm_extract.extract_one(
img, [], "", api_key="sk-ant-x",
toggles=LLMToggles(section_title=True), client=client,
)
assert res.section_title == "Projecting Future Climate"
assert client.messages.create.call_count == 2
def test_extract_json_object_tolerates_control_chars():
"""strict=False lets a literal newline inside a string value parse."""
assert llm_extract._extract_json_object('{"a": "line1\nline2"}') == {"a": "line1\nline2"}
def test_extract_json_object_raises_on_no_object():
"""A purely conversational reply ('I'm ready to analyze…') has no JSON;
the caller catches this and returns empty for that segment."""
with pytest.raises(ValueError):
llm_extract._extract_json_object("I'm ready to analyze — send me the image.")
def test_build_user_content_includes_ocr_by_default(tiny_image: Path):
content = _build_user_content(
tiny_image, ["narration here"], "OCR FROM SLIDE", include_ocr=True
)
text = _text_block(content)
assert "narration here" in text
assert "OCR FROM SLIDE" in text
assert "OCR text from the slide" in text
def test_build_user_content_omits_ocr_when_llm_is_doing_ocr(tiny_image: Path):
"""When toggles.ocr_text is on, Claude reads the slide directly —
sending Tesseract's version too just wastes input tokens."""
content = _build_user_content(
tiny_image, ["narration here"], "OCR FROM SLIDE", include_ocr=False
)
text = _text_block(content)
assert "narration here" in text
assert "OCR FROM SLIDE" not in text
assert "OCR text from the slide" not in text
def test_build_user_content_uses_placeholder_when_no_narration(tiny_image: Path):
content = _build_user_content(tiny_image, [], "some ocr", include_ocr=True)
text = _text_block(content)
assert "(no narration)" in text
def _fake_anthropic_module(monkeypatch, *, transient_exc=None, status_exc=None):
"""Install a fake `anthropic` module exposing just the exception
classes _create_with_retry imports. Lets us drive its retry logic
without the real SDK installed.
"""
import sys
import types
mod = types.ModuleType("anthropic")
class RateLimitError(Exception):
pass
class APIConnectionError(Exception):
pass
class APIStatusError(Exception):
def __init__(self, status_code: int, message: str = ""):
super().__init__(message)
self.status_code = status_code
mod.RateLimitError = RateLimitError
mod.APIConnectionError = APIConnectionError
mod.APIStatusError = APIStatusError
monkeypatch.setitem(sys.modules, "anthropic", mod)
# Speed: don't actually sleep during retry
monkeypatch.setattr(llm_extract, "_RETRY_BACKOFF_SECONDS", 0)
return mod
def test_create_with_retry_retries_once_on_rate_limit(monkeypatch):
mod = _fake_anthropic_module(monkeypatch)
client = MagicMock()
client.messages.create.side_effect = [mod.RateLimitError("slow down"), "ok"]
result = _create_with_retry(client, model="m")
assert result == "ok"
assert client.messages.create.call_count == 2
def test_create_with_retry_retries_on_overloaded_status(monkeypatch):
mod = _fake_anthropic_module(monkeypatch)
client = MagicMock()
client.messages.create.side_effect = [
mod.APIStatusError(529, "overloaded"),
"ok",
]
assert _create_with_retry(client, model="m") == "ok"
assert client.messages.create.call_count == 2
def test_create_with_retry_does_not_retry_4xx(monkeypatch):
mod = _fake_anthropic_module(monkeypatch)
client = MagicMock()
client.messages.create.side_effect = mod.APIStatusError(400, "bad request")
with pytest.raises(mod.APIStatusError):
_create_with_retry(client, model="m")
assert client.messages.create.call_count == 1
def test_create_with_retry_reraises_after_second_failure(monkeypatch):
mod = _fake_anthropic_module(monkeypatch)
client = MagicMock()
client.messages.create.side_effect = [
mod.RateLimitError("slow down"),
mod.RateLimitError("still slow"),
]
with pytest.raises(mod.RateLimitError):
_create_with_retry(client, model="m")
assert client.messages.create.call_count == 2
def test_extract_all_returns_empty_without_api_key():
"""No key → no client built, no network attempted, every segment
gets an empty result. (Sanity: the warm-cache refactor must not
regress the early-return path.)"""
results = extract_all(
[Path("frame.jpg")],
[["narration"]],
["ocr"],
api_key=None,
toggles=LLMToggles(alt_text=True),
)
assert len(results) == 1
assert results[0].alt_text == ""
assert results[0].math_equations == []
def test_description_toggle_adds_prompt_field_and_schema_slot():
"""The description toggle must appear in the system prompt AND in
the structured-output schema so Claude actually returns it."""
from app.pipeline.llm_extract import _build_extraction_model, _build_system_prompt
toggles = LLMToggles(description=True)
assert toggles.any_enabled is True
schema = _build_extraction_model(toggles)
assert "description" in schema.model_fields
prompt = _build_system_prompt(toggles)
assert "description" in prompt
assert "long-form image description" in prompt
def test_description_toggle_skipped_when_off():
"""The description toggle must not bleed into the prompt or schema
when it's off — that'd waste output tokens on every call."""
from app.pipeline.llm_extract import _build_extraction_model, _build_system_prompt
toggles = LLMToggles(alt_text=True) # description left default False
schema = _build_extraction_model(toggles)
assert "description" not in schema.model_fields
prompt = _build_system_prompt(toggles)
assert "description" not in prompt
def test_extract_all_degrades_when_client_construction_fails(monkeypatch, tmp_path):
"""extract_all's contract is that it always returns — a missing/broken
anthropic SDK must degrade to empty results, not abort the pipeline.
The shared client is built outside the per-segment loop, so its
construction must be guarded too."""
img = tmp_path / "frame.jpg"
Image.new("RGB", (8, 8), "white").save(img, "JPEG")
def _boom(_key):
raise ImportError("No module named 'anthropic'")
monkeypatch.setattr(llm_extract, "_build_client", _boom)
progress = []
results = llm_extract.extract_all(
[img],
[["narration"]],
[""],
api_key="sk-stub",
toggles=LLMToggles(alt_text=True),
on_progress=lambda done, total: progress.append((done, total)),
)
assert len(results) == 1
assert results[0] is llm_extract._EMPTY
# Progress is still reported as complete so the UI doesn't hang.
assert progress and progress[-1] == (1, 1)
def _cue(start, end, text):
from app.pipeline.transcript import Cue
return Cue(start=start, end=end, text=text)
def test_align_slides_returns_clean_non_decreasing_starts(monkeypatch):
"""Alignment maps ordered slides to transcript times: the model's raw
output is clamped to the transcript span, forced non-decreasing, and the
first slide pinned to the transcript's first cue."""
from app.pipeline.llm_extract import align_slides_to_transcript
_fake_anthropic_module(monkeypatch)
cues = [_cue(10.0, 20.0, "intro"), _cue(20.0, 40.0, "middle"), _cue(40.0, 60.0, "end")]
def resp(text):
blk = MagicMock(); blk.type = "text"; blk.text = text
r = MagicMock(); r.content = [blk]; return r
client = MagicMock()
# Raw model output: out-of-order + out-of-range values that must be cleaned.
client.messages.create.return_value = resp('{"slide_starts": [0, 5, 3, 999]}')
monkeypatch.setattr(llm_extract, "_build_client", lambda key: client)
starts = align_slides_to_transcript(
cues, ["a", "b", "c", "d"], api_key="sk-ant-x"
)
assert len(starts) == 4
assert starts == sorted(starts) # non-decreasing
assert starts[0] == 10.0 # pinned to first cue start
assert all(10.0 <= s <= 60.0 for s in starts) # within span
def test_align_slides_raises_without_key(monkeypatch):
from app.pipeline.llm_extract import SlideAlignError, align_slides_to_transcript
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
monkeypatch.delenv("PARLEY_API_KEY", raising=False)
cues = [_cue(0.0, 10.0, "x")]
with pytest.raises(SlideAlignError):
align_slides_to_transcript(cues, ["a"], api_key=None)
def test_align_slides_raises_on_count_mismatch(monkeypatch):
from app.pipeline.llm_extract import SlideAlignError, align_slides_to_transcript
_fake_anthropic_module(monkeypatch)
cues = [_cue(0.0, 10.0, "x"), _cue(10.0, 20.0, "y")]
def resp(text):
blk = MagicMock(); blk.type = "text"; blk.text = text
r = MagicMock(); r.content = [blk]; return r
client = MagicMock()
client.messages.create.return_value = resp('{"slide_starts": [0]}') # 1 for 2 slides
monkeypatch.setattr(llm_extract, "_build_client", lambda key: client)
with pytest.raises(SlideAlignError):
align_slides_to_transcript(cues, ["a", "b"], api_key="sk-ant-x")
def test_extract_all_no_op_when_no_toggles_enabled():
results = extract_all(
[Path("frame.jpg")],
[["narration"]],
["ocr"],
api_key="sk-stub",
toggles=LLMToggles(), # all off
)
assert len(results) == 1
assert results[0] is llm_extract._EMPTY