Spaces:
Sleeping
Sleeping
Download tests/test_llm_extract.py from joemartis/Video2Guide: direct link, hf CLI and curl.
- Browser
- Download file 14.3 kB
-
https://huggingface.co/spaces/joemartis/Video2Guide/resolve/main/tests/test_llm_extract.py
- Command line
-
hf download hf://spaces/joemartis/Video2Guide/tests/test_llm_extract.py
-
curl -L -o test_llm_extract.py https://huggingface.co/spaces/joemartis/Video2Guide/resolve/main/tests/test_llm_extract.py
14.3 kB
| """Tests for the small efficiency-critical helpers in llm_extract. | |
| These pin the behaviour that: | |
| - the Tesseract OCR block is dropped from the user message when the | |
| LLM is the OCR source (saves input tokens on every per-segment call); | |
| - a single transient error is retried once before giving up; | |
| - permanent errors (4xx) are NOT retried. | |
| """ | |
| from __future__ import annotations | |
| import io | |
| import base64 | |
| from pathlib import Path | |
| from unittest.mock import MagicMock | |
| import pytest | |
| from PIL import Image | |
| from app.pipeline import llm_extract | |
| from app.pipeline.llm_extract import ( | |
| LLMToggles, | |
| _build_user_content, | |
| _create_with_retry, | |
| extract_all, | |
| ) | |
| def tiny_image(tmp_path: Path) -> Path: | |
| """A 16×16 white JPEG — just enough that _encode_image succeeds.""" | |
| p = tmp_path / "frame.jpg" | |
| Image.new("RGB", (16, 16), "white").save(p, "JPEG") | |
| return p | |
| def _text_block(content: list[dict]) -> str: | |
| return next(b["text"] for b in content if b["type"] == "text") | |
| def test_resolve_base_url_routes_parley_keys(monkeypatch): | |
| """A Parley key goes to the Parley proxy; a normal Anthropic key uses | |
| the SDK default (None) unless ANTHROPIC_BASE_URL is explicitly set.""" | |
| monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) | |
| assert llm_extract._resolve_base_url("sk-parley-v1-abc") == llm_extract._PARLEY_BASE_URL | |
| assert llm_extract._resolve_base_url("sk-ant-abc") is None | |
| def test_resolve_base_url_env_override_for_non_parley(monkeypatch): | |
| """An explicit ANTHROPIC_BASE_URL routes a normal key to a custom proxy, | |
| but never overrides Parley-key routing.""" | |
| monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://proxy.example/ ") | |
| assert llm_extract._resolve_base_url("sk-ant-abc") == "https://proxy.example/" | |
| # Parley keys always go to Parley regardless of the env override. | |
| assert llm_extract._resolve_base_url("sk-parley-v1-abc") == llm_extract._PARLEY_BASE_URL | |
| def test_encode_image_handles_rgba_png(tmp_path: Path): | |
| """PNG slide exports are frequently RGBA; JPEG has no alpha channel, | |
| so encoding must flatten rather than raise 'cannot write mode RGBA as | |
| JPEG'. Regression for the images-input mode.""" | |
| p = tmp_path / "00-05-30.png" | |
| # Semi-transparent red on a transparent canvas. | |
| Image.new("RGBA", (24, 18), (255, 0, 0, 128)).save(p, "PNG") | |
| data, media_type = llm_extract._encode_image(p) | |
| assert media_type == "image/jpeg" | |
| # Round-trips to a valid JPEG. | |
| import base64 as _b64 | |
| decoded = _b64.b64decode(data) | |
| img = Image.open(io.BytesIO(decoded)) | |
| assert img.format == "JPEG" | |
| assert img.mode == "RGB" | |
| def test_encode_image_handles_palette_png_with_transparency(tmp_path: Path): | |
| """Palette-mode ('P') PNGs with a transparency entry also can't go | |
| straight to JPEG; they must flatten cleanly too.""" | |
| p = tmp_path / "00-01-00.png" | |
| base = Image.new("RGBA", (20, 20), (0, 128, 0, 200)) | |
| base.convert("P", palette=Image.ADAPTIVE).save(p, "PNG", transparency=0) | |
| data, media_type = llm_extract._encode_image(p) | |
| assert media_type == "image/jpeg" | |
| img = Image.open(io.BytesIO(base64.b64decode(data))) | |
| assert img.mode == "RGB" | |
| def test_extract_json_object_plain(): | |
| assert llm_extract._extract_json_object('{"a": 1, "b": "x"}') == {"a": 1, "b": "x"} | |
| def test_extract_json_object_strips_markdown_fence(): | |
| """The Parley proxy returns JSON wrapped in a ```json fence, which the | |
| strict .parse path rejected. It must parse cleanly now.""" | |
| text = '```json\n{\n "section_title": "Intro",\n "math_equations": []\n}\n```' | |
| assert llm_extract._extract_json_object(text) == { | |
| "section_title": "Intro", | |
| "math_equations": [], | |
| } | |
| def test_extract_json_object_ignores_surrounding_prose(): | |
| text = 'Here is the result:\n{"description": "A slide."}\nHope that helps!' | |
| assert llm_extract._extract_json_object(text) == {"description": "A slide."} | |
| def test_extract_one_retries_once_on_malformed_json(tmp_path, monkeypatch): | |
| """A malformed-JSON response (e.g. an unescaped quote) is retried once; | |
| the model usually returns clean JSON the second time.""" | |
| _fake_anthropic_module(monkeypatch) | |
| from unittest.mock import MagicMock | |
| img = tmp_path / "slide 1.jpg" | |
| Image.new("RGB", (16, 16), "white").save(img, "JPEG") | |
| def resp(text): | |
| blk = MagicMock(); blk.type = "text"; blk.text = text | |
| r = MagicMock(); r.content = [blk]; return r | |
| client = MagicMock() | |
| client.messages.create.side_effect = [ | |
| resp('{"section_title": "The "big" idea"}'), # invalid JSON | |
| resp('{"section_title": "Projecting Future Climate"}'), # valid | |
| ] | |
| res = llm_extract.extract_one( | |
| img, [], "", api_key="sk-ant-x", | |
| toggles=LLMToggles(section_title=True), client=client, | |
| ) | |
| assert res.section_title == "Projecting Future Climate" | |
| assert client.messages.create.call_count == 2 | |
| def test_extract_json_object_tolerates_control_chars(): | |
| """strict=False lets a literal newline inside a string value parse.""" | |
| assert llm_extract._extract_json_object('{"a": "line1\nline2"}') == {"a": "line1\nline2"} | |
| def test_extract_json_object_raises_on_no_object(): | |
| """A purely conversational reply ('I'm ready to analyze…') has no JSON; | |
| the caller catches this and returns empty for that segment.""" | |
| with pytest.raises(ValueError): | |
| llm_extract._extract_json_object("I'm ready to analyze — send me the image.") | |
| def test_build_user_content_includes_ocr_by_default(tiny_image: Path): | |
| content = _build_user_content( | |
| tiny_image, ["narration here"], "OCR FROM SLIDE", include_ocr=True | |
| ) | |
| text = _text_block(content) | |
| assert "narration here" in text | |
| assert "OCR FROM SLIDE" in text | |
| assert "OCR text from the slide" in text | |
| def test_build_user_content_omits_ocr_when_llm_is_doing_ocr(tiny_image: Path): | |
| """When toggles.ocr_text is on, Claude reads the slide directly — | |
| sending Tesseract's version too just wastes input tokens.""" | |
| content = _build_user_content( | |
| tiny_image, ["narration here"], "OCR FROM SLIDE", include_ocr=False | |
| ) | |
| text = _text_block(content) | |
| assert "narration here" in text | |
| assert "OCR FROM SLIDE" not in text | |
| assert "OCR text from the slide" not in text | |
| def test_build_user_content_uses_placeholder_when_no_narration(tiny_image: Path): | |
| content = _build_user_content(tiny_image, [], "some ocr", include_ocr=True) | |
| text = _text_block(content) | |
| assert "(no narration)" in text | |
| def _fake_anthropic_module(monkeypatch, *, transient_exc=None, status_exc=None): | |
| """Install a fake `anthropic` module exposing just the exception | |
| classes _create_with_retry imports. Lets us drive its retry logic | |
| without the real SDK installed. | |
| """ | |
| import sys | |
| import types | |
| mod = types.ModuleType("anthropic") | |
| class RateLimitError(Exception): | |
| pass | |
| class APIConnectionError(Exception): | |
| pass | |
| class APIStatusError(Exception): | |
| def __init__(self, status_code: int, message: str = ""): | |
| super().__init__(message) | |
| self.status_code = status_code | |
| mod.RateLimitError = RateLimitError | |
| mod.APIConnectionError = APIConnectionError | |
| mod.APIStatusError = APIStatusError | |
| monkeypatch.setitem(sys.modules, "anthropic", mod) | |
| # Speed: don't actually sleep during retry | |
| monkeypatch.setattr(llm_extract, "_RETRY_BACKOFF_SECONDS", 0) | |
| return mod | |
| def test_create_with_retry_retries_once_on_rate_limit(monkeypatch): | |
| mod = _fake_anthropic_module(monkeypatch) | |
| client = MagicMock() | |
| client.messages.create.side_effect = [mod.RateLimitError("slow down"), "ok"] | |
| result = _create_with_retry(client, model="m") | |
| assert result == "ok" | |
| assert client.messages.create.call_count == 2 | |
| def test_create_with_retry_retries_on_overloaded_status(monkeypatch): | |
| mod = _fake_anthropic_module(monkeypatch) | |
| client = MagicMock() | |
| client.messages.create.side_effect = [ | |
| mod.APIStatusError(529, "overloaded"), | |
| "ok", | |
| ] | |
| assert _create_with_retry(client, model="m") == "ok" | |
| assert client.messages.create.call_count == 2 | |
| def test_create_with_retry_does_not_retry_4xx(monkeypatch): | |
| mod = _fake_anthropic_module(monkeypatch) | |
| client = MagicMock() | |
| client.messages.create.side_effect = mod.APIStatusError(400, "bad request") | |
| with pytest.raises(mod.APIStatusError): | |
| _create_with_retry(client, model="m") | |
| assert client.messages.create.call_count == 1 | |
| def test_create_with_retry_reraises_after_second_failure(monkeypatch): | |
| mod = _fake_anthropic_module(monkeypatch) | |
| client = MagicMock() | |
| client.messages.create.side_effect = [ | |
| mod.RateLimitError("slow down"), | |
| mod.RateLimitError("still slow"), | |
| ] | |
| with pytest.raises(mod.RateLimitError): | |
| _create_with_retry(client, model="m") | |
| assert client.messages.create.call_count == 2 | |
| def test_extract_all_returns_empty_without_api_key(): | |
| """No key → no client built, no network attempted, every segment | |
| gets an empty result. (Sanity: the warm-cache refactor must not | |
| regress the early-return path.)""" | |
| results = extract_all( | |
| [Path("frame.jpg")], | |
| [["narration"]], | |
| ["ocr"], | |
| api_key=None, | |
| toggles=LLMToggles(alt_text=True), | |
| ) | |
| assert len(results) == 1 | |
| assert results[0].alt_text == "" | |
| assert results[0].math_equations == [] | |
| def test_description_toggle_adds_prompt_field_and_schema_slot(): | |
| """The description toggle must appear in the system prompt AND in | |
| the structured-output schema so Claude actually returns it.""" | |
| from app.pipeline.llm_extract import _build_extraction_model, _build_system_prompt | |
| toggles = LLMToggles(description=True) | |
| assert toggles.any_enabled is True | |
| schema = _build_extraction_model(toggles) | |
| assert "description" in schema.model_fields | |
| prompt = _build_system_prompt(toggles) | |
| assert "description" in prompt | |
| assert "long-form image description" in prompt | |
| def test_description_toggle_skipped_when_off(): | |
| """The description toggle must not bleed into the prompt or schema | |
| when it's off — that'd waste output tokens on every call.""" | |
| from app.pipeline.llm_extract import _build_extraction_model, _build_system_prompt | |
| toggles = LLMToggles(alt_text=True) # description left default False | |
| schema = _build_extraction_model(toggles) | |
| assert "description" not in schema.model_fields | |
| prompt = _build_system_prompt(toggles) | |
| assert "description" not in prompt | |
| def test_extract_all_degrades_when_client_construction_fails(monkeypatch, tmp_path): | |
| """extract_all's contract is that it always returns — a missing/broken | |
| anthropic SDK must degrade to empty results, not abort the pipeline. | |
| The shared client is built outside the per-segment loop, so its | |
| construction must be guarded too.""" | |
| img = tmp_path / "frame.jpg" | |
| Image.new("RGB", (8, 8), "white").save(img, "JPEG") | |
| def _boom(_key): | |
| raise ImportError("No module named 'anthropic'") | |
| monkeypatch.setattr(llm_extract, "_build_client", _boom) | |
| progress = [] | |
| results = llm_extract.extract_all( | |
| [img], | |
| [["narration"]], | |
| [""], | |
| api_key="sk-stub", | |
| toggles=LLMToggles(alt_text=True), | |
| on_progress=lambda done, total: progress.append((done, total)), | |
| ) | |
| assert len(results) == 1 | |
| assert results[0] is llm_extract._EMPTY | |
| # Progress is still reported as complete so the UI doesn't hang. | |
| assert progress and progress[-1] == (1, 1) | |
| def _cue(start, end, text): | |
| from app.pipeline.transcript import Cue | |
| return Cue(start=start, end=end, text=text) | |
| def test_align_slides_returns_clean_non_decreasing_starts(monkeypatch): | |
| """Alignment maps ordered slides to transcript times: the model's raw | |
| output is clamped to the transcript span, forced non-decreasing, and the | |
| first slide pinned to the transcript's first cue.""" | |
| from app.pipeline.llm_extract import align_slides_to_transcript | |
| _fake_anthropic_module(monkeypatch) | |
| cues = [_cue(10.0, 20.0, "intro"), _cue(20.0, 40.0, "middle"), _cue(40.0, 60.0, "end")] | |
| def resp(text): | |
| blk = MagicMock(); blk.type = "text"; blk.text = text | |
| r = MagicMock(); r.content = [blk]; return r | |
| client = MagicMock() | |
| # Raw model output: out-of-order + out-of-range values that must be cleaned. | |
| client.messages.create.return_value = resp('{"slide_starts": [0, 5, 3, 999]}') | |
| monkeypatch.setattr(llm_extract, "_build_client", lambda key: client) | |
| starts = align_slides_to_transcript( | |
| cues, ["a", "b", "c", "d"], api_key="sk-ant-x" | |
| ) | |
| assert len(starts) == 4 | |
| assert starts == sorted(starts) # non-decreasing | |
| assert starts[0] == 10.0 # pinned to first cue start | |
| assert all(10.0 <= s <= 60.0 for s in starts) # within span | |
| def test_align_slides_raises_without_key(monkeypatch): | |
| from app.pipeline.llm_extract import SlideAlignError, align_slides_to_transcript | |
| monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) | |
| monkeypatch.delenv("PARLEY_API_KEY", raising=False) | |
| cues = [_cue(0.0, 10.0, "x")] | |
| with pytest.raises(SlideAlignError): | |
| align_slides_to_transcript(cues, ["a"], api_key=None) | |
| def test_align_slides_raises_on_count_mismatch(monkeypatch): | |
| from app.pipeline.llm_extract import SlideAlignError, align_slides_to_transcript | |
| _fake_anthropic_module(monkeypatch) | |
| cues = [_cue(0.0, 10.0, "x"), _cue(10.0, 20.0, "y")] | |
| def resp(text): | |
| blk = MagicMock(); blk.type = "text"; blk.text = text | |
| r = MagicMock(); r.content = [blk]; return r | |
| client = MagicMock() | |
| client.messages.create.return_value = resp('{"slide_starts": [0]}') # 1 for 2 slides | |
| monkeypatch.setattr(llm_extract, "_build_client", lambda key: client) | |
| with pytest.raises(SlideAlignError): | |
| align_slides_to_transcript(cues, ["a", "b"], api_key="sk-ant-x") | |
| def test_extract_all_no_op_when_no_toggles_enabled(): | |
| results = extract_all( | |
| [Path("frame.jpg")], | |
| [["narration"]], | |
| ["ocr"], | |
| api_key="sk-stub", | |
| toggles=LLMToggles(), # all off | |
| ) | |
| assert len(results) == 1 | |
| assert results[0] is llm_extract._EMPTY | |