Claude commited on
Commit
13e0c0c
·
1 Parent(s): 31ce41a

fix: flatten RGBA/palette PNGs before JPEG-encoding for the LLM

Browse files

Live test of the images-input mode surfaced "cannot write mode RGBA as
JPEG" on every frame: PNG slide exports commonly carry an alpha channel
(RGBA) or a palette with transparency, and _encode_image saved straight
to JPEG, which has no alpha — so PIL raised and every per-segment LLM
call failed, leaving descriptions/alt-text/OCR empty.

Flatten onto a white background before encoding (transparent slide
regions read as white, matching how a sighted viewer sees them), and
convert any other non-RGB/L mode to RGB. Only affects the bytes sent to
Claude; the displayed/embedded image still uses the original PNG via the
data-URI path, so on-screen quality is unchanged.

Tests (runnable without cv2): RGBA and palette-with-transparency PNGs
both encode to a valid RGB JPEG instead of raising.

https://claude.ai/code/session_01EM8jiYxdVFy1w3LeJ9GBJg

app/pipeline/llm_extract.py CHANGED
@@ -274,6 +274,19 @@ def _encode_image(image_path: Path) -> tuple[str, str]:
274
  from PIL import Image
275
  import io
276
  with Image.open(image_path) as img:
 
 
 
 
 
 
 
 
 
 
 
 
 
277
  w, h = img.size
278
  if w > _MAX_IMAGE_WIDTH:
279
  new_h = round(h * _MAX_IMAGE_WIDTH / w)
 
274
  from PIL import Image
275
  import io
276
  with Image.open(image_path) as img:
277
+ # JPEG has no alpha channel; PNG slide exports are frequently RGBA
278
+ # (or palette 'P'), which makes img.save(format="JPEG") raise
279
+ # "cannot write mode RGBA as JPEG". Flatten onto white so the LLM
280
+ # sees the slide as a sighted viewer would (transparent regions on
281
+ # a slide read as white background).
282
+ if img.mode not in ("RGB", "L"):
283
+ if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
284
+ rgba = img.convert("RGBA")
285
+ background = Image.new("RGB", rgba.size, (255, 255, 255))
286
+ background.paste(rgba, mask=rgba.split()[-1])
287
+ img = background
288
+ else:
289
+ img = img.convert("RGB")
290
  w, h = img.size
291
  if w > _MAX_IMAGE_WIDTH:
292
  new_h = round(h * _MAX_IMAGE_WIDTH / w)
tests/test_llm_extract.py CHANGED
@@ -9,6 +9,7 @@ These pin the behaviour that:
9
  from __future__ import annotations
10
 
11
  import io
 
12
  from pathlib import Path
13
  from unittest.mock import MagicMock
14
 
@@ -36,6 +37,35 @@ def _text_block(content: list[dict]) -> str:
36
  return next(b["text"] for b in content if b["type"] == "text")
37
 
38
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
39
  def test_build_user_content_includes_ocr_by_default(tiny_image: Path):
40
  content = _build_user_content(
41
  tiny_image, ["narration here"], "OCR FROM SLIDE", include_ocr=True
 
9
  from __future__ import annotations
10
 
11
  import io
12
+ import base64
13
  from pathlib import Path
14
  from unittest.mock import MagicMock
15
 
 
37
  return next(b["text"] for b in content if b["type"] == "text")
38
 
39
 
40
+ def test_encode_image_handles_rgba_png(tmp_path: Path):
41
+ """PNG slide exports are frequently RGBA; JPEG has no alpha channel,
42
+ so encoding must flatten rather than raise 'cannot write mode RGBA as
43
+ JPEG'. Regression for the images-input mode."""
44
+ p = tmp_path / "00-05-30.png"
45
+ # Semi-transparent red on a transparent canvas.
46
+ Image.new("RGBA", (24, 18), (255, 0, 0, 128)).save(p, "PNG")
47
+ data, media_type = llm_extract._encode_image(p)
48
+ assert media_type == "image/jpeg"
49
+ # Round-trips to a valid JPEG.
50
+ import base64 as _b64
51
+ decoded = _b64.b64decode(data)
52
+ img = Image.open(io.BytesIO(decoded))
53
+ assert img.format == "JPEG"
54
+ assert img.mode == "RGB"
55
+
56
+
57
+ def test_encode_image_handles_palette_png_with_transparency(tmp_path: Path):
58
+ """Palette-mode ('P') PNGs with a transparency entry also can't go
59
+ straight to JPEG; they must flatten cleanly too."""
60
+ p = tmp_path / "00-01-00.png"
61
+ base = Image.new("RGBA", (20, 20), (0, 128, 0, 200))
62
+ base.convert("P", palette=Image.ADAPTIVE).save(p, "PNG", transparency=0)
63
+ data, media_type = llm_extract._encode_image(p)
64
+ assert media_type == "image/jpeg"
65
+ img = Image.open(io.BytesIO(base64.b64decode(data)))
66
+ assert img.mode == "RGB"
67
+
68
+
69
  def test_build_user_content_includes_ocr_by_default(tiny_image: Path):
70
  content = _build_user_content(
71
  tiny_image, ["narration here"], "OCR FROM SLIDE", include_ocr=True