Spaces:
Running
fix: flatten RGBA/palette PNGs before JPEG-encoding for the LLM
Browse filesLive test of the images-input mode surfaced "cannot write mode RGBA as
JPEG" on every frame: PNG slide exports commonly carry an alpha channel
(RGBA) or a palette with transparency, and _encode_image saved straight
to JPEG, which has no alpha — so PIL raised and every per-segment LLM
call failed, leaving descriptions/alt-text/OCR empty.
Flatten onto a white background before encoding (transparent slide
regions read as white, matching how a sighted viewer sees them), and
convert any other non-RGB/L mode to RGB. Only affects the bytes sent to
Claude; the displayed/embedded image still uses the original PNG via the
data-URI path, so on-screen quality is unchanged.
Tests (runnable without cv2): RGBA and palette-with-transparency PNGs
both encode to a valid RGB JPEG instead of raising.
https://claude.ai/code/session_01EM8jiYxdVFy1w3LeJ9GBJg
- app/pipeline/llm_extract.py +13 -0
- tests/test_llm_extract.py +30 -0
|
@@ -274,6 +274,19 @@ def _encode_image(image_path: Path) -> tuple[str, str]:
|
|
| 274 |
from PIL import Image
|
| 275 |
import io
|
| 276 |
with Image.open(image_path) as img:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 277 |
w, h = img.size
|
| 278 |
if w > _MAX_IMAGE_WIDTH:
|
| 279 |
new_h = round(h * _MAX_IMAGE_WIDTH / w)
|
|
|
|
| 274 |
from PIL import Image
|
| 275 |
import io
|
| 276 |
with Image.open(image_path) as img:
|
| 277 |
+
# JPEG has no alpha channel; PNG slide exports are frequently RGBA
|
| 278 |
+
# (or palette 'P'), which makes img.save(format="JPEG") raise
|
| 279 |
+
# "cannot write mode RGBA as JPEG". Flatten onto white so the LLM
|
| 280 |
+
# sees the slide as a sighted viewer would (transparent regions on
|
| 281 |
+
# a slide read as white background).
|
| 282 |
+
if img.mode not in ("RGB", "L"):
|
| 283 |
+
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
|
| 284 |
+
rgba = img.convert("RGBA")
|
| 285 |
+
background = Image.new("RGB", rgba.size, (255, 255, 255))
|
| 286 |
+
background.paste(rgba, mask=rgba.split()[-1])
|
| 287 |
+
img = background
|
| 288 |
+
else:
|
| 289 |
+
img = img.convert("RGB")
|
| 290 |
w, h = img.size
|
| 291 |
if w > _MAX_IMAGE_WIDTH:
|
| 292 |
new_h = round(h * _MAX_IMAGE_WIDTH / w)
|
|
@@ -9,6 +9,7 @@ These pin the behaviour that:
|
|
| 9 |
from __future__ import annotations
|
| 10 |
|
| 11 |
import io
|
|
|
|
| 12 |
from pathlib import Path
|
| 13 |
from unittest.mock import MagicMock
|
| 14 |
|
|
@@ -36,6 +37,35 @@ def _text_block(content: list[dict]) -> str:
|
|
| 36 |
return next(b["text"] for b in content if b["type"] == "text")
|
| 37 |
|
| 38 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
def test_build_user_content_includes_ocr_by_default(tiny_image: Path):
|
| 40 |
content = _build_user_content(
|
| 41 |
tiny_image, ["narration here"], "OCR FROM SLIDE", include_ocr=True
|
|
|
|
| 9 |
from __future__ import annotations
|
| 10 |
|
| 11 |
import io
|
| 12 |
+
import base64
|
| 13 |
from pathlib import Path
|
| 14 |
from unittest.mock import MagicMock
|
| 15 |
|
|
|
|
| 37 |
return next(b["text"] for b in content if b["type"] == "text")
|
| 38 |
|
| 39 |
|
| 40 |
+
def test_encode_image_handles_rgba_png(tmp_path: Path):
|
| 41 |
+
"""PNG slide exports are frequently RGBA; JPEG has no alpha channel,
|
| 42 |
+
so encoding must flatten rather than raise 'cannot write mode RGBA as
|
| 43 |
+
JPEG'. Regression for the images-input mode."""
|
| 44 |
+
p = tmp_path / "00-05-30.png"
|
| 45 |
+
# Semi-transparent red on a transparent canvas.
|
| 46 |
+
Image.new("RGBA", (24, 18), (255, 0, 0, 128)).save(p, "PNG")
|
| 47 |
+
data, media_type = llm_extract._encode_image(p)
|
| 48 |
+
assert media_type == "image/jpeg"
|
| 49 |
+
# Round-trips to a valid JPEG.
|
| 50 |
+
import base64 as _b64
|
| 51 |
+
decoded = _b64.b64decode(data)
|
| 52 |
+
img = Image.open(io.BytesIO(decoded))
|
| 53 |
+
assert img.format == "JPEG"
|
| 54 |
+
assert img.mode == "RGB"
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def test_encode_image_handles_palette_png_with_transparency(tmp_path: Path):
|
| 58 |
+
"""Palette-mode ('P') PNGs with a transparency entry also can't go
|
| 59 |
+
straight to JPEG; they must flatten cleanly too."""
|
| 60 |
+
p = tmp_path / "00-01-00.png"
|
| 61 |
+
base = Image.new("RGBA", (20, 20), (0, 128, 0, 200))
|
| 62 |
+
base.convert("P", palette=Image.ADAPTIVE).save(p, "PNG", transparency=0)
|
| 63 |
+
data, media_type = llm_extract._encode_image(p)
|
| 64 |
+
assert media_type == "image/jpeg"
|
| 65 |
+
img = Image.open(io.BytesIO(base64.b64decode(data)))
|
| 66 |
+
assert img.mode == "RGB"
|
| 67 |
+
|
| 68 |
+
|
| 69 |
def test_build_user_content_includes_ocr_by_default(tiny_image: Path):
|
| 70 |
content = _build_user_content(
|
| 71 |
tiny_image, ["narration here"], "OCR FROM SLIDE", include_ocr=True
|