File size: 3,605 Bytes
0810902 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 | """Image-input behaviour.
The vision tower in this checkpoint was copied verbatim from Qwen/Qwen3.5-9B and
never trained against Piko's fine-tuned language backbone. Empirically it works
anyway: OCR and document understanding both scored 10/10 on the custom suite, so
the assertions below are genuine regression guards.
What has NOT been tested is anything outside rendered documents — photographs,
handwriting, natural scenes, low-quality scans. Do not add assertions about those
without measuring first.
"""
from __future__ import annotations
from pathlib import Path
import pytest
pytestmark = pytest.mark.slow
def ask_about_image(loaded_model, image: Path, prompt: str, max_new_tokens: int = 96) -> str:
import torch
model, processor = loaded_model
messages = [
{
"role": "user",
"content": [
{"type": "image", "url": str(image)},
{"type": "text", "text": prompt},
],
}
]
inputs = processor.apply_chat_template(
messages,
add_generation_prompt=True,
tokenize=True,
return_dict=True,
return_tensors="pt",
).to(model.device)
with torch.inference_mode():
output = model.generate(**inputs, max_new_tokens=max_new_tokens, do_sample=False)
return processor.decode(
output[0][inputs["input_ids"].shape[1] :], skip_special_tokens=True
).strip()
def test_image_tokens_expand_the_prompt(loaded_model, receipt_image: Path) -> None:
"""Structural test: the processor must inject image tokens.
This passes regardless of whether the model reads the image correctly, and so
separates 'the multimodal plumbing works' from 'the model can see'.
"""
model, processor = loaded_model
text_only = processor.apply_chat_template(
[{"role": "user", "content": [{"type": "text", "text": "Describe it."}]}],
add_generation_prompt=True,
tokenize=True,
return_dict=True,
return_tensors="pt",
)
with_image = processor.apply_chat_template(
[
{
"role": "user",
"content": [
{"type": "image", "url": str(receipt_image)},
{"type": "text", "text": "Describe it."},
],
}
],
add_generation_prompt=True,
tokenize=True,
return_dict=True,
return_tensors="pt",
)
assert with_image["input_ids"].shape[1] > text_only["input_ids"].shape[1]
assert "pixel_values" in with_image
image_token_id = model.config.image_token_id
assert (with_image["input_ids"] == image_token_id).sum() > 0
def test_image_input_does_not_crash(loaded_model, receipt_image: Path) -> None:
text = ask_about_image(loaded_model, receipt_image, "Describe this image in one sentence.")
assert text
assert len(set(text.replace(" ", ""))) > 5, "degenerate output — check device placement"
def test_reads_total_from_receipt(loaded_model, receipt_image: Path) -> None:
"""OCR works on this checkpoint despite the tower never being re-aligned.
Measured at 10/10 on the custom suite, so this is a real regression guard
rather than an aspiration.
"""
assert "27.30" in ask_about_image(
loaded_model, receipt_image, "What is the TOTAL on this receipt? Number only."
)
def test_reads_merchant_from_receipt(loaded_model, receipt_image: Path) -> None:
answer = ask_about_image(loaded_model, receipt_image, "What is the merchant name? Name only.")
assert "northgate" in answer.lower()
|