File size: 5,358 Bytes
56f291f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 | # -*- coding: utf-8 -*-
import unittest
from Tokenizer.multimodal import (
IMAGE_END,
IMAGE_PATCH,
IMAGE_PLACEHOLDER,
IMAGE_START,
MultimodalProcessor,
expand_image_placeholders,
image_patch_count,
)
from Tokenizer.tests.test_dual_tokenizer import build_fake_tokenizer
class MultimodalProcessorTest(unittest.TestCase):
def test_single_image_placeholder_expansion(self):
expanded = expand_image_placeholders(IMAGE_PLACEHOLDER, 2)
self.assertEqual(expanded, IMAGE_START + IMAGE_PATCH + IMAGE_PATCH + IMAGE_END)
def test_multi_image_placeholder_expansion(self):
processor = MultimodalProcessor(build_fake_tokenizer())
out = processor(
IMAGE_PLACEHOLDER + " test " + IMAGE_PLACEHOLDER,
images=["img1", "img2"],
image_sizes=[(14, 14), (28, 28)],
)
self.assertEqual(len(out.image_token_spans), 2)
first_start, first_end = out.image_token_spans[0]
second_start, second_end = out.image_token_spans[1]
self.assertEqual(out.tokens[first_start].token, IMAGE_START)
self.assertEqual(out.tokens[first_end - 1].token, IMAGE_END)
self.assertEqual(out.tokens[second_start].token, IMAGE_START)
self.assertEqual(out.tokens[second_end - 1].token, IMAGE_END)
def test_image_token_spans_are_input_id_ranges(self):
processor = MultimodalProcessor(build_fake_tokenizer())
out = processor(IMAGE_PLACEHOLDER, images=["img"], image_sizes=[(28, 28)])
start, end = out.image_token_spans[0]
span_tokens = out.tokens[start:end]
self.assertEqual([tok.token for tok in span_tokens], [IMAGE_START, IMAGE_PATCH, IMAGE_END])
self.assertEqual(out.input_ids[start:end], [tok.id for tok in span_tokens])
def test_patch_count_stable(self):
self.assertEqual(image_patch_count(28, 28, patch_size=14, merge_size=2), 1)
self.assertEqual(image_patch_count(29, 29, patch_size=14, merge_size=2), 4)
def test_image_size_mismatch_raises(self):
processor = MultimodalProcessor(build_fake_tokenizer())
with self.assertRaises(ValueError):
processor(IMAGE_PLACEHOLDER + " " + IMAGE_PLACEHOLDER, image_sizes=[(28, 28)])
def test_image_count_mismatch_raises(self):
processor = MultimodalProcessor(build_fake_tokenizer())
with self.assertRaises(ValueError):
processor(IMAGE_PLACEHOLDER, images=[])
with self.assertRaises(ValueError):
processor(IMAGE_PLACEHOLDER, images=["a", "b"])
with self.assertRaises(ValueError):
processor("test", images=["orphan"])
def test_attention_mask_matches_input_ids(self):
processor = MultimodalProcessor(build_fake_tokenizer())
out = processor("hello " + IMAGE_PLACEHOLDER, images=["img"], image_sizes=[(28, 28)])
self.assertEqual(len(out.attention_mask), len(out.input_ids))
self.assertTrue(all(m == 1 for m in out.attention_mask))
def test_image_index_metadata(self):
processor = MultimodalProcessor(build_fake_tokenizer())
out = processor(
IMAGE_PLACEHOLDER + " test " + IMAGE_PLACEHOLDER,
images=["img1", "img2"],
image_sizes=[(14, 14), (14, 14)],
)
# Collect image_index values from start markers
starts = [t for t in out.tokens if t.token == IMAGE_START]
self.assertEqual(len(starts), 2)
self.assertEqual(starts[0].metadata["image_index"], 0)
self.assertEqual(starts[1].metadata["image_index"], 1)
def test_video_placeholder_expansion(self):
from Tokenizer.multimodal import VIDEO_END, VIDEO_PATCH, VIDEO_PLACEHOLDER, VIDEO_START
processor = MultimodalProcessor(build_fake_tokenizer())
out = processor(VIDEO_PLACEHOLDER, videos=["vid"], video_sizes=[(2, 28, 28)])
spans = out.video_token_spans
self.assertEqual(len(spans), 1)
start, end = spans[0]
seq = [t.token for t in out.tokens[start:end]]
self.assertEqual(seq[0], VIDEO_START)
self.assertEqual(seq[-1], VIDEO_END)
self.assertTrue(all(t == VIDEO_PATCH for t in seq[1:-1]))
def test_video_count_mismatch_raises(self):
from Tokenizer.multimodal import VIDEO_PLACEHOLDER
processor = MultimodalProcessor(build_fake_tokenizer())
with self.assertRaises(ValueError):
processor(VIDEO_PLACEHOLDER, videos=[])
with self.assertRaises(ValueError):
processor(VIDEO_PLACEHOLDER, videos=["a", "b"])
with self.assertRaises(ValueError):
processor("test", videos=["orphan"])
def test_bbox_normalize_roundtrip(self):
from Tokenizer.multimodal import decode_bbox_tokens, encode_bbox_tokens, normalize_bbox
bbox = (10.0, 20.0, 100.0, 200.0)
tok = encode_bbox_tokens(bbox, width=200, height=400)
self.assertTrue(tok.startswith("<bbox_") and tok.endswith(">"))
coords = normalize_bbox(bbox, 200, 400)
self.assertEqual(len(coords), 4)
# Decode is approximate (quantization), but ordering preserved
decoded = decode_bbox_tokens(tok, width=200, height=400)
self.assertLess(abs(decoded[0] - 10.0), 1.0)
self.assertLess(abs(decoded[3] - 200.0), 1.0)
if __name__ == "__main__":
unittest.main()
|