File size: 3,869 Bytes
2b605e8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 | """Picture -> the tower graph's patches, at one fixed square tile per tower graph (256 -> 8 x 8 = 64 picture tokens,
448 -> 14 x 14 = 196).
open -> convert("RGB") -> resize((tile, tile), BICUBIC) Pillow's own integer resampler; the aspect is NOT kept
-> /255 -> (x - 0.5) / 0.5 in float64 -> float32
-> merge-block-major patches [4 G^2, 1536], G = tile // 32, each vector (C, T=2, 16, 16)
This is what the reference's g256 / g448 arms fed the author's processor (the picture resized with Pillow before the
processor, whose own resize is the identity at multiples of 32), and the patches are the processor's `pixel_values`
bit for bit. The author's native arm (the processor's dynamic resolution) is not a tower graph here. EXIF orientation
is not applied (the reference passed opened PIL images, which the processor takes as they are)."""
from __future__ import annotations
import base64
import io
from pathlib import Path
import numpy as np
from .errors import UnsupportedMedia
PATCH, MERGE, TEMPORAL = 16, 2, 2
IMAGE_MEAN, IMAGE_STD = 0.5, 0.5
TILES = (256, 448)
def tile_grid(tile: int) -> int:
"""Merged rows per side of a square tile (256 -> 8, 448 -> 14)."""
if tile not in TILES:
raise UnsupportedMedia(f"tile {tile}: the tower graphs read {' or '.join(map(str, TILES))} px tiles")
return tile // (PATCH * MERGE)
def open_image(image):
"""A PIL image, a uint8 [H, W, 3] array, the bytes of an image file, a local path, or a base64 data URI -> PIL."""
from PIL import Image
if isinstance(image, Image.Image):
return image
if isinstance(image, np.ndarray):
return Image.fromarray(np.asarray(image, dtype=np.uint8))
try:
if isinstance(image, (bytes, bytearray)):
return Image.open(io.BytesIO(bytes(image)))
if isinstance(image, str) and image.startswith("data:image/"):
return Image.open(io.BytesIO(base64.b64decode(image.split(",", 1)[1])))
if isinstance(image, (str, Path)) and Path(image).is_file():
return Image.open(image)
except Exception as e: # Pillow's UnidentifiedImageError, a bad base64 string, ...
raise UnsupportedMedia(f"the picture could not be opened: {e}") from e
raise UnsupportedMedia("a picture is a PIL image, a uint8 array, image file bytes, a local path or a data URI "
"(the host does not fetch URLs)")
def patchify(x: np.ndarray) -> np.ndarray:
"""Normalized [H, W, C] -> [gh*gw, C*T*P*P] in Qwen's merge-block-major order: patches iterate
(block_row, block_col, y-in-block, x-in-block), channel outermost inside the vector, the still frame repeated at
both temporal slots (the Qwen2VLImageProcessor reshape / permute)."""
h, w, c = x.shape
if h % (PATCH * MERGE) or w % (PATCH * MERGE):
raise ValueError(f"{h}x{w} is not divisible by patch * merge = {PATCH * MERGE}")
gh, gw = h // PATCH, w // PATCH
t = x.transpose(2, 0, 1).reshape(c, gh // MERGE, MERGE, PATCH, gw // MERGE, MERGE, PATCH)
t = t.transpose(1, 4, 2, 5, 0, 3, 6).reshape(gh * gw, 1, c, PATCH, PATCH)
t = np.broadcast_to(t[:, :, :, None], (gh * gw, 1, c, TEMPORAL, PATCH, PATCH))
return t.reshape(gh * gw, c * TEMPORAL * PATCH * PATCH)
def tile_pixels(image, tile: int) -> np.ndarray:
"""Picture -> uint8 [tile, tile, 3]: RGB, Pillow BICUBIC to the tile."""
from PIL import Image
tile_grid(tile)
return np.asarray(open_image(image).convert("RGB").resize((tile, tile), Image.Resampling.BICUBIC))
def tile_patches(image, tile: int) -> np.ndarray:
"""Picture -> patches [4 G^2, 1536] float32, G = tile // 32 (the tower graph's input)."""
x = (tile_pixels(image, tile).astype(np.float64) / 255.0 - IMAGE_MEAN) / IMAGE_STD
return np.ascontiguousarray(patchify(x).astype(np.float32))
|