CLEF dual-task proper-protocol FT (seed 20261101, epoch 3) from jev-retrain MI355X run 2026-11-04
073de96 verified | """Picture -> the tower graph's patches, at one fixed square tile per tower graph (256 -> 8 x 8 = 64 picture tokens, | |
| 448 -> 14 x 14 = 196). | |
| open -> convert("RGB") -> resize((tile, tile), BICUBIC) Pillow's own integer resampler; the aspect is NOT kept | |
| -> /255 -> (x - 0.5) / 0.5 in float64 -> float32 | |
| -> merge-block-major patches [4 G^2, 1536], G = tile // 32, each vector (C, T=2, 16, 16) | |
| This is what the reference's g256 / g448 arms fed the author's processor (the picture resized with Pillow before the | |
| processor, whose own resize is the identity at multiples of 32), and the patches are the processor's `pixel_values` | |
| bit for bit. The author's native arm (the processor's dynamic resolution) is not a tower graph here. EXIF orientation | |
| is not applied (the reference passed opened PIL images, which the processor takes as they are).""" | |
| from __future__ import annotations | |
| import base64 | |
| import io | |
| from pathlib import Path | |
| import numpy as np | |
| from .errors import UnsupportedMedia | |
| PATCH, MERGE, TEMPORAL = 16, 2, 2 | |
| IMAGE_MEAN, IMAGE_STD = 0.5, 0.5 | |
| TILES = (256, 448) | |
| def tile_grid(tile: int) -> int: | |
| """Merged rows per side of a square tile (256 -> 8, 448 -> 14).""" | |
| if tile not in TILES: | |
| raise UnsupportedMedia(f"tile {tile}: the tower graphs read {' or '.join(map(str, TILES))} px tiles") | |
| return tile // (PATCH * MERGE) | |
| def open_image(image): | |
| """A PIL image, a uint8 [H, W, 3] array, the bytes of an image file, a local path, or a base64 data URI -> PIL.""" | |
| from PIL import Image | |
| if isinstance(image, Image.Image): | |
| return image | |
| if isinstance(image, np.ndarray): | |
| return Image.fromarray(np.asarray(image, dtype=np.uint8)) | |
| try: | |
| if isinstance(image, (bytes, bytearray)): | |
| return Image.open(io.BytesIO(bytes(image))) | |
| if isinstance(image, str) and image.startswith("data:image/"): | |
| return Image.open(io.BytesIO(base64.b64decode(image.split(",", 1)[1]))) | |
| if isinstance(image, (str, Path)) and Path(image).is_file(): | |
| return Image.open(image) | |
| except Exception as e: # Pillow's UnidentifiedImageError, a bad base64 string, ... | |
| raise UnsupportedMedia(f"the picture could not be opened: {e}") from e | |
| raise UnsupportedMedia("a picture is a PIL image, a uint8 array, image file bytes, a local path or a data URI " | |
| "(the host does not fetch URLs)") | |
| def patchify(x: np.ndarray) -> np.ndarray: | |
| """Normalized [H, W, C] -> [gh*gw, C*T*P*P] in Qwen's merge-block-major order: patches iterate | |
| (block_row, block_col, y-in-block, x-in-block), channel outermost inside the vector, the still frame repeated at | |
| both temporal slots (the Qwen2VLImageProcessor reshape / permute).""" | |
| h, w, c = x.shape | |
| if h % (PATCH * MERGE) or w % (PATCH * MERGE): | |
| raise ValueError(f"{h}x{w} is not divisible by patch * merge = {PATCH * MERGE}") | |
| gh, gw = h // PATCH, w // PATCH | |
| t = x.transpose(2, 0, 1).reshape(c, gh // MERGE, MERGE, PATCH, gw // MERGE, MERGE, PATCH) | |
| t = t.transpose(1, 4, 2, 5, 0, 3, 6).reshape(gh * gw, 1, c, PATCH, PATCH) | |
| t = np.broadcast_to(t[:, :, :, None], (gh * gw, 1, c, TEMPORAL, PATCH, PATCH)) | |
| return t.reshape(gh * gw, c * TEMPORAL * PATCH * PATCH) | |
| def tile_pixels(image, tile: int) -> np.ndarray: | |
| """Picture -> uint8 [tile, tile, 3]: RGB, Pillow BICUBIC to the tile.""" | |
| from PIL import Image | |
| tile_grid(tile) | |
| return np.asarray(open_image(image).convert("RGB").resize((tile, tile), Image.Resampling.BICUBIC)) | |
| def tile_patches(image, tile: int) -> np.ndarray: | |
| """Picture -> patches [4 G^2, 1536] float32, G = tile // 32 (the tower graph's input).""" | |
| x = (tile_pixels(image, tile).astype(np.float64) / 255.0 - IMAGE_MEAN) / IMAGE_STD | |
| return np.ascontiguousarray(patchify(x).astype(np.float32)) | |