File size: 2,312 Bytes
a381a62 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 | def _layer_hidden(output):
"""Return the hidden tensor from a decoder-layer output."""
import torch
if torch.is_tensor(output):
return output
if isinstance(output, (tuple, list)) and output and torch.is_tensor(output[0]):
return output[0]
raise TypeError(f"unsupported decoder-layer output type: {type(output)!r}")
def _closest_ratio(aspect, ratios, width, height, image_size):
best = (1, 1)
difference = float("inf")
area = width * height
for ratio in ratios:
candidate = ratio[0] / ratio[1]
current = abs(aspect - candidate)
if current < difference or (
current == difference
and area > 0.5 * image_size * image_size * ratio[0] * ratio[1]
):
difference = current
best = ratio
return best
def dynamic_tiles(image, *, image_size: int, max_tiles: int, thumbnail: bool):
"""Official InternVL dynamic tiling, kept local for reproducibility."""
ratios = sorted(
{
(i, j)
for n in range(1, max_tiles + 1)
for i in range(1, n + 1)
for j in range(1, n + 1)
if 1 <= i * j <= max_tiles
},
key=lambda item: item[0] * item[1],
)
width, height = image.size
columns, rows = _closest_ratio(
width / height, ratios, width, height, image_size
)
resized = image.convert("RGB").resize(
(image_size * columns, image_size * rows), resample=3
)
tiles = []
for index in range(columns * rows):
left = (index % columns) * image_size
top = (index // columns) * image_size
tiles.append(
resized.crop((left, top, left + image_size, top + image_size))
)
if thumbnail and len(tiles) != 1:
tiles.append(image.convert("RGB").resize((image_size, image_size), 3))
return tiles
def _normalize_tiles(tiles):
import numpy as np
import torch
mean = torch.tensor((0.485, 0.456, 0.406)).view(3, 1, 1)
std = torch.tensor((0.229, 0.224, 0.225)).view(3, 1, 1)
tensors = []
for tile in tiles:
array = np.asarray(tile, dtype=np.float32) / 255.0
tensor = torch.from_numpy(array).permute(2, 0, 1)
tensors.append((tensor - mean) / std)
return torch.stack(tensors)
|