File size: 6,033 Bytes
fe44a6e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 | """
Standalone PaddleOCR-VL Image Preprocessing (No PyTorch / No Transformers)
==========================================================================
Replicates the official PaddleOCRVLImageProcessor pipeline using only
Pillow + NumPy.
Pipeline:
1. Convert to RGB
2. smart_resize → both dims divisible by 28, within pixel budget
3. Rescale to [0, 1]
4. Normalize (CLIP mean/std)
5. Patchify → (num_patches, 3, 14, 14)
"""
from __future__ import annotations
import math
from typing import Tuple
import numpy as np
from PIL import Image
# ---------------------------------------------------------------------------
# Constants (from PaddleOCR-VL config)
# ---------------------------------------------------------------------------
PATCH_SIZE = 14
MERGE_SIZE = 2
FACTOR = PATCH_SIZE * MERGE_SIZE # 28
MIN_PIXELS = 28 * 28 * 130 # 101,920
MAX_PIXELS = 28 * 28 * 1280 # 1,003,520
# CLIP mean / std (OpenAI variant used by PaddleOCR-VL)
IMAGE_MEAN = np.array([0.48145466, 0.4578275, 0.40821073], dtype=np.float32)
IMAGE_STD = np.array([0.26862954, 0.26130258, 0.27577711], dtype=np.float32)
# ---------------------------------------------------------------------------
# smart_resize
# ---------------------------------------------------------------------------
def smart_resize(
height: int,
width: int,
factor: int = FACTOR,
min_pixels: int = MIN_PIXELS,
max_pixels: int = MAX_PIXELS,
) -> Tuple[int, int]:
"""
Rescale so that:
1. Both dimensions are divisible by `factor`.
2. Total pixels ∈ [min_pixels, max_pixels].
3. Aspect ratio is preserved as closely as possible.
"""
if height < factor:
width = round((width * factor) / height)
height = factor
if width < factor:
height = round((height * factor) / width)
width = factor
if max(height, width) / min(height, width) > 200:
raise ValueError(
f"Aspect ratio too extreme: {max(height, width) / min(height, width)}"
)
h_bar = round(height / factor) * factor
w_bar = round(width / factor) * factor
if h_bar * w_bar > max_pixels:
beta = math.sqrt((height * width) / max_pixels)
h_bar = math.floor(height / beta / factor) * factor
w_bar = math.floor(width / beta / factor) * factor
elif h_bar * w_bar < min_pixels:
beta = math.sqrt(min_pixels / (height * width))
h_bar = math.ceil(height * beta / factor) * factor
w_bar = math.ceil(width * beta / factor) * factor
return h_bar, w_bar
# ---------------------------------------------------------------------------
# Preprocessing
# ---------------------------------------------------------------------------
def preprocess(
image: Image.Image,
) -> Tuple[np.ndarray, Tuple[int, int, int]]:
"""
Preprocess a PIL image into patch tensor for ONNX inference.
Args:
image: PIL RGB image (any size).
Returns:
pixel_values: (num_patches, 3, 14, 14) float32 array
grid_thw: (grid_t, grid_h, grid_w) — temporal=1 always
"""
# 1. Convert to RGB
img = image.convert("RGB")
# 2. smart_resize
width, height = img.size
new_h, new_w = smart_resize(height, width)
if (new_w, new_h) != (width, height):
img = img.resize((new_w, new_h), Image.BICUBIC)
# 3. To numpy, rescale to [0, 1]
arr = np.array(img, dtype=np.float32) / 255.0
# 4. Normalize (CHW)
arr = (arr - IMAGE_MEAN.reshape(1, 1, 3)) / IMAGE_STD.reshape(1, 1, 3)
arr = arr.transpose(2, 0, 1) # HWC → CHW
# 5. Patchify: (C, H, W) → (num_patches, C, 14, 14)
c, h, w = arr.shape
grid_h = h // PATCH_SIZE
grid_w = w // PATCH_SIZE
grid_t = 1 # temporal patches = 1 for images
# Reshape into grid of patches
patches = arr.reshape(c, grid_h, PATCH_SIZE, grid_w, PATCH_SIZE)
patches = patches.transpose(1, 3, 0, 2, 4) # → (grid_h, grid_w, c, 14, 14)
patches = patches.reshape(-1, c, PATCH_SIZE, PATCH_SIZE) # → (N, 3, 14, 14)
# Handle temporal dim (always 1 for images)
patches = np.tile(patches, (grid_t, 1, 1, 1))
grid_thw = (grid_t, grid_h, grid_w)
return patches.astype(np.float32), grid_thw
def preprocess_for_onnx(image: Image.Image) -> Tuple[np.ndarray, np.ndarray]:
"""
Preprocess image and return ONNX-ready inputs.
Args:
image: PIL RGB image.
Returns:
pixel_values: (1, num_patches, 3, 14, 14) float32
position_ids: (1, 1) int64
"""
patches, _grid_thw = preprocess(image)
# Add batch dimension
pixel_values = patches[np.newaxis, ...] # (1, N, 3, 14, 14)
position_ids = np.zeros((1, 1), dtype=np.int64)
return pixel_values.astype(np.float32), position_ids
# ---------------------------------------------------------------------------
# Reverse: patches → image (for verification)
# ---------------------------------------------------------------------------
def patches_to_image(
pixel_values: np.ndarray,
grid_h: int,
grid_w: int,
) -> Image.Image:
"""
Reconstruct an image from patch tensor (for debugging).
Args:
pixel_values: (N, 3, 14, 14) or (1, N, 3, 14, 14)
grid_h, grid_w: grid dimensions
Returns:
PIL Image (approx reconstruction of preprocessed input)
"""
if pixel_values.ndim == 5:
pixel_values = pixel_values.squeeze(0) # remove batch
n_patches = grid_h * grid_w
patches = pixel_values[:n_patches] # (N, 3, 14, 14)
# Un-patchify
c = patches.shape[1]
ps = patches.shape[2]
patches = patches.reshape(grid_h, grid_w, c, ps, ps)
patches = patches.transpose(2, 0, 3, 1, 4) # → (c, grid_h, ps, grid_w, ps)
img = patches.reshape(c, grid_h * ps, grid_w * ps) # (c, H, W)
# De-normalize
img = img.transpose(1, 2, 0) # CHW → HWC
img = img * IMAGE_STD.reshape(1, 1, 3) + IMAGE_MEAN.reshape(1, 1, 3)
img = np.clip(img * 255, 0, 255).astype(np.uint8)
return Image.fromarray(img)
|