image_processing_panovlm.py

# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.

"""Standalone HF image processor for PanoVLM (FastViT, NCHW).

Reproduces the repo's ImageProcessorNCHW: RGB -> aspect-preserving pad-resize to
a fixed square -> rescale to [0,1] -> normalize -> NCHW pixel_values. Defaults
match training (``scripts_local/generate_panovlm.sh``): only ``image_size`` is
meant to be changed by users; ``image_resize_mode``/``image_mean``/``image_std``
are fixed.
"""
from __future__ import annotations

import numpy as np
from PIL import Image
from transformers.image_processing_utils import BaseImageProcessor, BatchFeature
from transformers.image_utils import ImageInput
from transformers.utils import TensorType


def _resize_pad(
    image: Image.Image, image_size: int, fill_mean: tuple[float, float, float]
) -> Image.Image:
    orig_w, orig_h = image.size
    scale = min(image_size / orig_w, image_size / orig_h)
    new_w, new_h = max(1, int(orig_w * scale)), max(1, int(orig_h * scale))
    image = image.resize((new_w, new_h))
    padded = Image.new(
        "RGB", (image_size, image_size), tuple(int(255 * x) for x in fill_mean)
    )
    padded.paste(image, ((image_size - new_w) // 2, (image_size - new_h) // 2))
    return padded


class PanoVLMImageProcessor(BaseImageProcessor):
    model_input_names = ["pixel_values"]

    def __init__(
        self,
        image_size: int = 1024,
        image_resize_mode: str = "pad",
        image_mean=(0.0, 0.0, 0.0),
        image_std=(1.0, 1.0, 1.0),
        **kwargs,
    ):
        if image_resize_mode not in ("pad", "square"):
            raise ValueError(
                f"Unknown image_resize_mode {image_resize_mode!r}; expected 'pad' or 'square'."
            )
        super().__init__(**kwargs)
        self.image_size = image_size
        self.image_resize_mode = image_resize_mode
        self.image_mean = tuple(image_mean)
        self.image_std = tuple(image_std)

    def _to_pil(self, image) -> Image.Image:
        if isinstance(image, Image.Image):
            img = image
        else:
            img = Image.fromarray(np.asarray(image))
        return img.convert("RGB") if img.mode != "RGB" else img

    def _process_one(self, image) -> np.ndarray:
        img = self._to_pil(image)
        if self.image_resize_mode == "pad":
            img = _resize_pad(img, self.image_size, self.image_mean)
        elif self.image_resize_mode == "square":
            img = img.resize((self.image_size, self.image_size))
        else:
            raise ValueError(
                f"Unknown image_resize_mode {self.image_resize_mode!r}; "
                "expected 'pad' or 'square'."
            )
        arr = np.asarray(img, dtype=np.float32) / 255.0
        arr = (arr - np.asarray(self.image_mean, np.float32)) / np.asarray(
            self.image_std, np.float32
        )
        return arr.transpose(2, 0, 1)  # HWC -> CHW

    def preprocess(
        self,
        images: ImageInput,
        return_tensors: str | TensorType | None = None,
        **kwargs,
    ) -> BatchFeature:
        # **kwargs accepted for HF API compat (do_rescale/do_normalize/etc.);
        # processing is fixed by the configured size/mean/std.
        if not isinstance(images, (list, tuple)):
            images = [images]
        pixel_values = np.stack([self._process_one(im) for im in images], axis=0)
        return BatchFeature(
            data={"pixel_values": pixel_values}, tensor_type=return_tensors
        )