Upload PaddleOCR-VL split vision encoder artifacts

README.md

@@ -0,0 +1,59 @@
+---
+license: apache-2.0
+library_name: PaddleOCR
+tags:
+- PaddleOCR
+- PaddleOCR-VL
+- vision-encoder
+- multimodal
+- document-parsing
+---
+
+# PaddleOCR-VL Split Vision Encoder
+
+This repository contains the extracted PaddleOCR-VL split visual artifacts uploaded separately from the full VLM.
+
+## Contents
+
+- `vision_tower_config.json`
+- `vision_tower.safetensors`
+- `projector_config.json`
+- `projector.safetensors`
+
+
+## Architecture
+
+- Vision tower hidden size: `1152`
+- Projector output hidden size: `1024`
+- Target repo: `acsfid/PaddleOCR-VL-VisionEncoder`
+
+## Usage
+
+```python
+from model.extracted_vision_encoder import PaddleOCRVLVisionTower, PaddleOCRVLProjector
+
+artifact_dir = "."
+vision_tower = PaddleOCRVLVisionTower.from_pretrained(artifact_dir)
+projector = PaddleOCRVLProjector.from_pretrained(artifact_dir)
+```
+
+The intended split flow is:
+
+```text
+image_processor -> vision_tower -> projector -> decoder-ready image embeddings
+```
+
+## Included Python Source
+
+This repo also includes the Python source files needed to load and use the split artifacts:
+
+- `model/__init__.py`
+- `model/configuration_paddleocr_vl.py`
+- `model/image_processing_paddleocr_vl.py`
+- `model/modeling_paddleocr_vl.py`
+- `model/extracted_vision_encoder.py`
+- `requirements.txt`
+
+That means after cloning or downloading this repo, you can directly import the split classes for inference or later training work.
+
+

model/__init__.py

@@ -0,0 +1,11 @@
+from .extracted_vision_encoder import (
+    PaddleOCRVLProjector,
+    PaddleOCRVLVisionEncoder,
+    PaddleOCRVLVisionTower,
+)
+
+__all__ = [
+    "PaddleOCRVLVisionTower",
+    "PaddleOCRVLProjector",
+    "PaddleOCRVLVisionEncoder",
+]

model/configuration_paddleocr_vl.py

@@ -0,0 +1,191 @@
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.modeling_rope_utils import rope_config_validation
+
+class PaddleOCRVisionConfig(PretrainedConfig):
+    model_type = "paddleocr_vl"
+    base_config_key = "vision_config"
+
+    def __init__(
+        self,
+        hidden_size=768,
+        intermediate_size=3072,
+        num_hidden_layers=12,
+        num_attention_heads=12,
+        num_channels=3,
+        image_size=224,
+        patch_size=14,
+        hidden_act="gelu_pytorch_tanh",
+        layer_norm_eps=1e-6,
+        attention_dropout=0.0,
+        spatial_merge_size=2,
+        temporal_patch_size=2,
+        tokens_per_second=2,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_channels = num_channels
+        self.patch_size = patch_size
+        self.image_size = image_size
+        self.attention_dropout = attention_dropout
+        self.layer_norm_eps = layer_norm_eps
+        self.hidden_act = hidden_act
+        self.spatial_merge_size = spatial_merge_size
+        self.temporal_patch_size = temporal_patch_size
+        self.tokens_per_second = tokens_per_second
+
+
+
+class PaddleOCRVLConfig(PretrainedConfig):
+    """
+    Configuration class.
+
+    This class stores the configuration of an Ernie model, defining the model architecture.
+    It inherits from PretrainedConfig and can be used to control model outputs.
+    """
+
+    model_type = "paddleocr_vl"
+    keys_to_ignore_at_inference = ["past_key_values"]
+    sub_configs = {"vision_config": PaddleOCRVisionConfig}
+
+    # Default tensor parallel plan for base model `Qwen3`
+    base_model_tp_plan = {
+        "layers.*.self_attn.q_proj": "colwise",
+        "layers.*.self_attn.k_proj": "colwise",
+        "layers.*.self_attn.v_proj": "colwise",
+        "layers.*.self_attn.o_proj": "rowwise",
+        "layers.*.mlp.gate_proj": "colwise",
+        "layers.*.mlp.up_proj": "colwise",
+        "layers.*.mlp.down_proj": "rowwise",
+    }
+    base_model_pp_plan = {
+        "embed_tokens": (["input_ids"], ["inputs_embeds"]),
+        "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
+        "norm": (["hidden_states"], ["hidden_states"]),
+    }
+
+    def __init__(
+        self,
+        vocab_size=32000,
+        hidden_size=768,
+        intermediate_size=11008,
+        max_position_embeddings=32768,
+        num_hidden_layers=2,
+        num_attention_heads=2,
+        image_token_id=101304,
+        video_token_id=101305,
+        vision_start_token_id=101306,
+        rms_norm_eps=1e-6,
+        use_cache=False,
+        use_flash_attention=False,
+        pad_token_id=0,
+        bos_token_id=1,
+        eos_token_id=2,
+        head_dim=128,
+        hidden_act="silu",
+        use_bias=False,
+        rope_theta=10000,
+        weight_share_add_bias=True,
+        ignored_index=-100,
+        attention_probs_dropout_prob=0.0,
+        hidden_dropout_prob=0.0,
+        compression_ratio: float = 1.0,
+        num_key_value_heads=None,
+        max_sequence_length=None,
+        tie_word_embeddings=False,
+        vision_config=None,
+        rope_scaling=None,
+        **kwargs,
+    ):
+        """
+        Initialize configuration with default or specified parameters.
+
+        Args:
+            vocab_size (int): Size of the vocabulary (number of unique tokens)
+            hidden_size (int): Dimensionality of the encoder layers and the pooler layer
+            intermediate_size (int): Dimensionality of the "intermediate" (feed-forward) layer
+            max_position_embeddings (int): Maximum sequence length the model can handle
+            num_hidden_layers (int): Number of hidden layers in the Transformer encoder
+            num_attention_heads (int): Number of attention heads for each attention layer
+            rms_norm_eps (float): The epsilon used by the RMS normalization layers
+            use_cache (bool): Whether to use caching for faster generation (decoding)
+            use_flash_attention (bool): Whether to use FlashAttention for optimized attention computation
+            pad_token_id (int): Token ID used for padding sequences
+            bos_token_id (int): Token ID used for beginning-of-sequence
+            eos_token_id (int): Token ID used for end-of-sequence
+            use_bias (bool): Whether to use bias terms in linear layers
+            rope_theta (float): The base period of the RoPE embeddings
+            weight_share_add_bias (bool): Whether to share bias weights in certain layers
+            ignored_index (int): Target value that is ignored during loss computation
+            attention_probs_dropout_prob (float): Dropout probability for attention weights
+            hidden_dropout_prob (float): Dropout probability for hidden layers
+            compression_ratio (float): Ratio for KV cache compression (1.0 = no compression)
+            num_key_value_heads (int): Number of key/value heads (for Grouped Query Attention)
+            max_sequence_length (int): Maximum sequence length for positional embeddings
+            **kwargs: Additional keyword arguments passed to parent class
+        """
+
+        # Set default for tied embeddings if not specified.
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            **kwargs,
+        )
+        if isinstance(vision_config, dict):
+            self.vision_config = self.sub_configs["vision_config"](**vision_config)
+        elif vision_config is None:
+            self.vision_config = self.sub_configs["vision_config"]()        
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.max_position_embeddings = max_position_embeddings
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.rms_norm_eps = rms_norm_eps
+        self.use_cache = use_cache
+        self.use_flash_attention = use_flash_attention
+        self.pad_token_id = pad_token_id
+        self.bos_token_id = bos_token_id
+        self.eos_token_id = eos_token_id
+        self.image_token_id = image_token_id
+        self.video_token_id = video_token_id
+        self.vision_start_token_id = vision_start_token_id
+        self.head_dim = head_dim
+        self.hidden_act=hidden_act
+        self.sliding_window = None
+        self.hidden_size = hidden_size
+        self.use_bias = use_bias
+        self.weight_share_add_bias = weight_share_add_bias
+        self.rope_theta = rope_theta
+        self.ignored_index = ignored_index
+        self.attention_probs_dropout_prob = attention_probs_dropout_prob
+        self.hidden_dropout_prob = hidden_dropout_prob
+        self.compression_ratio = compression_ratio
+        self.num_key_value_heads = num_key_value_heads
+        self.max_sequence_length = max_sequence_length
+        self.rope_scaling = rope_scaling
+        if self.rope_scaling is not None and "type" in self.rope_scaling:
+            if self.rope_scaling["type"] == "mrope":
+                self.rope_scaling["type"] = "default"
+            self.rope_scaling["rope_type"] = self.rope_scaling["type"]
+        rope_config_validation(self, ignore_keys={"mrope_section"})        
+        super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)

model/extracted_vision_encoder.py

@@ -0,0 +1,512 @@
+import json
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
+
+import numpy as np
+import torch
+from transformers.modeling_outputs import BaseModelOutputWithPooling
+from transformers.processing_utils import BatchFeature
+
+from .configuration_paddleocr_vl import PaddleOCRVLConfig
+from .image_processing_paddleocr_vl import PaddleOCRVLImageProcessor
+from .modeling_paddleocr_vl import PaddleOCRVisionModel, Projector
+
+
+VISION_TOWER_CONFIG_NAME = "vision_tower_config.json"
+VISION_TOWER_WEIGHTS_NAME = "vision_tower.safetensors"
+PROJECTOR_CONFIG_NAME = "projector_config.json"
+PROJECTOR_WEIGHTS_NAME = "projector.safetensors"
+FULL_MODEL_CONFIG_NAME = "config.json"
+FULL_MODEL_WEIGHTS_NAME = "model.safetensors"
+FULL_VISUAL_PREFIX = "visual."
+FULL_PROJECTOR_PREFIX = "mlp_AR."
+STANDALONE_VISUAL_PREFIX = "visual."
+STANDALONE_PROJECTOR_PREFIX = "projector."
+
+
+def _read_json(path: Union[str, Path]) -> Dict[str, Any]:
+    with open(path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def _write_json(path: Union[str, Path], payload: Dict[str, Any]) -> None:
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(payload, f, indent=2, ensure_ascii=False)
+
+
+def _normalize_image_grid_thw(
+    image_grid_thw: Union[torch.Tensor, Sequence[Any]]
+) -> List[Tuple[int, int, int]]:
+    if isinstance(image_grid_thw, torch.Tensor):
+        return [tuple(int(v) for v in row.tolist()) for row in image_grid_thw]
+
+    normalized: List[Tuple[int, int, int]] = []
+    for item in image_grid_thw:
+        if isinstance(item, torch.Tensor):
+            normalized.append(tuple(int(v) for v in item.tolist()))
+        else:
+            normalized.append(tuple(int(v) for v in item))
+    return normalized
+
+
+def build_vision_encoder_export_config(
+    full_config: Union[PaddleOCRVLConfig, Dict[str, Any]]
+) -> Dict[str, Any]:
+    if isinstance(full_config, PaddleOCRVLConfig):
+        full_config_dict = full_config.to_dict()
+    else:
+        full_config_dict = dict(full_config)
+
+    vision_config = dict(full_config_dict["vision_config"])
+
+    return {
+        "model_type": "paddleocr_vl_vision_encoder",
+        "architectures": ["PaddleOCRVLVisionEncoder"],
+        "source_model_type": full_config_dict.get("model_type", "paddleocr_vl"),
+        "source_architecture": "PaddleOCRVLForConditionalGeneration",
+        "text_hidden_size": full_config_dict["hidden_size"],
+        "image_token_id": full_config_dict.get("image_token_id"),
+        "vision_start_token_id": full_config_dict.get("vision_start_token_id"),
+        "vision_end_token_id": full_config_dict.get("vision_end_token_id"),
+        "torch_dtype": full_config_dict.get("torch_dtype"),
+        "vision_config": vision_config,
+        "projector": {
+            "merge_kernel_size": [2, 2],
+            "input_hidden_size": vision_config["hidden_size"],
+            "output_hidden_size": full_config_dict["hidden_size"],
+        },
+        "required_weight_prefixes": [
+            STANDALONE_VISUAL_PREFIX,
+            STANDALONE_PROJECTOR_PREFIX,
+        ],
+        "source_weight_prefixes": {
+            "visual": FULL_VISUAL_PREFIX,
+            "projector": FULL_PROJECTOR_PREFIX,
+        },
+        "full_model_config": full_config_dict,
+    }
+
+
+def build_vision_tower_export_config(
+    full_config: Union[PaddleOCRVLConfig, Dict[str, Any]]
+) -> Dict[str, Any]:
+    combined = build_vision_encoder_export_config(full_config)
+    return {
+        "model_type": "paddleocr_vl_vision_tower",
+        "architectures": ["PaddleOCRVLVisionTower"],
+        "torch_dtype": combined.get("torch_dtype"),
+        "vision_config": combined["vision_config"],
+        "required_weight_prefixes": [STANDALONE_VISUAL_PREFIX],
+        "source_weight_prefixes": {"visual": FULL_VISUAL_PREFIX},
+        "full_model_config": combined["full_model_config"],
+    }
+
+
+def build_projector_export_config(
+    full_config: Union[PaddleOCRVLConfig, Dict[str, Any]]
+) -> Dict[str, Any]:
+    combined = build_vision_encoder_export_config(full_config)
+    return {
+        "model_type": "paddleocr_vl_projector",
+        "architectures": ["PaddleOCRVLProjector"],
+        "torch_dtype": combined.get("torch_dtype"),
+        "vision_config": combined["vision_config"],
+        "text_hidden_size": combined["text_hidden_size"],
+        "projector": combined["projector"],
+        "required_weight_prefixes": [STANDALONE_PROJECTOR_PREFIX],
+        "source_weight_prefixes": {"projector": FULL_PROJECTOR_PREFIX},
+        "full_model_config": combined["full_model_config"],
+    }
+
+
+def remap_full_model_state_dict_to_vision_encoder_parts(
+    full_state_dict: Dict[str, torch.Tensor]
+) -> Tuple[Dict[str, torch.Tensor], Dict[str, torch.Tensor], Dict[str, List[str]]]:
+    visual_state_dict: Dict[str, torch.Tensor] = {}
+    projector_state_dict: Dict[str, torch.Tensor] = {}
+    consumed_visual: List[str] = []
+    consumed_projector: List[str] = []
+
+    for key, value in full_state_dict.items():
+        if key.startswith(FULL_VISUAL_PREFIX):
+            new_key = STANDALONE_VISUAL_PREFIX + key[len(FULL_VISUAL_PREFIX) :]
+            visual_state_dict[new_key] = value
+            consumed_visual.append(key)
+        elif key.startswith(FULL_PROJECTOR_PREFIX):
+            new_key = STANDALONE_PROJECTOR_PREFIX + key[len(FULL_PROJECTOR_PREFIX) :]
+            projector_state_dict[new_key] = value
+            consumed_projector.append(key)
+
+    if not consumed_visual:
+        raise ValueError("No visual.* weights were found in the full model state dict.")
+    if not consumed_projector:
+        raise ValueError("No mlp_AR.* weights were found in the full model state dict.")
+
+    return visual_state_dict, projector_state_dict, {
+        "visual": sorted(consumed_visual),
+        "projector": sorted(consumed_projector),
+    }
+
+
+def remap_full_model_state_dict_to_vision_encoder(
+    full_state_dict: Dict[str, torch.Tensor]
+) -> Tuple[Dict[str, torch.Tensor], Dict[str, List[str]]]:
+    visual_state_dict, projector_state_dict, consumed = (
+        remap_full_model_state_dict_to_vision_encoder_parts(full_state_dict)
+    )
+    remapped = {}
+    remapped.update(visual_state_dict)
+    remapped.update(projector_state_dict)
+    return remapped, consumed
+
+
+def _load_safetensors_state_dict(path: Union[str, Path]) -> Dict[str, torch.Tensor]:
+    try:
+        from safetensors.torch import load_file
+    except ImportError as e:
+        raise RuntimeError(
+            "Loading safetensors requires the `safetensors` package to be installed."
+        ) from e
+
+    return load_file(str(path))
+
+
+def _save_safetensors_state_dict(
+    state_dict: Dict[str, torch.Tensor], path: Union[str, Path]
+) -> None:
+    try:
+        from safetensors.torch import save_file
+    except ImportError as e:
+        raise RuntimeError(
+            "Saving safetensors requires the `safetensors` package to be installed."
+        ) from e
+
+    save_file(state_dict, str(path))
+
+
+def extract_and_save_vision_encoder_artifacts(
+    full_config: Union[PaddleOCRVLConfig, Dict[str, Any]],
+    full_state_dict: Dict[str, torch.Tensor],
+    output_dir: Union[str, Path],
+) -> Dict[str, Any]:
+    output_dir = Path(output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    vision_tower_config = build_vision_tower_export_config(full_config)
+    projector_config = build_projector_export_config(full_config)
+    visual_state_dict, projector_state_dict, consumed = (
+        remap_full_model_state_dict_to_vision_encoder_parts(full_state_dict)
+    )
+    _save_safetensors_state_dict(
+        visual_state_dict, output_dir / VISION_TOWER_WEIGHTS_NAME
+    )
+    _write_json(output_dir / VISION_TOWER_CONFIG_NAME, vision_tower_config)
+    _save_safetensors_state_dict(
+        projector_state_dict, output_dir / PROJECTOR_WEIGHTS_NAME
+    )
+    _write_json(output_dir / PROJECTOR_CONFIG_NAME, projector_config)
+
+    combined_export_config = build_vision_encoder_export_config(full_config)
+    combined_state_dict, _ = remap_full_model_state_dict_to_vision_encoder(
+        full_state_dict
+    )
+    combined_dir = output_dir / "combined"
+    combined_dir.mkdir(parents=True, exist_ok=True)
+    _save_safetensors_state_dict(
+        combined_state_dict, combined_dir / "vision_encoder.safetensors"
+    )
+    _write_json(combined_dir / "vision_encoder_config.json", combined_export_config)
+
+    metadata = {
+        "vision_tower_config_path": str(output_dir / VISION_TOWER_CONFIG_NAME),
+        "vision_tower_weights_path": str(output_dir / VISION_TOWER_WEIGHTS_NAME),
+        "projector_config_path": str(output_dir / PROJECTOR_CONFIG_NAME),
+        "projector_weights_path": str(output_dir / PROJECTOR_WEIGHTS_NAME),
+        "combined_config_path": str(combined_dir / "vision_encoder_config.json"),
+        "combined_weights_path": str(combined_dir / "vision_encoder.safetensors"),
+        "num_exported_visual_tensors": len(visual_state_dict),
+        "num_exported_projector_tensors": len(projector_state_dict),
+        "consumed_full_model_keys": consumed,
+    }
+    return metadata
+
+
+class PaddleOCRVLVisionTower(torch.nn.Module):
+    def __init__(self, config: PaddleOCRVLConfig):
+        super().__init__()
+        self.config = config
+        self.visual = PaddleOCRVisionModel(config.vision_config)
+        self.export_config = build_vision_tower_export_config(config)
+
+    @staticmethod
+    def _resolve_full_config(config_payload: Dict[str, Any]) -> PaddleOCRVLConfig:
+        if config_payload.get("model_type") == "paddleocr_vl_vision_tower":
+            config_payload = config_payload["full_model_config"]
+        return PaddleOCRVLConfig(**config_payload)
+
+    @classmethod
+    def from_pretrained(cls, model_dir: Union[str, Path]) -> "PaddleOCRVLVisionTower":
+        model_dir = Path(model_dir)
+        config_path = model_dir / VISION_TOWER_CONFIG_NAME
+        weights_path = model_dir / VISION_TOWER_WEIGHTS_NAME
+        if config_path.exists():
+            config_payload = _read_json(config_path)
+        else:
+            config_payload = _read_json(model_dir / FULL_MODEL_CONFIG_NAME)
+        model = cls(cls._resolve_full_config(config_payload))
+        if weights_path.exists():
+            state_dict = _load_safetensors_state_dict(weights_path)
+        else:
+            full_state_dict = _load_safetensors_state_dict(model_dir / FULL_MODEL_WEIGHTS_NAME)
+            state_dict, _, _ = remap_full_model_state_dict_to_vision_encoder_parts(
+                full_state_dict
+            )
+        missing, unexpected = model.load_state_dict(state_dict, strict=True)
+        if missing or unexpected:
+            raise RuntimeError(
+                f"Failed to load standalone vision tower weights. Missing: {missing}, unexpected: {unexpected}"
+            )
+        return model
+
+    def save_pretrained(self, output_dir: Union[str, Path]) -> None:
+        output_dir = Path(output_dir)
+        output_dir.mkdir(parents=True, exist_ok=True)
+        _save_safetensors_state_dict(self.state_dict(), output_dir / VISION_TOWER_WEIGHTS_NAME)
+        _write_json(output_dir / VISION_TOWER_CONFIG_NAME, self.export_config)
+
+    @staticmethod
+    def _build_visual_inputs(
+        pixel_values: torch.Tensor,
+        image_grid_thw: List[Tuple[int, int, int]],
+        device: torch.device,
+    ) -> Tuple[
+        torch.Tensor,
+        torch.Tensor,
+        List[Tuple[int, int, int]],
+        torch.Tensor,
+        torch.Tensor,
+    ]:
+        if pixel_values.dim() == 4:
+            pixel_values = pixel_values.unsqueeze(0)
+        elif pixel_values.dim() != 5:
+            raise ValueError(
+                "pixel_values must have shape [num_patches, C, H, W] or [1, num_patches, C, H, W]."
+            )
+
+        siglip_position_ids = []
+        sample_indices = []
+        cu_seqlens = [0]
+
+        for idx, thw in enumerate(image_grid_thw):
+            numel = int(np.prod(thw))
+            image_position_ids = torch.arange(numel, device=device) % int(np.prod(thw[1:]))
+            siglip_position_ids.append(image_position_ids)
+            sample_indices.append(torch.full((numel,), idx, dtype=torch.int64, device=device))
+            cu_seqlens.append(cu_seqlens[-1] + numel)
+
+        if siglip_position_ids:
+            siglip_position_ids = torch.cat(siglip_position_ids, dim=0)
+            sample_indices = torch.cat(sample_indices, dim=0)
+        else:
+            siglip_position_ids = torch.empty(0, dtype=torch.long, device=device)
+            sample_indices = torch.empty(0, dtype=torch.long, device=device)
+
+        cu_seqlens_tensor = torch.tensor(cu_seqlens, dtype=torch.int32, device=device)
+        return pixel_values, siglip_position_ids, image_grid_thw, sample_indices, cu_seqlens_tensor
+
+    def forward(
+        self,
+        pixel_values: torch.Tensor,
+        image_grid_thw: Union[torch.Tensor, Sequence[Any]],
+    ) -> Dict[str, Any]:
+        image_grid_thw_list = _normalize_image_grid_thw(image_grid_thw)
+        vision_dtype = next(self.visual.parameters()).dtype
+        pixel_values = pixel_values.to(dtype=vision_dtype)
+        device = pixel_values.device
+
+        (
+            pixel_values_5d,
+            siglip_position_ids,
+            image_grid_hws,
+            sample_indices,
+            cu_seqlens,
+        ) = self._build_visual_inputs(pixel_values, image_grid_thw_list, device)
+
+        vision_outputs: BaseModelOutputWithPooling = self.visual(
+            pixel_values=pixel_values_5d,
+            image_grid_thw=image_grid_hws,
+            position_ids=siglip_position_ids,
+            vision_return_embed_list=True,
+            interpolate_pos_encoding=True,
+            sample_indices=sample_indices,
+            cu_seqlens=cu_seqlens,
+            return_pooler_output=False,
+            use_rope=True,
+            window_size=-1,
+        )
+        return {
+            "visual_embeds": vision_outputs.last_hidden_state,
+            "image_grid_thw": image_grid_thw_list,
+            "siglip_position_ids": siglip_position_ids,
+            "sample_indices": sample_indices,
+            "cu_seqlens": cu_seqlens,
+        }
+
+    def encode_images(
+        self,
+        images: Any,
+        image_processor: Optional[PaddleOCRVLImageProcessor] = None,
+        **processor_kwargs: Any,
+    ) -> Dict[str, Any]:
+        image_processor = image_processor or PaddleOCRVLImageProcessor(
+            patch_size=self.config.vision_config.patch_size,
+            temporal_patch_size=self.config.vision_config.temporal_patch_size,
+            merge_size=self.config.vision_config.spatial_merge_size,
+        )
+        encoded: BatchFeature = image_processor(
+            images=images, return_tensors="pt", **processor_kwargs
+        )
+        return self.forward(
+            pixel_values=encoded["pixel_values"], image_grid_thw=encoded["image_grid_thw"]
+        )
+
+
+class PaddleOCRVLProjector(torch.nn.Module):
+    def __init__(self, config: PaddleOCRVLConfig):
+        super().__init__()
+        self.config = config
+        self.projector = Projector(config, config.vision_config)
+        self.export_config = build_projector_export_config(config)
+
+    @staticmethod
+    def _resolve_full_config(config_payload: Dict[str, Any]) -> PaddleOCRVLConfig:
+        if config_payload.get("model_type") == "paddleocr_vl_projector":
+            config_payload = config_payload["full_model_config"]
+        return PaddleOCRVLConfig(**config_payload)
+
+    @classmethod
+    def from_pretrained(cls, model_dir: Union[str, Path]) -> "PaddleOCRVLProjector":
+        model_dir = Path(model_dir)
+        config_path = model_dir / PROJECTOR_CONFIG_NAME
+        weights_path = model_dir / PROJECTOR_WEIGHTS_NAME
+
+        if config_path.exists():
+            config_payload = _read_json(config_path)
+        else:
+            config_payload = _read_json(model_dir / FULL_MODEL_CONFIG_NAME)
+
+        model = cls(cls._resolve_full_config(config_payload))
+
+        if weights_path.exists():
+            state_dict = _load_safetensors_state_dict(weights_path)
+        else:
+            full_state_dict = _load_safetensors_state_dict(model_dir / FULL_MODEL_WEIGHTS_NAME)
+            _, state_dict, _ = remap_full_model_state_dict_to_vision_encoder_parts(
+                full_state_dict
+            )
+
+        missing, unexpected = model.load_state_dict(state_dict, strict=True)
+        if missing or unexpected:
+            raise RuntimeError(
+                f"Failed to load standalone projector weights. Missing: {missing}, unexpected: {unexpected}"
+            )
+        return model
+
+    def save_pretrained(self, output_dir: Union[str, Path]) -> None:
+        output_dir = Path(output_dir)
+        output_dir.mkdir(parents=True, exist_ok=True)
+        _save_safetensors_state_dict(self.state_dict(), output_dir / PROJECTOR_WEIGHTS_NAME)
+        _write_json(output_dir / PROJECTOR_CONFIG_NAME, self.export_config)
+
+    def forward(
+        self,
+        visual_embeds: Union[torch.Tensor, List[torch.Tensor], Tuple[torch.Tensor, ...]],
+        image_grid_thw: Union[torch.Tensor, Sequence[Any]],
+    ) -> Dict[str, Any]:
+        image_grid_thw_list = _normalize_image_grid_thw(image_grid_thw)
+        image_embeds = self.projector(visual_embeds, image_grid_thw_list)
+        projector_dtype = next(self.projector.parameters()).dtype
+        projector_device = next(self.projector.parameters()).device
+        concat_image_embeds = (
+            torch.cat(image_embeds, dim=0)
+            if image_embeds
+            else torch.empty(
+                0,
+                self.config.hidden_size,
+                device=projector_device,
+                dtype=projector_dtype,
+            )
+        )
+        return {
+            "image_embeds": image_embeds,
+            "concat_image_embeds": concat_image_embeds,
+            "image_grid_thw": image_grid_thw_list,
+        }
+
+class PaddleOCRVLVisionEncoder(torch.nn.Module):
+    def __init__(self, config: PaddleOCRVLConfig):
+        super().__init__()
+        self.config = config
+        self.vision_tower = PaddleOCRVLVisionTower(config)
+        self.projector = PaddleOCRVLProjector(config)
+        self.export_config = build_vision_encoder_export_config(config)
+
+    @classmethod
+    def from_pretrained(cls, model_dir: Union[str, Path]) -> "PaddleOCRVLVisionEncoder":
+        model_dir = Path(model_dir)
+        config_candidates = [
+            model_dir / FULL_MODEL_CONFIG_NAME,
+            model_dir / VISION_TOWER_CONFIG_NAME,
+            model_dir / PROJECTOR_CONFIG_NAME,
+        ]
+        config_path = next((path for path in config_candidates if path.exists()), None)
+        if config_path is None:
+            raise FileNotFoundError(
+                "Could not find config.json, vision_tower_config.json, or projector_config.json."
+            )
+        config_payload = _read_json(config_path)
+        if config_payload.get("model_type") == "paddleocr_vl_vision_tower":
+            config = PaddleOCRVLVisionTower._resolve_full_config(config_payload)
+        elif config_payload.get("model_type") == "paddleocr_vl_projector":
+            config = PaddleOCRVLProjector._resolve_full_config(config_payload)
+        else:
+            config = PaddleOCRVLProjector._resolve_full_config(config_payload)
+        model = cls(config)
+        model.vision_tower = PaddleOCRVLVisionTower.from_pretrained(model_dir)
+        model.projector = PaddleOCRVLProjector.from_pretrained(model_dir)
+        return model
+
+    def forward(
+        self,
+        pixel_values: torch.Tensor,
+        image_grid_thw: Union[torch.Tensor, Sequence[Any]],
+    ) -> Dict[str, Any]:
+        vision_outputs = self.vision_tower(
+            pixel_values=pixel_values,
+            image_grid_thw=image_grid_thw,
+        )
+        projector_outputs = self.projector(
+            visual_embeds=vision_outputs["visual_embeds"],
+            image_grid_thw=vision_outputs["image_grid_thw"],
+        )
+        return {
+            **vision_outputs,
+            **projector_outputs,
+        }
+
+    def encode_images(
+        self,
+        images: Any,
+        image_processor: Optional[PaddleOCRVLImageProcessor] = None,
+        **processor_kwargs: Any,
+    ) -> Dict[str, Any]:
+        vision_outputs = self.vision_tower.encode_images(
+            images=images,
+            image_processor=image_processor,
+            **processor_kwargs,
+        )
+        projector_outputs = self.projector(
+            visual_embeds=vision_outputs["visual_embeds"],
+            image_grid_thw=vision_outputs["image_grid_thw"],
+        )
+        return {**vision_outputs, **projector_outputs}

model/image_processing_paddleocr_vl.py

@@ -0,0 +1,569 @@
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Image processor class for PaddleOCR-VL."""
+
+import math
+from typing import Dict, List, Optional, Union
+
+import numpy as np
+import torch
+from transformers.image_processing_utils import BaseImageProcessor, BatchFeature
+from torchvision.transforms import functional as TF
+from transformers.image_transforms import (
+    convert_to_rgb,
+    resize,
+    to_channel_dimension_format,
+)
+from transformers.image_utils import (
+    OPENAI_CLIP_MEAN,
+    OPENAI_CLIP_STD,
+    ChannelDimension,
+    PILImageResampling,
+    get_image_size,
+    infer_channel_dimension_format,
+    is_scaled_image,
+    is_valid_image,
+    make_list_of_images,
+    to_numpy_array,
+    valid_images,
+    validate_preprocess_arguments,
+)
+from transformers.utils import TensorType, is_vision_available, logging
+
+
+logger = logging.get_logger(__name__)
+
+
+if is_vision_available():
+    from PIL import Image
+
+ImageInput = Union[
+    "PIL.Image.Image",
+    np.ndarray,
+    "torch.Tensor",
+    List["PIL.Image.Image"],
+    List[np.ndarray],
+    List["torch.Tensor"],
+]  # noqa
+
+
+VideoInput = Union[
+    List["PIL.Image.Image"],
+    "np.ndarray",
+    "torch.Tensor",
+    List["np.ndarray"],
+    List["torch.Tensor"],
+    List[List["PIL.Image.Image"]],
+    List[List["np.ndarrray"]],
+    List[List["torch.Tensor"]],
+]  # noqa
+
+
+def make_batched_images(images) -> List[List[ImageInput]]:
+    """
+    Accepts images in list or nested list format, and makes a list of images for preprocessing.
+
+    Args:
+        images (`Union[List[List[ImageInput]], List[ImageInput], ImageInput]`):
+            The input image.
+
+    Returns:
+        list: A list of images.
+    """
+    if (
+        isinstance(images, (list, tuple))
+        and isinstance(images[0], (list, tuple))
+        and is_valid_image(images[0][0])
+    ):
+        return [img for img_list in images for img in img_list]
+
+    elif isinstance(images, (list, tuple)) and is_valid_image(images[0]):
+        return images
+
+    elif is_valid_image(images):
+        return [images]
+
+    raise ValueError(f"Could not make batched images from {images}")
+
+
+def adjust_size(size, patch_size):
+    num_patches = size // patch_size
+    if num_patches % 2 != 0:  # 如果是奇数，减1
+        num_patches -= 1
+    return num_patches * patch_size
+
+
+def make_batched_videos(videos) -> List[VideoInput]:
+    if (
+        isinstance(videos, (list, tuple))
+        and isinstance(videos[0], (list, tuple))
+        and is_valid_image(videos[0][0])
+    ):
+        return videos
+
+    elif isinstance(videos, (list, tuple)) and is_valid_image(videos[0]):
+        if isinstance(videos[0], Image.Image):
+            return [videos]
+        elif len(videos[0].shape) == 4:
+            return [list(video) for video in videos]
+
+    elif is_valid_image(videos) and len(videos.shape) == 4:
+        return [list(videos)]
+
+    raise ValueError(f"Could not make batched video from {videos}")
+
+
+def smart_resize(
+    height: int,
+    width: int,
+    factor: int = 28,
+    min_pixels: int = 28 * 28 * 130,
+    max_pixels: int = 28 * 28 * 1280,
+):
+    """Rescales the image so that the following conditions are met:
+
+    1. Both dimensions (height and width) are divisible by 'factor'.
+
+    2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
+
+    3. The aspect ratio of the image is maintained as closely as possible.
+
+    """
+    # if height < factor or width < factor:
+    #    raise ValueError(f"height:{height} or width:{width} must be larger than factor:{factor}")
+    # if int(height < factor//4) + int(width < factor//4):
+    #     raise ValueError(f"height:{height} or width:{width} must be larger than factor:{factor//4}")
+
+    if height < factor:
+        print(f"smart_resize: height={height} < factor={factor}, reset height=factor")
+        width = round((width * factor) / height)
+        height = factor
+
+    if width < factor:
+        print(f"smart_resize: width={width} < factor={factor}, reset width=factor")
+        height = round((height * factor) / width)
+        width = factor
+
+    if max(height, width) / min(height, width) > 200:
+        raise ValueError(
+            f"absolute aspect ratio must be smaller than 200, got {max(height, width) / min(height, width)}"
+        )
+    h_bar = round(height / factor) * factor
+    w_bar = round(width / factor) * factor
+    if h_bar * w_bar > max_pixels:
+        beta = math.sqrt((height * width) / max_pixels)
+        h_bar = math.floor(height / beta / factor) * factor
+        w_bar = math.floor(width / beta / factor) * factor
+    elif h_bar * w_bar < min_pixels:
+        beta = math.sqrt(min_pixels / (height * width))
+        h_bar = math.ceil(height * beta / factor) * factor
+        w_bar = math.ceil(width * beta / factor) * factor
+    return h_bar, w_bar
+
+
+class PaddleOCRVLImageProcessor(BaseImageProcessor):
+    r"""
+    Constructs a Siglip image processor that dynamically resizes images based on the original images.
+
+    Args:
+        do_resize (`bool`, *optional*, defaults to `True`):
+            Whether to resize the image's (height, width) dimensions.
+        resample (`PILImageResampling`, *optional*, defaults to `Resampling.BICUBIC`):
+            Resampling filter to use when resizing the image.
+        do_rescale (`bool`, *optional*, defaults to `True`):
+            Whether to rescale the image by the specified scale `rescale_factor`.
+        rescale_factor (`int` or `float`, *optional*, defaults to `1/255`):
+            Scale factor to use if rescaling the image.
+        do_normalize (`bool`, *optional*, defaults to `True`):
+            Whether to normalize the image.
+        image_mean (`float` or `List[float]`, *optional*, defaults to `[0.48145466, 0.4578275, 0.40821073]`):
+            Mean to use if normalizing the image. This is a float or list of floats for each channel in the image.
+        image_std (`float` or `List[float]`, *optional*, defaults to `[0.26862954, 0.26130258, 0.27577711]`):
+            Standard deviation to use if normalizing the image. This is a float or list of floats for each channel in the image.
+        do_convert_rgb (`bool`, *optional*, defaults to `True`):
+            Whether to convert the image to RGB.
+        min_pixels (`int`, *optional*, defaults to `28 * 28 * 130`):
+            The min pixels of the image to resize the image.
+        max_pixels (`int`, *optional*, defaults to `28 * 28 * 1670`):
+            The max pixels of the image to resize the image.
+        patch_size (`int`, *optional*, defaults to 14):
+            The spacial patch size of the vision encoder.
+        temporal_patch_size (`int`, *optional*, defaults to 2):
+            The temporal patch size of the vision encoder.
+        merge_size (`int`, *optional*, defaults to 2):
+            The merge size of the vision encoder to llm encoder.
+    """
+
+    model_input_names = [
+        "pixel_values",
+        "image_grid_thw",
+        "pixel_values_videos",
+        "video_grid_thw",
+    ]
+
+    def __init__(
+        self,
+        do_resize: bool = True,
+        resample: PILImageResampling = PILImageResampling.BICUBIC,
+        do_rescale: bool = True,
+        rescale_factor: Union[int, float] = 1 / 255,
+        do_normalize: bool = True,
+        image_mean: Optional[Union[float, List[float]]] = None,
+        image_std: Optional[Union[float, List[float]]] = None,
+        do_convert_rgb: bool = True,
+        min_pixels: int = 28 * 28 * 130,
+        max_pixels: int = 28 * 28 * 1280,
+        patch_size: int = 14,
+        temporal_patch_size: int = 1,
+        merge_size: int = 2,
+        **kwargs,
+    ) -> None:
+        super().__init__(**kwargs)
+        self.do_resize = do_resize
+        self.resample = resample
+        self.do_rescale = do_rescale
+        self.rescale_factor = rescale_factor
+        self.do_normalize = do_normalize
+        self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN
+        self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD
+        self.min_pixels = min_pixels
+        self.max_pixels = max_pixels
+        self.patch_size = patch_size
+        self.temporal_patch_size = temporal_patch_size
+        self.merge_size = merge_size
+        self.size = {"min_pixels": min_pixels, "max_pixels": max_pixels}  # not used
+        self.do_convert_rgb = do_convert_rgb
+
+    def mvit_rescale(self, image: Image.Image, merge_size: int = 2) -> Image.Image:
+        try:
+            w, h = image.size
+        except:
+            raise ValueError(str((type(image), image)))
+        patch_size = self.patch_size
+
+        if (w // patch_size) * (h // patch_size) > self.in_token_limit:
+            scale = math.sqrt(
+                self.in_token_limit / ((w // patch_size) * (h // patch_size))
+            )
+            new_w, new_h = int(w * scale), int(h * scale)
+
+            image = image.resize((new_w, new_h), Image.Resampling.BICUBIC)
+        if self.pad_input:
+            new_w, new_h = image.size
+            pad_size_h = merge_size * patch_size
+            pad_size_w = merge_size * patch_size
+
+            pad_h = (pad_size_h - new_h % pad_size_h) % pad_size_h
+            pad_w = (pad_size_w - new_w % pad_size_w) % pad_size_w
+
+            image = TF.pad(image, (0, 0, pad_w, pad_h))
+        else:
+            new_w, new_h = image.size
+            new_w = new_w - new_w % patch_size
+            new_h = new_h - new_h % patch_size
+
+            new_w = adjust_size(new_w, patch_size)
+            new_h = adjust_size(new_h, patch_size)
+
+            image = TF.center_crop(image, (new_h, new_w))
+
+        w, h = image.size
+        if w // patch_size >= 512 or h // patch_size >= 512:
+            new_h = min(patch_size * 510, h)
+            new_w = min(patch_size * 510, w)
+            image = TF.center_crop(image, (new_h, new_w))
+            # raise ValueError("Exceed pos emb")
+        return image
+
+    def _preprocess(
+        self,
+        images: Union[ImageInput, VideoInput],
+        do_resize: bool = None,
+        resample: PILImageResampling = None,
+        do_rescale: bool = None,
+        rescale_factor: float = None,
+        do_normalize: bool = None,
+        image_mean: Optional[Union[float, List[float]]] = None,
+        image_std: Optional[Union[float, List[float]]] = None,
+        do_convert_rgb: bool = None,
+        data_format: Optional[ChannelDimension] = ChannelDimension.FIRST,
+        input_data_format: Optional[Union[str, ChannelDimension]] = None,
+    ):
+        """
+        Preprocess an image or batch of images. Copy of the `preprocess` method from `CLIPImageProcessor`.
+
+        Args:
+            images (`ImageInput`):
+                Image or batch of images to preprocess. Expects pixel values ranging from 0 to 255. If pixel values range from 0 to 1, set `do_rescale=False`.
+            vision_info (`List[Dict]`, *optional*):
+                Optional list of dictionaries containing additional information about vision inputs.
+            do_resize (`bool`, *optional*, defaults to `self.do_resize`):
+                Whether to resize the image.
+            resample (`PILImageResampling`, *optional*, defaults to `self.resample`):
+                Resampling filter to use if resizing the image. This can be one of the `PILImageResampling` enums.
+            do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):
+                Whether to rescale the image.
+            rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
+                Scale factor to use if rescaling the image.
+            do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):
+                Whether to normalize the image.
+            image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):
+                Mean to use if normalizing the image. Can be a float or a list of floats corresponding to the number of channels in the image.
+            image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):
+                Standard deviation to use if normalizing the image. Can be a float or a list of floats corresponding to the number of channels in the image.
+            do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
+                Whether to convert the image to RGB.
+            data_format (`ChannelDimension`, *optional*, defaults to `ChannelDimension.FIRST`):
+                The channel dimension format for the output image. Can be one of:
+                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                - Unset: Use the channel dimension format of the input image.
+            input_data_format (`ChannelDimension` or `str`, *optional*):
+                The channel dimension format for the input image. Can be one of:
+                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.   - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
+        """
+        images = make_list_of_images(images)
+
+        if do_convert_rgb:
+            images = [convert_to_rgb(image) for image in images]
+
+        # All transformations expect numpy arrays.
+        images = [to_numpy_array(image) for image in images]
+
+        if is_scaled_image(images[0]) and do_rescale:
+            logger.warning_once(
+                "It looks like you are trying to rescale already rescaled images. If the input"
+                " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again."
+            )
+        if input_data_format is None:
+            # We assume that all images have the same channel dimension format.
+            input_data_format = infer_channel_dimension_format(images[0])
+
+        height, width = get_image_size(images[0], channel_dim=input_data_format)
+        resized_height, resized_width = height, width
+        processed_images = []
+
+        for image in images:
+            if do_resize:
+                resized_height, resized_width = smart_resize(
+                    height,
+                    width,
+                    factor=self.patch_size * self.merge_size,
+                    min_pixels=self.min_pixels,
+                    max_pixels=self.max_pixels,
+                )
+                image = resize(
+                    image,
+                    size=(resized_height, resized_width),
+                    resample=resample,
+                    input_data_format=input_data_format,
+                )
+
+            if do_rescale:
+                image = self.rescale(
+                    image, scale=rescale_factor, input_data_format=input_data_format
+                )
+
+            if do_normalize:
+                image = self.normalize(
+                    image=image,
+                    mean=image_mean,
+                    std=image_std,
+                    input_data_format=input_data_format,
+                )
+            image = to_channel_dimension_format(
+                image, data_format, input_channel_dim=input_data_format
+            )
+            processed_images.append(image)
+
+        patches = np.array(processed_images)
+        if data_format == ChannelDimension.LAST:
+            patches = patches.transpose(0, 3, 1, 2)
+        if patches.shape[0] == 1:
+            patches = np.tile(patches, (self.temporal_patch_size, 1, 1, 1))
+        init_patches = patches
+        channel = patches.shape[1]
+        grid_t = patches.shape[0] // self.temporal_patch_size
+        grid_h, grid_w = (
+            resized_height // self.patch_size,
+            resized_width // self.patch_size,
+        )
+        patches = patches.reshape(
+            grid_t,
+            self.temporal_patch_size,
+            channel,
+            grid_h,
+            self.patch_size,
+            grid_w,
+            self.patch_size,
+        )
+        patches = patches.transpose(0, 3, 5, 2, 1, 4, 6)
+        assert self.temporal_patch_size == 1
+        flatten_patches = patches.reshape(
+            grid_t * grid_h * grid_w, channel, self.patch_size, self.patch_size
+        )
+        return flatten_patches, (grid_t, grid_h, grid_w)
+
+    def preprocess(
+        self,
+        images: ImageInput,
+        videos: VideoInput = None,
+        do_resize: bool = None,
+        size: Dict[str, int] = None,
+        resample: PILImageResampling = None,
+        do_rescale: bool = None,
+        rescale_factor: float = None,
+        do_normalize: bool = None,
+        image_mean: Optional[Union[float, List[float]]] = None,
+        image_std: Optional[Union[float, List[float]]] = None,
+        do_convert_rgb: bool = None,
+        return_tensors: Optional[Union[str, TensorType]] = None,
+        data_format: Optional[ChannelDimension] = ChannelDimension.FIRST,
+        input_data_format: Optional[Union[str, ChannelDimension]] = None,
+    ):
+        """
+        Args:
+            images (`ImageInput`):
+                Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If
+                passing in images with pixel values between 0 and 1, set `do_rescale=False`.
+            videos (`VideoInput`):
+                Video to preprocess. Expects a single or batch of videos with pixel values ranging from 0 to 255. If
+                passing in videos with pixel values between 0 and 1, set `do_rescale=False`.
+            do_resize (`bool`, *optional*, defaults to `self.do_resize`):
+                Whether to resize the image.
+            size (`Dict[str, int]`, *optional*, defaults to `self.size`):
+                Size of the image after resizing. Shortest edge of the image is resized to size["shortest_edge"], with
+                the longest edge resized to keep the input aspect ratio.
+            resample (`int`, *optional*, defaults to `self.resample`):
+                Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only
+                has an effect if `do_resize` is set to `True`.
+            do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):
+                Whether to rescale the image.
+            rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
+                Rescale factor to rescale the image by if `do_rescale` is set to `True`.
+            do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):
+                Whether to normalize the image.
+            image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):
+                Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.
+            image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):
+                Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to
+                `True`.
+            do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
+                Whether to convert the image to RGB.
+            return_tensors (`str` or `TensorType`, *optional*):
+                The type of tensors to return. Can be one of:
+                - Unset: Return a list of `np.ndarray`.
+                - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`.
+                - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`.
+                - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`.
+                - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`.
+            data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`):
+                The channel dimension format for the output image. Can be one of:
+                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                - Unset: Use the channel dimension format of the input image.
+            input_data_format (`ChannelDimension` or `str`, *optional*):
+                The channel dimension format for the input image. If unset, the channel dimension format is inferred
+                from the input image. Can be one of:
+                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
+
+        """
+        do_resize = do_resize if do_resize is not None else self.do_resize
+        size = size if size is not None else self.size
+        resample = resample if resample is not None else self.resample
+        do_rescale = do_rescale if do_rescale is not None else self.do_rescale
+        rescale_factor = (
+            rescale_factor if rescale_factor is not None else self.rescale_factor
+        )
+        do_normalize = do_normalize if do_normalize is not None else self.do_normalize
+        image_mean = image_mean if image_mean is not None else self.image_mean
+        image_std = image_std if image_std is not None else self.image_std
+        do_convert_rgb = (
+            do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb
+        )
+
+        if images is not None:
+            images = make_batched_images(images)
+        if videos is not None:
+            videos = make_batched_videos(videos)
+
+        if images is not None and not valid_images(images):
+            raise ValueError(
+                "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
+                "torch.Tensor, tf.Tensor or jax.ndarray."
+            )
+
+        validate_preprocess_arguments(
+            rescale_factor=rescale_factor,
+            do_normalize=do_normalize,
+            image_mean=image_mean,
+            image_std=image_std,
+            do_resize=do_resize,
+            size=size,
+            resample=resample,
+        )
+
+        if images is not None:
+            pixel_values, vision_grid_thws = [], []
+            for image in images:
+                patches, image_grid_thw = self._preprocess(
+                    image,
+                    do_resize=do_resize,
+                    resample=resample,
+                    do_rescale=do_rescale,
+                    rescale_factor=rescale_factor,
+                    do_normalize=do_normalize,
+                    image_mean=image_mean,
+                    image_std=image_std,
+                    data_format=data_format,
+                    do_convert_rgb=do_convert_rgb,
+                    input_data_format=input_data_format,
+                )
+                pixel_values.extend(patches)
+                vision_grid_thws.append(image_grid_thw)
+            pixel_values = np.array(pixel_values)
+            vision_grid_thws = np.array(vision_grid_thws)
+            data = {"pixel_values": pixel_values, "image_grid_thw": vision_grid_thws}
+
+        if videos is not None:
+            pixel_values, vision_grid_thws = [], []
+            for images in videos:
+                patches, video_grid_thw = self._preprocess(
+                    images,
+                    do_resize=do_resize,
+                    resample=resample,
+                    do_rescale=do_rescale,
+                    rescale_factor=rescale_factor,
+                    do_normalize=do_normalize,
+                    image_mean=image_mean,
+                    image_std=image_std,
+                    data_format=data_format,
+                    do_convert_rgb=do_convert_rgb,
+                    input_data_format=input_data_format,
+                )
+                pixel_values.extend(patches)
+                vision_grid_thws.append(video_grid_thw)
+            pixel_values = np.array(pixel_values)
+            vision_grid_thws = np.array(vision_grid_thws)
+            data = {
+                "pixel_values_videos": pixel_values,
+                "video_grid_thw": vision_grid_thws,
+            }
+
+        return BatchFeature(data=data, tensor_type=return_tensors)

projector.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8ab6a35716b6b7d79a760b5653de4e7c17bd9146784c11fd92bde20d65e72be
+size 51920952

projector_config.json

@@ -0,0 +1,123 @@
+{
+  "model_type": "paddleocr_vl_projector",
+  "architectures": [
+    "PaddleOCRVLProjector"
+  ],
+  "torch_dtype": "bfloat16",
+  "vision_config": {
+    "architectures": [
+      "PaddleOCRVisionModel"
+    ],
+    "attention_dropout": 0.0,
+    "auto_map": {
+      "AutoConfig": "configuration_paddleocr_vl.PaddleOCRVLConfig",
+      "AutoModel": "modeling_paddleocr_vl.PaddleOCRVisionModel"
+    },
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 1152,
+    "image_size": 384,
+    "intermediate_size": 4304,
+    "layer_norm_eps": 1e-06,
+    "model_type": "paddleocr_vl",
+    "num_attention_heads": 16,
+    "num_channels": 3,
+    "num_hidden_layers": 27,
+    "pad_token_id": 0,
+    "patch_size": 14,
+    "spatial_merge_size": 2,
+    "temporal_patch_size": 2,
+    "tokens_per_second": 2,
+    "torch_dtype": "bfloat16"
+  },
+  "text_hidden_size": 1024,
+  "projector": {
+    "merge_kernel_size": [
+      2,
+      2
+    ],
+    "input_hidden_size": 1152,
+    "output_hidden_size": 1024
+  },
+  "required_weight_prefixes": [
+    "projector."
+  ],
+  "source_weight_prefixes": {
+    "projector": "mlp_AR."
+  },
+  "full_model_config": {
+    "architectures": [
+      "PaddleOCRVLForConditionalGeneration"
+    ],
+    "attention_probs_dropout_prob": 0.0,
+    "auto_map": {
+      "AutoConfig": "configuration_paddleocr_vl.PaddleOCRVLConfig",
+      "AutoModel": "modeling_paddleocr_vl.PaddleOCRVLForConditionalGeneration",
+      "AutoModelForCausalLM": "modeling_paddleocr_vl.PaddleOCRVLForConditionalGeneration"
+    },
+    "compression_ratio": 1.0,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_dropout_prob": 0.0,
+    "hidden_size": 1024,
+    "ignored_index": -100,
+    "image_token_id": 100295,
+    "intermediate_size": 3072,
+    "max_position_embeddings": 131072,
+    "max_sequence_length": null,
+    "model_type": "paddleocr_vl",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 18,
+    "num_key_value_heads": 2,
+    "pad_token_id": 0,
+    "rms_norm_eps": 1e-05,
+    "rope_scaling": {
+      "mrope_section": [
+        16,
+        24,
+        24
+      ],
+      "rope_type": "default",
+      "type": "default"
+    },
+    "rope_theta": 500000,
+    "sliding_window": null,
+    "tie_word_embeddings": false,
+    "torch_dtype": "bfloat16",
+    "transformers_version": "4.55.0",
+    "use_bias": false,
+    "use_cache": false,
+    "use_flash_attention": false,
+    "video_token_id": 101307,
+    "vision_config": {
+      "architectures": [
+        "PaddleOCRVisionModel"
+      ],
+      "attention_dropout": 0.0,
+      "auto_map": {
+        "AutoConfig": "configuration_paddleocr_vl.PaddleOCRVLConfig",
+        "AutoModel": "modeling_paddleocr_vl.PaddleOCRVisionModel"
+      },
+      "hidden_act": "gelu_pytorch_tanh",
+      "hidden_size": 1152,
+      "image_size": 384,
+      "intermediate_size": 4304,
+      "layer_norm_eps": 1e-06,
+      "model_type": "paddleocr_vl",
+      "num_attention_heads": 16,
+      "num_channels": 3,
+      "num_hidden_layers": 27,
+      "pad_token_id": 0,
+      "patch_size": 14,
+      "spatial_merge_size": 2,
+      "temporal_patch_size": 2,
+      "tokens_per_second": 2,
+      "torch_dtype": "bfloat16"
+    },
+    "vision_start_token_id": 101305,
+    "vision_end_token_id": 101306,
+    "vocab_size": 103424,
+    "weight_share_add_bias": true,
+    "use_3d_rope": true,
+    "rope_is_neox_style": true
+  }
+}

requirements.txt

@@ -0,0 +1,7 @@
+torch
+transformers
+safetensors
+numpy
+Pillow
+torchvision
+einops

vision_tower.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:560ed1b44203e3bb34023750848033d50a0b73fff8c571ffbcae0b5b18a42e5e
+size 932006944

vision_tower_config.json

@@ -0,0 +1,114 @@
+{
+  "model_type": "paddleocr_vl_vision_tower",
+  "architectures": [
+    "PaddleOCRVLVisionTower"
+  ],
+  "torch_dtype": "bfloat16",
+  "vision_config": {
+    "architectures": [
+      "PaddleOCRVisionModel"
+    ],
+    "attention_dropout": 0.0,
+    "auto_map": {
+      "AutoConfig": "configuration_paddleocr_vl.PaddleOCRVLConfig",
+      "AutoModel": "modeling_paddleocr_vl.PaddleOCRVisionModel"
+    },
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 1152,
+    "image_size": 384,
+    "intermediate_size": 4304,
+    "layer_norm_eps": 1e-06,
+    "model_type": "paddleocr_vl",
+    "num_attention_heads": 16,
+    "num_channels": 3,
+    "num_hidden_layers": 27,
+    "pad_token_id": 0,
+    "patch_size": 14,
+    "spatial_merge_size": 2,
+    "temporal_patch_size": 2,
+    "tokens_per_second": 2,
+    "torch_dtype": "bfloat16"
+  },
+  "required_weight_prefixes": [
+    "visual."
+  ],
+  "source_weight_prefixes": {
+    "visual": "visual."
+  },
+  "full_model_config": {
+    "architectures": [
+      "PaddleOCRVLForConditionalGeneration"
+    ],
+    "attention_probs_dropout_prob": 0.0,
+    "auto_map": {
+      "AutoConfig": "configuration_paddleocr_vl.PaddleOCRVLConfig",
+      "AutoModel": "modeling_paddleocr_vl.PaddleOCRVLForConditionalGeneration",
+      "AutoModelForCausalLM": "modeling_paddleocr_vl.PaddleOCRVLForConditionalGeneration"
+    },
+    "compression_ratio": 1.0,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_dropout_prob": 0.0,
+    "hidden_size": 1024,
+    "ignored_index": -100,
+    "image_token_id": 100295,
+    "intermediate_size": 3072,
+    "max_position_embeddings": 131072,
+    "max_sequence_length": null,
+    "model_type": "paddleocr_vl",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 18,
+    "num_key_value_heads": 2,
+    "pad_token_id": 0,
+    "rms_norm_eps": 1e-05,
+    "rope_scaling": {
+      "mrope_section": [
+        16,
+        24,
+        24
+      ],
+      "rope_type": "default",
+      "type": "default"
+    },
+    "rope_theta": 500000,
+    "sliding_window": null,
+    "tie_word_embeddings": false,
+    "torch_dtype": "bfloat16",
+    "transformers_version": "4.55.0",
+    "use_bias": false,
+    "use_cache": false,
+    "use_flash_attention": false,
+    "video_token_id": 101307,
+    "vision_config": {
+      "architectures": [
+        "PaddleOCRVisionModel"
+      ],
+      "attention_dropout": 0.0,
+      "auto_map": {
+        "AutoConfig": "configuration_paddleocr_vl.PaddleOCRVLConfig",
+        "AutoModel": "modeling_paddleocr_vl.PaddleOCRVisionModel"
+      },
+      "hidden_act": "gelu_pytorch_tanh",
+      "hidden_size": 1152,
+      "image_size": 384,
+      "intermediate_size": 4304,
+      "layer_norm_eps": 1e-06,
+      "model_type": "paddleocr_vl",
+      "num_attention_heads": 16,
+      "num_channels": 3,
+      "num_hidden_layers": 27,
+      "pad_token_id": 0,
+      "patch_size": 14,
+      "spatial_merge_size": 2,
+      "temporal_patch_size": 2,
+      "tokens_per_second": 2,
+      "torch_dtype": "bfloat16"
+    },
+    "vision_start_token_id": 101305,
+    "vision_end_token_id": 101306,
+    "vocab_size": 103424,
+    "weight_share_add_bias": true,
+    "use_3d_rope": true,
+    "rope_is_neox_style": true
+  }
+}