Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,84 @@
+# Capri
+
+Capri is a compact image captioning model designed for high-throughput, plain-language descriptions.
+It supports two inference paths: direct image input or precomputed SigLIP2 pooled embeddings.
+
+The project started from a practical pipeline constraint: existing captioning models were either too slow or too weak for reliable image understanding. Since SigLIP embeddings were already computed upstream, Capri was built to reuse them with a small LLM decoder, combining strong visual representations with fast text generation.
+
+The name comes from the small Italian island of Capri and also hints at the goal of the project: a small CAPtioner with Rapid Inference.
+
+## Model Architecture
+
+- Vision encoder: `google/siglip2-base-patch16-224` (pooled embeddings)
+- Projector: MLP `768 -> 3072 -> 896`
+- Decoder: `Qwen/Qwen2.5-0.5B`
+- Adaptation: LoRA on `q_proj` and `v_proj`
+
+## Load Modes
+
+Embedding-only mode keeps SigLIP out of downloads and VRAM:
+
+```python
+from transformers import AutoModel, AutoProcessor
+import torch
+
+processor = AutoProcessor.from_pretrained("Ligul/capri", trust_remote_code=True)
+model = AutoModel.from_pretrained(
+    "Ligul/capri",
+    trust_remote_code=True,
+    load_vision_tower=False,
+    torch_dtype=torch.bfloat16,
+)
+
+inputs = processor(
+    pooled_embeddings=torch.randn(2, 768),
+    return_tensors="pt",
+)
+captions = model.generate_captions(
+    pooled_embeddings=inputs["pooled_embeddings"],
+    processor=processor,
+    max_new_tokens=32,
+    decode_batch_size=2048,
+)
+```
+
+Image mode loads SigLIP lazily:
+
+```python
+from PIL import Image
+from transformers import AutoModel, AutoProcessor
+import torch
+
+processor = AutoProcessor.from_pretrained("Ligul/capri", trust_remote_code=True)
+model = AutoModel.from_pretrained(
+    "Ligul/capri",
+    trust_remote_code=True,
+    load_vision_tower=True,
+    torch_dtype=torch.bfloat16,
+)
+
+image = Image.open("example.jpg").convert("RGB")
+captions = model.generate_captions(
+    images=[image],
+    processor=processor,
+    max_new_tokens=32,
+    vision_batch_size=64,
+    decode_batch_size=2048,
+)
+```
+
+`generate()` is still available for low-level token generation if you want raw token ids.
+
+## Batch Guidance
+
+Use different knobs for the two stages:
+
+- `vision_batch_size`: moderate, image preprocessing + SigLIP is the expensive vision pass
+- `decode_batch_size`: much larger, pooled embeddings are tiny and Qwen generation batches well
+
+Reasonable defaults:
+
+- `vision_batch_size=64`
+- `decode_batch_size=1024`
+
+On larger GPUs, decode often scales to `2048+`.

__init__.py

@@ -0,0 +1,5 @@
+from .configuration_capri import CapriConfig
+from .modeling_capri import CapriForConditionalGeneration
+from .processing_capri import CapriProcessor
+
+__all__ = ["CapriConfig", "CapriForConditionalGeneration", "CapriProcessor"]

chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

config.json

@@ -0,0 +1,24 @@
+{
+  "architectures": [
+    "CapriForConditionalGeneration"
+  ],
+  "model_type": "capri",
+  "text_model_name_or_path": "Qwen/Qwen2.5-0.5B",
+  "vision_model_name_or_path": "google/siglip2-base-patch16-224",
+  "adapter_subdir": "text_adapter",
+  "projector_type": "mlp",
+  "projector_in_dim": 768,
+  "projector_hidden_dim": 3072,
+  "projector_out_dim": 896,
+  "image_token": "<image>",
+  "image_token_id": 151665,
+  "prompt_prefix": "<image> Caption:",
+  "max_length": 64,
+  "load_vision_tower_by_default": false,
+  "processor_class": "CapriProcessor",
+  "auto_map": {
+    "AutoConfig": "configuration_capri.CapriConfig",
+    "AutoModel": "modeling_capri.CapriForConditionalGeneration",
+    "AutoModelForCausalLM": "modeling_capri.CapriForConditionalGeneration"
+  }
+}

configuration_capri.py

@@ -0,0 +1,37 @@
+from transformers import PretrainedConfig
+
+
+class CapriConfig(PretrainedConfig):
+    model_type = "capri"
+
+    def __init__(
+        self,
+        text_model_name_or_path="Qwen/Qwen2.5-0.5B",
+        vision_model_name_or_path="google/siglip2-base-patch16-224",
+        adapter_subdir="text_adapter",
+        projector_type="mlp",
+        projector_in_dim=768,
+        projector_hidden_dim=3072,
+        projector_out_dim=896,
+        image_token="<image>",
+        image_token_id=151665,
+        prompt_prefix="<image> Caption:",
+        max_length=64,
+        load_vision_tower_by_default=False,
+        processor_class="CapriProcessor",
+        **kwargs,
+    ):
+        self.text_model_name_or_path = text_model_name_or_path
+        self.vision_model_name_or_path = vision_model_name_or_path
+        self.adapter_subdir = adapter_subdir
+        self.projector_type = projector_type
+        self.projector_in_dim = projector_in_dim
+        self.projector_hidden_dim = projector_hidden_dim
+        self.projector_out_dim = projector_out_dim
+        self.image_token = image_token
+        self.image_token_id = image_token_id
+        self.prompt_prefix = prompt_prefix
+        self.max_length = max_length
+        self.load_vision_tower_by_default = load_vision_tower_by_default
+        self.processor_class = processor_class
+        super().__init__(**kwargs)

generation_config.json

@@ -0,0 +1,5 @@
+{
+  "_from_model_config": false,
+  "do_sample": false,
+  "max_new_tokens": 32
+}

modeling_capri.py

@@ -0,0 +1,334 @@
+import os
+from typing import Any
+
+import torch
+import torch.nn as nn
+from peft import PeftModel
+from safetensors.torch import load_file, save_file
+from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizer, PreTrainedModel
+from transformers.utils import cached_file
+
+from .configuration_capri import CapriConfig
+
+
+class MLPProjector(nn.Module):
+    def __init__(self, in_dim: int, hidden_dim: int, out_dim: int):
+        super().__init__()
+        self.net = nn.Sequential(
+            nn.Linear(in_dim, hidden_dim),
+            nn.GELU(),
+            nn.Linear(hidden_dim, out_dim),
+        )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.net(x)
+
+
+class CapriForConditionalGeneration(PreTrainedModel):
+    config_class = CapriConfig
+    base_model_prefix = "capri"
+    main_input_name = "input_ids"
+
+    def __init__(self, config: CapriConfig):
+        super().__init__(config)
+        self.projector = MLPProjector(
+            in_dim=config.projector_in_dim,
+            hidden_dim=config.projector_hidden_dim,
+            out_dim=config.projector_out_dim,
+        )
+        self.text_model = None
+        self.vision_model = None
+        self.tokenizer = None
+        self._repo_id_or_path = None
+        self._hub_kwargs = {}
+        self._text_model_kwargs = {}
+        self._vision_model_kwargs = {}
+        self.post_init()
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path: str, *model_args, config=None, **kwargs):
+        load_vision_tower = kwargs.pop("load_vision_tower", None)
+        if config is None:
+            config, model_kwargs = CapriConfig.from_pretrained(
+                pretrained_model_name_or_path,
+                return_unused_kwargs=True,
+                **kwargs,
+            )
+        else:
+            model_kwargs = dict(kwargs)
+
+        model = cls(config, *model_args)
+        model._repo_id_or_path = pretrained_model_name_or_path
+        model._hub_kwargs = {
+            "cache_dir": model_kwargs.get("cache_dir"),
+            "force_download": model_kwargs.get("force_download"),
+            "local_files_only": model_kwargs.get("local_files_only"),
+            "revision": model_kwargs.get("revision"),
+            "token": model_kwargs.get("token"),
+            "trust_remote_code": model_kwargs.get("trust_remote_code", True),
+        }
+        base_runtime = {
+            "cache_dir": model_kwargs.get("cache_dir"),
+            "force_download": model_kwargs.get("force_download"),
+            "local_files_only": model_kwargs.get("local_files_only"),
+            "revision": model_kwargs.get("revision"),
+            "token": model_kwargs.get("token"),
+            "torch_dtype": model_kwargs.get("torch_dtype", model_kwargs.get("dtype")),
+            "device_map": model_kwargs.get("device_map"),
+            "attn_implementation": model_kwargs.get("attn_implementation"),
+        }
+        model._text_model_kwargs = {k: v for k, v in base_runtime.items() if v is not None}
+        model._vision_model_kwargs = {k: v for k, v in base_runtime.items() if k != "attn_implementation" and v is not None}
+
+        model._load_tokenizer()
+        model._load_text_model()
+        model._load_projector_weights()
+
+        should_load_vision = (
+            config.load_vision_tower_by_default if load_vision_tower is None else load_vision_tower
+        )
+        if should_load_vision:
+            model._load_vision_model()
+
+        model.eval()
+        return model
+
+    def save_pretrained(self, save_directory: str, **kwargs):
+        os.makedirs(save_directory, exist_ok=True)
+        self.config.save_pretrained(save_directory)
+        save_file(
+            self.projector.state_dict(),
+            os.path.join(save_directory, "projector.safetensors"),
+        )
+        if self.text_model is not None:
+            self.text_model.save_pretrained(
+                os.path.join(save_directory, self.config.adapter_subdir)
+            )
+        if self.tokenizer is not None:
+            self.tokenizer.save_pretrained(save_directory)
+
+    def _resolve_repo_file(self, filename: str, subfolder: str | None = None) -> str:
+        if os.path.isdir(self._repo_id_or_path):
+            parts = [self._repo_id_or_path]
+            if subfolder:
+                parts.append(subfolder)
+            parts.append(filename)
+            return os.path.join(*parts)
+        return cached_file(self._repo_id_or_path, filename, subfolder=subfolder, **self._hub_kwargs)
+
+    def _load_tokenizer(self):
+        if self.tokenizer is not None:
+            return
+        
+        if self.config.image_token_id is None or self.config.image_token is None:
+            raise ValueError("`image_token_id` and `image_token` must be set in the config.")
+        
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            self._repo_id_or_path,
+            **self._hub_kwargs,
+        )
+        if self.tokenizer.pad_token is None:
+            self.tokenizer.pad_token = self.tokenizer.eos_token
+
+    def _load_text_model(self):
+        if self.text_model is not None:
+            return
+        base_model = AutoModelForCausalLM.from_pretrained(
+            self.config.text_model_name_or_path,
+            **self._text_model_kwargs,
+        )
+        self.text_model = PeftModel.from_pretrained(
+            base_model,
+            self._repo_id_or_path,
+            subfolder=self.config.adapter_subdir,
+            is_trainable=False,
+            **self._hub_kwargs,
+        )
+        self.text_model.eval()
+
+    def _load_vision_model(self):
+        if self.vision_model is not None:
+            return
+        model = AutoModel.from_pretrained(
+            self.config.vision_model_name_or_path,
+            **self._vision_model_kwargs,
+        )
+        self.vision_model = getattr(model, "vision_model", model)
+        self.vision_model.eval()
+
+    def _load_projector_weights(self):
+        projector_path = self._resolve_repo_file("projector.safetensors")
+        state_dict = load_file(projector_path)
+        self.projector.load_state_dict(state_dict)
+
+        embed_weight = self.text_model.get_input_embeddings().weight
+        self.projector.to(device=embed_weight.device, dtype=embed_weight.dtype)
+
+    @property
+    def vision_loaded(self) -> bool:
+        return self.vision_model is not None
+
+    @staticmethod
+    def _module_device_dtype(module: nn.Module) -> tuple[torch.device, torch.dtype]:
+        param = next(module.parameters())
+        return param.device, param.dtype
+
+    @staticmethod
+    def _chunk_list(items: list[Any], chunk_size: int) -> list[list[Any]]:
+        return [items[i : i + chunk_size] for i in range(0, len(items), chunk_size)]
+
+    def encode_images(self, pixel_values: torch.Tensor) -> torch.Tensor:
+        self._load_vision_model()
+        vision_device, vision_dtype = self._module_device_dtype(self.vision_model)
+        pixel_values = pixel_values.to(device=vision_device, dtype=vision_dtype)
+        outputs = self.vision_model(pixel_values=pixel_values)
+        pooled = getattr(outputs, "pooler_output", None)
+        if pooled is None:
+            last_hidden = getattr(outputs, "last_hidden_state", None)
+            if last_hidden is None:
+                raise ValueError("Vision model did not return pooler_output or last_hidden_state.")
+            pooled = last_hidden[:, 0]
+        return pooled
+
+    def _prompt_inputs(self, batch_size: int, device: torch.device) -> tuple[torch.Tensor, torch.Tensor]:
+        encoded = self.tokenizer(
+            [self.config.prompt_prefix] * batch_size,
+            add_special_tokens=False,
+            return_tensors="pt",
+            padding=True,
+        )
+        return encoded["input_ids"].to(device), encoded["attention_mask"].to(device)
+
+    def _prepare_inputs(
+        self,
+        *,
+        input_ids: torch.Tensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        pooled_embeddings: torch.Tensor | None = None,
+        pixel_values: torch.Tensor | None = None,
+    ) -> tuple[torch.Tensor, torch.Tensor]:
+        if pooled_embeddings is None:
+            if pixel_values is None:
+                raise ValueError("Provide either `pooled_embeddings` or `pixel_values`.")
+            pooled_embeddings = self.encode_images(pixel_values)
+
+        if pooled_embeddings.ndim == 1:
+            pooled_embeddings = pooled_embeddings.unsqueeze(0)
+
+        target_device = self.text_model.get_input_embeddings().weight.device
+        if input_ids is None:
+            input_ids, attention_mask = self._prompt_inputs(pooled_embeddings.size(0), target_device)
+        else:
+            input_ids = input_ids.to(target_device)
+            if attention_mask is None:
+                attention_mask = torch.ones_like(input_ids, device=target_device)
+            else:
+                attention_mask = attention_mask.to(target_device)
+
+        inputs_embeds = self.text_model.get_input_embeddings()(input_ids)
+        pooled_embeddings = pooled_embeddings.to(device=inputs_embeds.device, dtype=inputs_embeds.dtype)
+        projected = self.projector(pooled_embeddings)
+
+        image_mask = input_ids.eq(self.config.image_token_id)
+        image_count = image_mask.sum(dim=1)
+        if not torch.all(image_count == 1):
+            raise ValueError("Each sample must contain exactly one `<image>` token.")
+
+        token_positions = image_mask.float().argmax(dim=1)
+        batch_positions = torch.arange(input_ids.size(0), device=input_ids.device)
+        inputs_embeds[batch_positions, token_positions] = projected
+        return inputs_embeds, attention_mask
+
+    def forward(
+        self,
+        input_ids: torch.Tensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        pooled_embeddings: torch.Tensor | None = None,
+        pixel_values: torch.Tensor | None = None,
+        labels: torch.Tensor | None = None,
+        **kwargs: Any,
+    ):
+        if input_ids is None and labels is not None:
+            raise ValueError("`input_ids` are required when passing `labels`.")
+
+        inputs_embeds, attention_mask = self._prepare_inputs(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            pooled_embeddings=pooled_embeddings,
+            pixel_values=pixel_values,
+        )
+        return self.text_model(
+            inputs_embeds=inputs_embeds,
+            attention_mask=attention_mask,
+            labels=labels,
+            **kwargs,
+        )
+
+    @torch.no_grad()
+    def generate(
+        self,
+        input_ids: torch.Tensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        pooled_embeddings: torch.Tensor | None = None,
+        pixel_values: torch.Tensor | None = None,
+        **generate_kwargs: Any,
+    ) -> torch.Tensor:
+        inputs_embeds, attention_mask = self._prepare_inputs(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            pooled_embeddings=pooled_embeddings,
+            pixel_values=pixel_values,
+        )
+
+        generate_kwargs.setdefault("do_sample", False)
+        generate_kwargs.setdefault("eos_token_id", self.tokenizer.eos_token_id)
+        generate_kwargs.setdefault("pad_token_id", self.tokenizer.pad_token_id)
+        return self.text_model.generate(
+            inputs_embeds=inputs_embeds,
+            attention_mask=attention_mask,
+            **generate_kwargs,
+        )
+
+    @torch.no_grad()
+    def generate_captions(
+        self,
+        *,
+        images: Any = None,
+        pooled_embeddings: Any = None,
+        processor=None,
+        vision_batch_size: int = 64,
+        decode_batch_size: int = 1024,
+        **generate_kwargs: Any,
+    ) -> list[str]:
+        if processor is None:
+            raise ValueError("`processor` is required for `generate_captions()`.")
+        if images is None and pooled_embeddings is None:
+            raise ValueError("Provide either `images` or `pooled_embeddings`.")
+        if images is not None and pooled_embeddings is not None:
+            raise ValueError("Provide only one of `images` or `pooled_embeddings`.")
+        if vision_batch_size <= 0 or decode_batch_size <= 0:
+            raise ValueError("Batch sizes must be positive integers.")
+
+        if images is not None:
+            image_items = processor.normalize_images(images)
+            all_pooled = []
+            for image_chunk in self._chunk_list(image_items, vision_batch_size):
+                image_inputs = processor(images=image_chunk, return_tensors="pt")
+                pooled_chunk = self.encode_images(image_inputs["pixel_values"]).detach().cpu()
+                all_pooled.append(pooled_chunk)
+            pooled_embeddings = torch.cat(all_pooled, dim=0)
+        else:
+            pooled_embeddings = processor.normalize_pooled_embeddings(pooled_embeddings).detach().cpu()
+
+        captions = []
+        total = pooled_embeddings.shape[0]
+        for start in range(0, total, decode_batch_size):
+            pooled_chunk = pooled_embeddings[start : start + decode_batch_size]
+            model_inputs = dict(processor(
+                pooled_embeddings=pooled_chunk,
+                return_tensors="pt",
+            ))
+            sequences = self.generate(**model_inputs, **generate_kwargs)
+            captions.extend(processor.batch_decode(sequences, skip_special_tokens=True))
+
+        return captions

preprocessor_config.json

@@ -0,0 +1,23 @@
+{
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "SiglipImageProcessor",
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "processor_class": "CapriProcessor",
+  "resample": 2,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 224,
+    "width": 224
+  }
+}

processing_capri.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+from typing import Any
+
+import torch
+from transformers import AutoConfig
+from transformers.feature_extraction_utils import BatchFeature
+from transformers.image_processing_utils import BaseImageProcessor
+from transformers.processing_utils import ProcessorMixin
+from transformers.tokenization_utils_base import PreTrainedTokenizerBase
+
+
+class CapriProcessor(ProcessorMixin):
+    attributes = ["image_processor", "tokenizer"]
+    image_processor_class = "SiglipImageProcessor"
+    tokenizer_class = ("Qwen2Tokenizer", "Qwen2TokenizerFast")
+
+    def __init__(
+        self,
+        image_processor: BaseImageProcessor,
+        tokenizer: PreTrainedTokenizerBase,
+        prompt_prefix: str = "<image> Caption:",
+        image_token: str = "<image>",
+        pooled_embedding_dim: int = 768,
+    ):
+        self.prompt_prefix = prompt_prefix
+        self.image_token = image_token
+        self.pooled_embedding_dim = pooled_embedding_dim
+        super().__init__(image_processor, tokenizer)
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
+        kwargs.setdefault("use_fast", False)
+        processor = super().from_pretrained(pretrained_model_name_or_path, **kwargs)
+        config = AutoConfig.from_pretrained(
+            pretrained_model_name_or_path,
+            trust_remote_code=kwargs.get("trust_remote_code", False),
+        )
+        processor.prompt_prefix = getattr(config, "prompt_prefix", processor.prompt_prefix)
+        processor.image_token = getattr(config, "image_token", processor.image_token)
+        processor.pooled_embedding_dim = getattr(config, "projector_in_dim", processor.pooled_embedding_dim)
+        return processor
+
+    def normalize_images(self, images) -> list[Any]:
+        if isinstance(images, torch.Tensor):
+            if images.ndim == 4:
+                return [images[i] for i in range(images.shape[0])]
+            return [images]
+        if isinstance(images, (list, tuple)):
+            return list(images)
+        return [images]
+
+    def normalize_pooled_embeddings(self, pooled_embeddings) -> torch.Tensor:
+        pooled = torch.as_tensor(pooled_embeddings)
+        if pooled.ndim == 1:
+            pooled = pooled.unsqueeze(0)
+        if pooled.ndim != 2:
+            raise ValueError("`pooled_embeddings` must be a 1D embedding or a 2D batch of embeddings.")
+        if pooled.shape[-1] != self.pooled_embedding_dim:
+            raise ValueError(
+                f"Expected pooled embedding dim {self.pooled_embedding_dim}, got {pooled.shape[-1]}."
+            )
+        return pooled
+
+    def __call__(
+        self,
+        images=None,
+        pooled_embeddings=None,
+        text=None,
+        return_tensors: str | None = "pt",
+        padding: bool | str = True,
+        truncation: bool = False,
+        max_length: int | None = None,
+        **kwargs: Any,
+    ) -> BatchFeature:
+        if images is None and pooled_embeddings is None and text is None:
+            raise ValueError("Provide `images`, `pooled_embeddings`, or `text`.")
+
+        batch = {}
+        batch_size = None
+
+        if images is not None:
+            image_features = self.image_processor(images=images, return_tensors=return_tensors, **kwargs)
+            batch.update(dict(image_features))
+            batch_size = batch["pixel_values"].shape[0]
+
+        if pooled_embeddings is not None:
+            pooled = self.normalize_pooled_embeddings(pooled_embeddings)
+            batch["pooled_embeddings"] = pooled
+            batch_size = pooled.shape[0]
+
+        if text is None and batch_size is not None:
+            text = [self.prompt_prefix] * batch_size
+
+        if text is not None:
+            if isinstance(text, str):
+                text = [text]
+            tokenized = self.tokenizer(
+                text,
+                add_special_tokens=False,
+                padding=padding,
+                truncation=truncation,
+                max_length=max_length,
+                return_tensors=return_tensors,
+            )
+            batch.update(dict(tokenized))
+
+        return BatchFeature(data=batch, tensor_type=return_tensors)
+
+    def batch_decode(self, *args, **kwargs):
+        return self.tokenizer.batch_decode(*args, **kwargs)
+
+    def decode(self, *args, **kwargs):
+        return self.tokenizer.decode(*args, **kwargs)

processor_config.json

@@ -0,0 +1,9 @@
+{
+  "processor_class": "CapriProcessor",
+  "prompt_prefix": "<image> Caption:",
+  "image_token": "<image>",
+  "pooled_embedding_dim": 768,
+  "auto_map": {
+    "AutoProcessor": "processing_capri.CapriProcessor"
+  }
+}

projector.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eef2cd9c916ace570cad3ffc03f8ba0b2ae7a13d79c3cd26b555b53008645415
+size 20463424

text_adapter/README.md

@@ -0,0 +1,207 @@
+---
+base_model: Qwen/Qwen2.5-0.5B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-0.5B
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

text_adapter/adapter_config.json

@@ -0,0 +1,41 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-0.5B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

text_adapter/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:895e9cc5ceaa3b35e74bf764b286f848ee31900859e2b849994f3168a39ebc8e
+size 4337968

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d5cbd0458fdb84541ddde44bf8ad64825062559d7e0ff402d601905ec49941c
+size 11422076

tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<image>"
+  ],
+  "is_local": true,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}