Add tiny random LFM3-VL model for optimum-intel tests

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

chat_template.jinja

@@ -0,0 +1,229 @@
+{{- bos_token -}}
+{%- set preserve_thinking = preserve_thinking | default(false) -%}
+{%- set enable_thinking = enable_thinking | default(true) -%}
+
+{%- macro render_value(v) -%}
+    {%- if v is string -%}
+        {{- v -}}
+    {%- elif v is boolean -%}
+        {{- "true" if v else "false" -}}
+    {%- elif v is none -%}
+        {{- "null" -}}
+    {%- elif v is mapping or (v is sequence and v is not string) -%}
+        {{- v | tojson -}}
+    {%- else -%}
+        {{- v -}}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro parse_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- elif content is mapping and content.get("type") == "image" -%}
+        {{- "<|vision_start|><|image|><|vision_end|>" -}}
+    {%- elif content is mapping and content.get("type") == "video" -%}
+        {{- "<|vision_start|><|video|><|vision_end|>" -}}
+    {%- elif content is mapping and content.get("type") == "text" -%}
+        {{- (content.get("text") or "") | string -}}
+    {%- elif content is mapping -%}
+        {{- content | tojson -}}
+    {%- elif content is iterable -%}
+        {%- set _ns = namespace(result="") -%}
+        {%- for item in content -%}
+            {%- if item is string -%}
+                {%- set _ns.result = _ns.result + item -%}
+            {%- elif item is mapping and item.get("type") == "image" -%}
+                {%- set _ns.result = _ns.result + "<|vision_start|><|image|><|vision_end|>" -%}
+            {%- elif item is mapping and item.get("type") == "video" -%}
+                {%- set _ns.result = _ns.result + "<|vision_start|><|video|><|vision_end|>" -%}
+            {%- elif item is mapping and item.get("type") == "text" -%}
+                {%- set _ns.result = _ns.result + ((item.get("text") or "") | string) -%}
+            {%- else -%}
+                {%- set _ns.result = _ns.result + (item | tojson) -%}
+            {%- endif -%}
+        {%- endfor -%}
+        {{- _ns.result -}}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro render_tool_calls(tool_calls) -%}
+    {%- for tool_call in tool_calls -%}
+        {%- if not loop.first -%}{{- "\n\n" -}}{%- endif -%}
+        {%- set func = tool_call["function"] if "function" in tool_call else tool_call -%}
+        {%- set func_args = func.get("arguments") -%}
+        {%- if func_args is string and (func_args | trim) not in ["", "{}", "null"] -%}
+            {{- raise_exception("Tool call arguments must be a mapping, got a JSON-encoded string: parse arguments with json.loads() before applying the chat template") -}}
+        {%- endif -%}
+        {{- "<tool_call>\n<function=" + func["name"] + ">\n" -}}
+        {%- if func_args is mapping -%}
+            {%- for arg_name, arg_value in func_args.items() -%}
+                {{- "<parameter=" + arg_name + ">\n" + render_value(arg_value) + "\n</parameter>\n" -}}
+            {%- endfor -%}
+        {%- endif -%}
+        {{- "</function>\n</tool_call>" -}}
+    {%- endfor -%}
+{%- endmacro -%}
+
+{%- macro tool_to_json(tool) -%}
+    {%- set fn = tool.function if tool.function is defined else tool -%}
+    {%- set ns_t = namespace(first=true) -%}
+    {{- "{" -}}
+    {%- for k, v in fn.items() -%}
+        {%- if k != "defer_loading" -%}
+            {%- if not ns_t.first -%}{{- ", " -}}{%- endif -%}
+            {%- set ns_t.first = false -%}
+            {{- k | tojson -}}{{- ": " -}}{{- v | tojson -}}
+        {%- endif -%}
+    {%- endfor -%}
+    {{- "}" -}}
+{%- endmacro -%}
+
+{%- macro render_tool_response(content) -%}
+    {#- content that already carries the tag (pre-formatted by the caller) is
+        passed through as-is instead of being wrapped a second time -#}
+    {%- if content is string and "<tool_response>" in content -%}
+        {{- content -}}
+    {%- else -%}
+    {{- "<tool_response>\n" -}}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- elif content is mapping -%}
+        {{- parse_content(content) -}}
+    {%- elif content is iterable -%}
+        {%- for item in content -%}
+            {%- if not loop.first -%}{{- "\n" -}}{%- endif -%}
+            {%- if item is mapping and item.result is defined and item.result is mapping and item.result.tools is defined -%}
+                {{- "<additional_tools>\n" -}}
+                {%- set _tl = item.result.tools -%}
+                {%- if _tl is string -%}
+                    {{- _tl -}}
+                {%- elif _tl is mapping -%}
+                    {{- tool_to_json(_tl) -}}
+                {%- else -%}
+                    {%- for t in _tl -%}
+                        {%- if not loop.first -%}{{- "\n" -}}{%- endif -%}
+                        {{- tool_to_json(t) -}}
+                    {%- endfor -%}
+                {%- endif -%}
+                {{- "\n</additional_tools>" -}}
+            {%- else -%}
+                {{- parse_content(item) -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+    {{- "\n</tool_response>" -}}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- set ns = namespace(system_prompt="", last_user_index=-1) -%}
+{%- if messages and messages[0]["role"] == "system" -%}
+    {%- if messages[0].get("content") -%}
+        {%- set ns.system_prompt = parse_content(messages[0]["content"]) -%}
+    {%- endif -%}
+    {%- set messages = messages[1:] -%}
+{%- endif -%}
+{%- if tools -%}
+    {%- set tools_ns = namespace(lines=[]) -%}
+    {%- for tool in tools -%}
+        {%- set fn = tool.function if tool.function is defined else tool -%}
+        {%- set _dyn = (fn.defer_loading is defined and fn.defer_loading) or (tool.defer_loading is defined and tool.defer_loading) -%}
+        {%- if not _dyn -%}
+            {%- set tools_ns.lines = tools_ns.lines + [tool_to_json(fn)] -%}
+        {%- endif -%}
+    {%- endfor -%}
+    {%- if tools_ns.lines -%}
+        {%- set ns.system_prompt = ns.system_prompt + ("\n\n" if ns.system_prompt else "") + "<tools>\n" + (tools_ns.lines | join("\n")) + "\n</tools>" -%}
+    {%- endif -%}
+{%- endif -%}
+{%- if ns.system_prompt -%}
+    {{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}}
+{%- endif -%}
+{%- for message in messages -%}
+    {%- if message["role"] == "user" -%}
+        {%- set ns.last_user_index = loop.index0 -%}
+    {%- endif -%}
+{%- endfor -%}
+{%- for message in messages -%}
+    {{- "<|im_start|>" + message.role + "\n" -}}
+    {%- if message.role == "assistant" -%}
+        {%- generation -%}
+        {%- set thinking = message.thinking if message.thinking is defined else message.reasoning if message.reasoning is defined else message.reasoning_content -%}
+        {%- set keep_thinking = preserve_thinking or loop.index0 > ns.last_user_index -%}
+        {%- if thinking is defined and keep_thinking -%}
+            {{- "<think>" + (thinking or "") + "</think>" -}}
+        {%- endif -%}
+        {%- set _cfm_tag = "CONTINUE_FINAL_MESSAGE_TAG " -%}
+        {%- set _has_cfm = false -%}
+        {%- set content = "" -%}
+        {%- if message.get("content") -%}
+            {%- set content = parse_content(message.content) -%}
+        {%- endif -%}
+        {%- if not keep_thinking -%}
+            {%- if "</think>" in content -%}
+                {%- set content = content.split("</think>")[-1] | trim -%}
+            {%- endif -%}
+            {%- set content = "<think></think>" + content -%}
+        {%- endif -%}
+        {%- if content.endswith(_cfm_tag) -%}
+            {%- set _has_cfm = true -%}
+            {%- set _trunc_len = (content | length) - (_cfm_tag | length) -%}
+            {%- set content = content[:_trunc_len] -%}
+        {%- endif -%}
+        {{- content -}}
+        {%- if message.tool_calls -%}
+            {{- render_tool_calls(message.tool_calls) -}}
+        {%- endif -%}
+        {%- if _has_cfm -%}
+            {{- _cfm_tag -}}
+        {%- endif -%}
+        {{- "<|im_end|>\n" -}}
+        {%- endgeneration -%}
+    {%- elif message.role == "tool" -%}
+        {%- if message.get("additional_tools") -%}
+            {%- set _at = message["additional_tools"] -%}
+            {#- definitions may live in additional_tools itself, or in content
+                with additional_tools used as a plain truthy flag -#}
+            {%- if _at is mapping or (_at is sequence and _at is not string) -%}
+                {%- set _src = _at -%}
+            {%- else -%}
+                {%- set _src = message.get("content") -%}
+            {%- endif -%}
+            {#- nothing to reveal: skip the wrapper entirely rather than
+                emitting an empty <additional_tools> block -#}
+            {%- if _src is string and "<tool_response>" in _src -%}
+                {{- _src -}}
+            {%- elif _src is string and "<additional_tools>" in _src -%}
+                {{- "<tool_response>\n" + _src + "\n</tool_response>" -}}
+            {%- elif _src -%}
+                {{- "<tool_response>\n<additional_tools>\n" -}}
+                {%- if _src is string -%}
+                    {{- _src -}}
+                {%- elif _src is mapping -%}
+                    {{- tool_to_json(_src) -}}
+                {%- else -%}
+                    {%- for t in _src -%}
+                        {%- if not loop.first -%}{{- "\n" -}}{%- endif -%}
+                        {{- tool_to_json(t) -}}
+                    {%- endfor -%}
+                {%- endif -%}
+                {{- "\n</additional_tools>\n</tool_response>" -}}
+            {%- endif -%}
+        {%- elif message.get("content") -%}
+            {{- render_tool_response(message["content"]) -}}
+        {%- endif -%}
+        {{- "<|im_end|>\n" -}}
+    {%- else %}
+        {%- if message.get("content") -%}
+            {{- parse_content(message["content"]) -}}
+        {%- endif -%}
+        {{- "<|im_end|>\n" -}}
+    {%- endif %}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{- "<|im_start|>assistant\n" -}}
+    {%- if enable_thinking -%}
+        {{- "<think>" -}}
+    {%- else -%}
+        {{- "<think></think>" -}}
+    {%- endif -%}
+{%- endif -%}

config.json

@@ -0,0 +1,68 @@
+{
+  "architectures": [
+    "Lfm3VlForConditionalGeneration"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_lfm3_vl.Lfm3VlConfig",
+    "AutoModelForCausalLM": "modeling_lfm3_vl.Lfm3VlForConditionalGeneration",
+    "AutoModelForImageTextToText": "modeling_lfm3_vl.Lfm3VlForConditionalGeneration"
+  },
+  "bos_token_id": 124894,
+  "conv_L_cache": 7,
+  "conv_bias": false,
+  "downsample_factor": 2,
+  "dtype": "float32",
+  "eos_token_id": 124900,
+  "full_attention_rope_theta": 1000000.0,
+  "head_dim": 8,
+  "hidden_size": 16,
+  "image_token_id": 125030,
+  "intermediate_size": 32,
+  "layer_types": [
+    "conv",
+    "conv",
+    "full_attention",
+    "conv",
+    "sliding_attention",
+    "conv",
+    "sliding_attention"
+  ],
+  "max_position_embeddings": 128000,
+  "model_type": "lfm3_vl",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 7,
+  "num_key_value_heads": 2,
+  "pad_token_id": 124893,
+  "partial_rotary_factor": 0.5,
+  "projector_hidden_size": 32,
+  "projector_layer_norm_eps": 1e-06,
+  "rms_norm_eps": 1e-05,
+  "sliding_attention_rope_theta": 10000,
+  "sliding_window": 4,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.20.0",
+  "use_cache": true,
+  "use_sandwich_norm": true,
+  "video_token_id": 125029,
+  "vision_end_token_id": 125010,
+  "vision_hidden_size": 16,
+  "vision_intermediate_size": 32,
+  "vision_layer_types": [
+    "conv",
+    "full_attention",
+    "conv",
+    "full_attention"
+  ],
+  "vision_max_position_embeddings": 2304,
+  "vision_num_attention_heads": 2,
+  "vision_num_channels": 3,
+  "vision_num_hidden_layers": 4,
+  "vision_num_key_value_heads": 1,
+  "vision_patch_dim": 1536,
+  "vision_patch_size": 16,
+  "vision_rope_theta": 1000000.0,
+  "vision_spatial_merge_size": 2,
+  "vision_start_token_id": 125009,
+  "vision_temporal_patch_size": 2,
+  "vocab_size": 128000
+}

configuration_lfm3_vl.py

@@ -0,0 +1,149 @@
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+# Licensed under the Apache License, Version 2.0
+
+from transformers.configuration_utils import PreTrainedConfig
+
+
+class Lfm3VlConfig(PreTrainedConfig):
+    model_type = "lfm3_vl"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size=128000,
+        hidden_size=2048,
+        intermediate_size=7680,
+        num_hidden_layers=24,
+        num_attention_heads=16,
+        num_key_value_heads=4,
+        head_dim=None,
+        layer_types=None,
+        conv_L_cache=7,
+        sliding_window=256,
+        full_attention_rope_theta=1_000_000.0,
+        sliding_attention_rope_theta=10_000.0,
+        partial_rotary_factor=0.5,
+        max_position_embeddings=128000,
+        vision_patch_size=16,
+        vision_temporal_patch_size=2,
+        vision_spatial_merge_size=2,
+        vision_num_channels=3,
+        vision_patch_dim=1536,
+        vision_hidden_size=1024,
+        vision_intermediate_size=2560,
+        vision_num_hidden_layers=14,
+        vision_num_attention_heads=16,
+        vision_num_key_value_heads=8,
+        vision_layer_types=None,
+        vision_rope_theta=1_000_000.0,
+        vision_max_position_embeddings=2304,
+        projector_hidden_size=4096,
+        projector_layer_norm_eps=1e-6,
+        downsample_factor=2,
+        image_token_id=125030,
+        video_token_id=125029,
+        vision_start_token_id=125009,
+        vision_end_token_id=125010,
+        rms_norm_eps=1e-5,
+        use_cache=True,
+        pad_token_id=124893,
+        bos_token_id=124894,
+        eos_token_id=124900,
+        tie_word_embeddings=True,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_key_value_heads = num_key_value_heads
+        self.head_dim = head_dim
+        self.layer_types = layer_types or [
+            "conv",
+            "conv",
+            "full_attention",
+            "conv",
+            "sliding_attention",
+            "conv",
+            "full_attention",
+            "conv",
+            "sliding_attention",
+            "conv",
+            "full_attention",
+            "conv",
+            "sliding_attention",
+            "conv",
+            "full_attention",
+            "conv",
+            "sliding_attention",
+            "conv",
+            "full_attention",
+            "conv",
+            "sliding_attention",
+            "conv",
+            "full_attention",
+            "conv",
+        ]
+        self.conv_L_cache = conv_L_cache
+        self.sliding_window = sliding_window
+        self.full_attention_rope_theta = full_attention_rope_theta
+        self.sliding_attention_rope_theta = sliding_attention_rope_theta
+        self.partial_rotary_factor = partial_rotary_factor
+        self.max_position_embeddings = max_position_embeddings
+        self.vision_patch_size = vision_patch_size
+        self.vision_temporal_patch_size = vision_temporal_patch_size
+        self.vision_spatial_merge_size = vision_spatial_merge_size
+        self.vision_num_channels = vision_num_channels
+        self.vision_patch_dim = vision_patch_dim
+        self.vision_hidden_size = vision_hidden_size
+        self.vision_intermediate_size = vision_intermediate_size
+        self.vision_num_hidden_layers = vision_num_hidden_layers
+        self.vision_num_attention_heads = vision_num_attention_heads
+        self.vision_num_key_value_heads = vision_num_key_value_heads
+        self.vision_layer_types = vision_layer_types or [
+            "conv",
+            "conv",
+            "full_attention",
+            "conv",
+            "full_attention",
+            "conv",
+            "full_attention",
+            "conv",
+            "full_attention",
+            "conv",
+            "full_attention",
+            "conv",
+            "full_attention",
+            "conv",
+        ]
+        self.vision_rope_theta = vision_rope_theta
+        self.vision_max_position_embeddings = vision_max_position_embeddings
+        self.projector_hidden_size = projector_hidden_size
+        self.projector_layer_norm_eps = projector_layer_norm_eps
+        self.downsample_factor = downsample_factor
+        self.image_token_id = image_token_id
+        self.video_token_id = video_token_id
+        self.vision_start_token_id = vision_start_token_id
+        self.vision_end_token_id = vision_end_token_id
+        self.rms_norm_eps = rms_norm_eps
+        self.use_cache = use_cache
+        if len(self.layer_types) != self.num_hidden_layers:
+            raise ValueError("layer_types must contain one entry per language layer")
+        if len(self.vision_layer_types) != self.vision_num_hidden_layers:
+            raise ValueError("vision_layer_types must contain one entry per vision layer")
+        expected_patch_dim = vision_temporal_patch_size * vision_num_channels * vision_patch_size**2
+        if vision_patch_dim != expected_patch_dim:
+            raise ValueError(f"vision_patch_dim must be {expected_patch_dim}, got {vision_patch_dim}")
+        if downsample_factor != vision_spatial_merge_size:
+            raise ValueError("downsample_factor and vision_spatial_merge_size must match")
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
+
+
+__all__ = ["Lfm3VlConfig"]

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 124894,
+  "eos_token_id": 124900,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 124893,
+  "transformers_version": "5.20.0",
+  "use_cache": true
+}

image_processing_lfm3_vl.py

@@ -0,0 +1,17 @@
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+# Licensed under the Apache License, Version 2.0
+"""Qwen-style dynamic-resolution image preprocessing for LFM3-VL."""
+
+from transformers.models.qwen2_vl.image_processing_qwen2_vl import Qwen2VLImageProcessor
+
+
+class Lfm3VlImageProcessor(Qwen2VLImageProcessor):
+    patch_size = 16
+    temporal_patch_size = 2
+    merge_size = 2
+    size = {"shortest_edge": 65_536, "longest_edge": 16_777_216}
+    image_mean = [0.5, 0.5, 0.5]
+    image_std = [0.5, 0.5, 0.5]
+
+
+__all__ = ["Lfm3VlImageProcessor"]

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b203ba09e586a19c594bb6d20f71f07763ef847c59ff6301969240533261c11
+size 8436616

modeling_lfm3_vl.py

@@ -0,0 +1,499 @@
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+# Licensed under the Apache License, Version 2.0
+"""Simple PyTorch LFM3-VL with a variable-shape Liquid vision encoder."""
+
+from dataclasses import dataclass
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+from transformers.generation import GenerationMixin
+from transformers.modeling_outputs import CausalLMOutputWithPast
+from transformers.modeling_utils import PreTrainedModel
+from .configuration_lfm3_vl import Lfm3VlConfig
+
+
+class Lfm3VlRMSNorm(nn.Module):
+    def __init__(self, hidden_size, eps):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.eps = eps
+
+    def forward(self, hidden_states):
+        dtype = hidden_states.dtype
+        normalized = hidden_states.float()
+        normalized = normalized * torch.rsqrt(normalized.square().mean(-1, keepdim=True) + self.eps)
+        return (normalized.to(dtype) * self.weight).to(dtype)
+
+
+def _rope_1d(hidden_states, position_ids, theta, rotary_factor):
+    head_dim = hidden_states.shape[-1]
+    pairs = int(head_dim * rotary_factor) // 2
+    frequencies = theta ** (-(2.0 * torch.arange(pairs, device=hidden_states.device, dtype=torch.float32) / head_dim))
+    angles = position_ids.float()[..., None] * frequencies
+    cos = angles.cos().to(hidden_states.dtype)[:, None, :, :]
+    sin = angles.sin().to(hidden_states.dtype)[:, None, :, :]
+    first = hidden_states[..., :pairs]
+    second = hidden_states[..., head_dim // 2 : head_dim // 2 + pairs]
+    output = hidden_states.clone()
+    output[..., :pairs] = first * cos - second * sin
+    output[..., head_dim // 2 : head_dim // 2 + pairs] = first * sin + second * cos
+    return output
+
+
+def _rope_2d(hidden_states, height, width, theta):
+    head_dim = hidden_states.shape[-1]
+    quarter = head_dim // 4
+    frequencies = theta ** (
+        -(2.0 * torch.arange(quarter, device=hidden_states.device, dtype=torch.float32) / (head_dim // 2))
+    )
+    rows = torch.arange(height, device=hidden_states.device).repeat_interleave(width)
+    cols = torch.arange(width, device=hidden_states.device).repeat(height)
+    angles = torch.cat((rows[:, None] * frequencies, cols[:, None] * frequencies), dim=-1)
+    cos = angles.cos().to(hidden_states.dtype)[:, None, :]
+    sin = angles.sin().to(hidden_states.dtype)[:, None, :]
+    first, second = hidden_states[..., : head_dim // 2], hidden_states[..., head_dim // 2 :]
+    return torch.cat((first * cos - second * sin, first * sin + second * cos), dim=-1)
+
+
+class Lfm3VlCache:
+    def __init__(self, config, batch_size, dtype, device):
+        self.key_cache = [None] * config.num_hidden_layers
+        self.value_cache = [None] * config.num_hidden_layers
+        self.conv_cache = [
+            torch.zeros(batch_size, config.hidden_size, config.conv_L_cache - 1, dtype=dtype, device=device)
+            for _ in range(config.num_hidden_layers)
+        ]
+        self.layer_types = config.layer_types
+        self.is_decode = False
+
+    def get_seq_length(self):
+        for layer_type, key in zip(self.layer_types, self.key_cache):
+            if layer_type != "conv" and key is not None:
+                return key.shape[-2]
+        return 0
+
+    def reorder_cache(self, beam_idx):
+        for index in range(len(self.layer_types)):
+            if self.key_cache[index] is not None:
+                self.key_cache[index] = self.key_cache[index].index_select(0, beam_idx)
+                self.value_cache[index] = self.value_cache[index].index_select(0, beam_idx)
+            self.conv_cache[index] = self.conv_cache[index].index_select(0, beam_idx)
+
+
+class Lfm3VlMLP(nn.Module):
+    def __init__(self, hidden_size, intermediate_size):
+        super().__init__()
+        self.gate_up = nn.Linear(hidden_size, 2 * intermediate_size, bias=False)
+        self.w2 = nn.Linear(intermediate_size, hidden_size, bias=False)
+
+    def forward(self, hidden_states):
+        gate, up = self.gate_up(hidden_states).chunk(2, dim=-1)
+        return self.w2(F.silu(gate) * up)
+
+
+class Lfm3VlAttention(nn.Module):
+    def __init__(self, config, layer_idx, layer_type):
+        super().__init__()
+        self.layer_idx = layer_idx
+        self.layer_type = layer_type
+        self.num_heads = config.num_attention_heads
+        self.num_key_value_heads = config.num_key_value_heads
+        self.head_dim = config.head_dim or config.hidden_size // config.num_attention_heads
+        qkv_size = (self.num_heads + 2 * self.num_key_value_heads) * self.head_dim
+        self.qkv = nn.Linear(config.hidden_size, qkv_size, bias=False)
+        self.gate_proj = nn.Linear(config.hidden_size, self.num_heads, bias=False)
+        self.o_proj = nn.Linear(self.num_heads * self.head_dim, config.hidden_size, bias=False)
+        self.q_norm = Lfm3VlRMSNorm(self.head_dim, config.rms_norm_eps)
+        self.k_norm = Lfm3VlRMSNorm(self.head_dim, config.rms_norm_eps)
+        self.theta = (
+            config.full_attention_rope_theta if layer_type == "full_attention" else config.sliding_attention_rope_theta
+        )
+        self.rotary_factor = config.partial_rotary_factor if layer_type == "full_attention" else 1.0
+        self.sliding_window = config.sliding_window
+
+    def forward(self, hidden_states, position_ids, attention_mask, past_key_values):
+        batch_size, query_length, _ = hidden_states.shape
+        q_size = self.num_heads * self.head_dim
+        kv_size = self.num_key_value_heads * self.head_dim
+        q, k, v = self.qkv(hidden_states).split((q_size, kv_size, kv_size), dim=-1)
+        q = self.q_norm(q.view(batch_size, query_length, self.num_heads, self.head_dim)).transpose(1, 2)
+        k = self.k_norm(k.view(batch_size, query_length, self.num_key_value_heads, self.head_dim)).transpose(1, 2)
+        v = v.view(batch_size, query_length, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        q = _rope_1d(q, position_ids, self.theta, self.rotary_factor)
+        k = _rope_1d(k, position_ids, self.theta, self.rotary_factor)
+        if past_key_values is not None:
+            old_k = past_key_values.key_cache[self.layer_idx]
+            old_v = past_key_values.value_cache[self.layer_idx]
+            if old_k is not None:
+                k = torch.cat((old_k, k), dim=-2)
+                v = torch.cat((old_v, v), dim=-2)
+            past_key_values.key_cache[self.layer_idx] = k
+            past_key_values.value_cache[self.layer_idx] = v
+        groups = self.num_heads // self.num_key_value_heads
+        k = k.repeat_interleave(groups, dim=1)
+        v = v.repeat_interleave(groups, dim=1)
+        key_length = k.shape[-2]
+        query_positions = torch.arange(key_length - query_length, key_length, device=hidden_states.device)
+        key_positions = torch.arange(key_length, device=hidden_states.device)
+        allowed = key_positions[None, :] <= query_positions[:, None]
+        if self.layer_type == "sliding_attention":
+            allowed &= key_positions[None, :] > query_positions[:, None] - self.sliding_window
+        if attention_mask is not None:
+            allowed = allowed[None, None] & attention_mask[:, None, None, :key_length].bool()
+        else:
+            allowed = allowed[None, None]
+        additive_mask = torch.zeros(allowed.shape, dtype=q.dtype, device=q.device)
+        additive_mask.masked_fill_(~allowed, torch.finfo(q.dtype).min)
+        output = F.scaled_dot_product_attention(q, k, v, attn_mask=additive_mask)
+        gate = torch.sigmoid(self.gate_proj(hidden_states)).transpose(1, 2).unsqueeze(-1)
+        output = (output * gate).transpose(1, 2).reshape(batch_size, query_length, -1)
+        return self.o_proj(output)
+
+
+class Lfm3VlShortConv(nn.Module):
+    def __init__(self, config, layer_idx):
+        super().__init__()
+        self.layer_idx = layer_idx
+        self.kernel_size = config.conv_L_cache
+        self.in_proj = nn.Linear(config.hidden_size, 3 * config.hidden_size, bias=False)
+        self.conv = nn.Parameter(torch.empty(config.hidden_size, 1, self.kernel_size))
+        self.out_proj = nn.Linear(config.hidden_size, config.hidden_size, bias=False)
+
+    def forward(self, hidden_states, attention_mask, past_key_values):
+        first, middle, third = self.in_proj(hidden_states).chunk(3, dim=-1)
+        gated = first * third
+        if attention_mask is not None:
+            gated = gated * attention_mask[:, -hidden_states.shape[1] :, None]
+        gated = gated.transpose(1, 2)
+        if past_key_values is not None and past_key_values.is_decode:
+            state = past_key_values.conv_cache[self.layer_idx]
+            full = torch.cat((state, gated), dim=-1)
+            output = (full * self.conv[:, 0, :][None]).sum(-1, keepdim=True)
+            past_key_values.conv_cache[self.layer_idx] = full[..., -self.kernel_size + 1 :]
+        else:
+            output = F.conv1d(gated, self.conv, padding=self.kernel_size - 1, groups=gated.shape[1])
+            output = output[..., : hidden_states.shape[1]]
+            if past_key_values is not None:
+                past_key_values.conv_cache[self.layer_idx] = F.pad(
+                    gated, (max(0, self.kernel_size - 1 - gated.shape[-1]), 0)
+                )[..., -self.kernel_size + 1 :]
+        return self.out_proj((middle.transpose(1, 2) * output).transpose(1, 2))
+
+
+class Lfm3VlLanguageLayer(nn.Module):
+    def __init__(self, config, layer_idx, layer_type):
+        super().__init__()
+        self.layer_type = layer_type
+        self.operator_norm = Lfm3VlRMSNorm(config.hidden_size, config.rms_norm_eps)
+        self.post_operator_norm = Lfm3VlRMSNorm(config.hidden_size, config.rms_norm_eps)
+        self.ffn_norm = Lfm3VlRMSNorm(config.hidden_size, config.rms_norm_eps)
+        self.post_ffn_norm = Lfm3VlRMSNorm(config.hidden_size, config.rms_norm_eps)
+        if layer_type == "conv":
+            self.conv = Lfm3VlShortConv(config, layer_idx)
+        else:
+            self.self_attn = Lfm3VlAttention(config, layer_idx, layer_type)
+        self.feed_forward = Lfm3VlMLP(config.hidden_size, config.intermediate_size)
+
+    def forward(self, hidden_states, position_ids, attention_mask, past_key_values):
+        operator_input = self.operator_norm(hidden_states)
+        if self.layer_type == "conv":
+            operator_output = self.conv(operator_input, attention_mask, past_key_values)
+        else:
+            operator_output = self.self_attn(operator_input, position_ids, attention_mask, past_key_values)
+        hidden_states = hidden_states + self.post_operator_norm(operator_output)
+        return hidden_states + self.post_ffn_norm(self.feed_forward(self.ffn_norm(hidden_states)))
+
+
+class Lfm3VlBackbone(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id)
+        self.layers = nn.ModuleList(
+            [Lfm3VlLanguageLayer(config, index, layer_type) for index, layer_type in enumerate(config.layer_types)]
+        )
+        self.embedding_norm = Lfm3VlRMSNorm(config.hidden_size, config.rms_norm_eps)
+
+    def forward(self, hidden_states, attention_mask=None, position_ids=None, past_key_values=None):
+        if past_key_values is not None:
+            past_key_values.is_decode = past_key_values.get_seq_length() > 0
+        if position_ids is None:
+            past_length = past_key_values.get_seq_length() if past_key_values is not None else 0
+            position_ids = (
+                torch.arange(past_length, past_length + hidden_states.shape[1], device=hidden_states.device)
+                .unsqueeze(0)
+                .expand(hidden_states.shape[0], -1)
+            )
+        for layer in self.layers:
+            hidden_states = layer(hidden_states, position_ids, attention_mask, past_key_values)
+        return self.embedding_norm(hidden_states)
+
+
+class Lfm3VlVisionAttention(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.num_heads = config.vision_num_attention_heads
+        self.num_key_value_heads = config.vision_num_key_value_heads
+        self.head_dim = config.vision_hidden_size // self.num_heads
+        self.theta = config.vision_rope_theta
+        self.q_proj = nn.Linear(config.vision_hidden_size, self.num_heads * self.head_dim, bias=False)
+        self.k_proj = nn.Linear(config.vision_hidden_size, self.num_key_value_heads * self.head_dim, bias=False)
+        self.v_proj = nn.Linear(config.vision_hidden_size, self.num_key_value_heads * self.head_dim, bias=False)
+        self.out_proj = nn.Linear(config.vision_hidden_size, config.vision_hidden_size, bias=False)
+        self.q_layernorm = Lfm3VlRMSNorm(self.head_dim, config.rms_norm_eps)
+        self.k_layernorm = Lfm3VlRMSNorm(self.head_dim, config.rms_norm_eps)
+
+    def forward(self, hidden_states, height, width):
+        tokens = hidden_states.shape[0]
+        q = self.q_layernorm(self.q_proj(hidden_states).view(tokens, self.num_heads, self.head_dim))
+        k = self.k_layernorm(self.k_proj(hidden_states).view(tokens, self.num_key_value_heads, self.head_dim))
+        v = self.v_proj(hidden_states).view(tokens, self.num_key_value_heads, self.head_dim)
+        q = _rope_2d(q, height, width, self.theta)
+        k = _rope_2d(k, height, width, self.theta)
+        groups = self.num_heads // self.num_key_value_heads
+        output = F.scaled_dot_product_attention(
+            q.transpose(0, 1).unsqueeze(0),
+            k.repeat_interleave(groups, dim=1).transpose(0, 1).unsqueeze(0),
+            v.repeat_interleave(groups, dim=1).transpose(0, 1).unsqueeze(0),
+        )
+        return self.out_proj(output.squeeze(0).transpose(0, 1).reshape(tokens, -1))
+
+
+class Lfm3VlVisionConv2d(nn.Module):
+    def __init__(self, hidden_size):
+        super().__init__()
+        self.in_proj = nn.Linear(hidden_size, 3 * hidden_size, bias=False)
+        self.spatial_conv = nn.Parameter(torch.empty(hidden_size, 1, 3, 3))
+        self.out_proj = nn.Linear(hidden_size, hidden_size, bias=False)
+
+    def forward(self, hidden_states, height, width):
+        first, middle, third = self.in_proj(hidden_states).chunk(3, dim=-1)
+        gated = (first * third).transpose(0, 1).reshape(1, -1, height, width)
+        output = F.conv2d(gated, self.spatial_conv, padding=1, groups=gated.shape[1])
+        return self.out_proj(middle * output.reshape(-1, height * width).transpose(0, 1))
+
+
+class Lfm3VlVisionLayer(nn.Module):
+    def __init__(self, config, layer_type):
+        super().__init__()
+        self.layer_type = layer_type
+        self.operator_norm = Lfm3VlRMSNorm(config.vision_hidden_size, config.rms_norm_eps)
+        self.ffn_norm = Lfm3VlRMSNorm(config.vision_hidden_size, config.rms_norm_eps)
+        if layer_type == "conv":
+            self.conv = Lfm3VlVisionConv2d(config.vision_hidden_size)
+        else:
+            self.self_attn = Lfm3VlVisionAttention(config)
+        self.feed_forward = Lfm3VlMLP(config.vision_hidden_size, config.vision_intermediate_size)
+
+    def forward(self, hidden_states, height, width):
+        operator_input = self.operator_norm(hidden_states)
+        operator_output = (
+            self.conv(operator_input, height, width)
+            if self.layer_type == "conv"
+            else self.self_attn(operator_input, height, width)
+        )
+        hidden_states = hidden_states + operator_output
+        return hidden_states + self.feed_forward(self.ffn_norm(hidden_states))
+
+
+class Lfm3VlVisionEncoder(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.spatial_merge_size = config.vision_spatial_merge_size
+        self.patchifier = nn.Linear(config.vision_patch_dim, config.vision_hidden_size, bias=False)
+        self.embedding_norm = Lfm3VlRMSNorm(config.vision_hidden_size, config.rms_norm_eps)
+        self.layers = nn.ModuleList(
+            [Lfm3VlVisionLayer(config, layer_type) for layer_type in config.vision_layer_types]
+        )
+
+    @staticmethod
+    def _block_major_to_raster(hidden_states, temporal, height, width, merge_size):
+        return (
+            hidden_states.view(temporal, height // merge_size, width // merge_size, merge_size, merge_size, -1)
+            .permute(0, 1, 3, 2, 4, 5)
+            .reshape(temporal, height, width, -1)
+        )
+
+    def forward(self, pixel_values, grid_thw):
+        """Encode Qwen-preprocessed patches, one independent spatial sequence per temporal patch."""
+        outputs = []
+        offset = 0
+        for temporal, height, width in grid_thw.tolist():
+            temporal, height, width = int(temporal), int(height), int(width)
+            tokens = temporal * height * width
+            hidden_states = pixel_values[offset : offset + tokens]
+            offset += tokens
+            hidden_states = self._block_major_to_raster(
+                hidden_states, temporal, height, width, self.spatial_merge_size
+            )
+            for temporal_hidden_states in hidden_states:
+                temporal_hidden_states = self.embedding_norm(
+                    self.patchifier(temporal_hidden_states.reshape(-1, hidden_states.shape[-1]))
+                )
+                for layer in self.layers:
+                    temporal_hidden_states = layer(temporal_hidden_states, height, width)
+                outputs.append((temporal_hidden_states, height, width))
+        if offset != pixel_values.shape[0]:
+            raise ValueError("pixel_values and grid_thw disagree")
+        return outputs
+
+
+class Lfm3VlProjector(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.spatial_merge_size = config.vision_spatial_merge_size
+        self.layer_norm = nn.LayerNorm(config.vision_hidden_size, eps=config.projector_layer_norm_eps)
+        self.linear_1 = nn.Linear(
+            self.spatial_merge_size**2 * config.vision_hidden_size,
+            config.projector_hidden_size,
+            bias=True,
+        )
+        self.linear_2 = nn.Linear(config.projector_hidden_size, config.hidden_size, bias=True)
+
+    def forward(self, hidden_states, height, width):
+        if height % self.spatial_merge_size or width % self.spatial_merge_size:
+            raise ValueError("projector requires even patch-grid dimensions")
+        hidden_states = self.layer_norm(hidden_states).view(height, width, -1)
+        merge = self.spatial_merge_size
+        hidden_states = hidden_states.view(height // merge, merge, width // merge, merge, -1)
+        hidden_states = hidden_states.permute(0, 2, 1, 3, 4).reshape(-1, merge**2 * hidden_states.shape[-1])
+        return self.linear_2(F.gelu(self.linear_1(hidden_states), approximate="none"))
+
+
+@dataclass
+class Lfm3VlCausalLMOutputWithPast(CausalLMOutputWithPast):
+    image_hidden_states: torch.FloatTensor | None = None
+    video_hidden_states: torch.FloatTensor | None = None
+
+
+class Lfm3VlPreTrainedModel(PreTrainedModel):
+    config_class = Lfm3VlConfig
+    base_model_prefix = ""
+    input_modalities = ("image", "video", "text")
+    _no_split_modules = ["Lfm3VlLanguageLayer", "Lfm3VlVisionLayer"]
+
+
+class Lfm3VlForConditionalGeneration(Lfm3VlPreTrainedModel, GenerationMixin):
+    def __init__(self, config):
+        super().__init__(config)
+        self.language_model = Lfm3VlBackbone(config)
+        self.vision_encoder = Lfm3VlVisionEncoder(config)
+        self.vision_projector = Lfm3VlProjector(config)
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.language_model.embed_tokens
+
+    @classmethod
+    def _supports_default_dynamic_cache(cls):
+        return False
+
+    def set_input_embeddings(self, value):
+        self.language_model.embed_tokens = value
+
+    def get_output_embeddings(self):
+        return self.language_model.embed_tokens
+
+    def get_visual_features(self, pixel_values, grid_thw):
+        target_dtype = self.vision_encoder.patchifier.weight.dtype
+        encoded = self.vision_encoder(pixel_values.to(dtype=target_dtype), grid_thw)
+        return [self.vision_projector(hidden_states, height, width) for hidden_states, height, width in encoded]
+
+    def get_image_features(self, pixel_values, image_grid_thw):
+        return self.get_visual_features(pixel_values, image_grid_thw)
+
+    def get_video_features(self, pixel_values_videos, video_grid_thw):
+        return self.get_visual_features(pixel_values_videos, video_grid_thw)
+
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        position_ids=None,
+        pixel_values=None,
+        image_grid_thw=None,
+        pixel_values_videos=None,
+        video_grid_thw=None,
+        past_key_values=None,
+        inputs_embeds=None,
+        labels=None,
+        use_cache=None,
+        logits_to_keep=0,
+        **kwargs,
+    ):
+        del kwargs
+        if inputs_embeds is None:
+            inputs_embeds = self.language_model.embed_tokens(input_ids)
+        image_hidden_states = None
+        if pixel_values is not None:
+            if image_grid_thw is None:
+                raise ValueError("image_grid_thw is required with pixel_values")
+            image_features = self.get_image_features(pixel_values, image_grid_thw)
+            image_hidden_states = torch.cat(image_features, dim=0).to(inputs_embeds.dtype)
+            image_mask = input_ids == self.config.image_token_id
+            if int(image_mask.sum()) != image_hidden_states.shape[0]:
+                raise ValueError("image token count does not match projected image features")
+            inputs_embeds = inputs_embeds.clone()
+            inputs_embeds[image_mask] = image_hidden_states
+        video_hidden_states = None
+        if pixel_values_videos is not None:
+            if video_grid_thw is None:
+                raise ValueError("video_grid_thw is required with pixel_values_videos")
+            video_features = self.get_video_features(pixel_values_videos, video_grid_thw)
+            video_hidden_states = torch.cat(video_features, dim=0).to(inputs_embeds.dtype)
+            video_mask = input_ids == self.config.video_token_id
+            if int(video_mask.sum()) != video_hidden_states.shape[0]:
+                raise ValueError("video token count does not match projected video features")
+            if image_hidden_states is None:
+                inputs_embeds = inputs_embeds.clone()
+            inputs_embeds[video_mask] = video_hidden_states
+        use_cache = self.config.use_cache if use_cache is None else use_cache
+        if use_cache and past_key_values is None:
+            past_key_values = Lfm3VlCache(
+                self.config, inputs_embeds.shape[0], inputs_embeds.dtype, inputs_embeds.device
+            )
+        hidden_states = self.language_model(inputs_embeds, attention_mask, position_ids, past_key_values)
+        hidden_for_logits = (
+            hidden_states[:, -logits_to_keep:] if isinstance(logits_to_keep, int) and logits_to_keep else hidden_states
+        )
+        logits = F.linear(hidden_for_logits, self.language_model.embed_tokens.weight)
+        loss = None
+        if labels is not None:
+            loss = F.cross_entropy(
+                logits[:, :-1].reshape(-1, logits.shape[-1]), labels[:, 1:].reshape(-1), ignore_index=-100
+            )
+        return Lfm3VlCausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=past_key_values,
+            image_hidden_states=image_hidden_states,
+            video_hidden_states=video_hidden_states,
+        )
+
+    def prepare_inputs_for_generation(
+        self, input_ids, past_key_values=None, attention_mask=None, position_ids=None, **kwargs
+    ):
+        if past_key_values is not None and past_key_values.get_seq_length() > 0:
+            input_ids = input_ids[:, -1:]
+            if position_ids is not None:
+                position_ids = position_ids[:, -1:]
+            kwargs["pixel_values"] = None
+            kwargs["image_grid_thw"] = None
+            kwargs["pixel_values_videos"] = None
+            kwargs["video_grid_thw"] = None
+        return {
+            "input_ids": input_ids,
+            "past_key_values": past_key_values,
+            "attention_mask": attention_mask,
+            "position_ids": position_ids,
+            "use_cache": kwargs.get("use_cache", True),
+            "pixel_values": kwargs.get("pixel_values"),
+            "image_grid_thw": kwargs.get("image_grid_thw"),
+            "pixel_values_videos": kwargs.get("pixel_values_videos"),
+            "video_grid_thw": kwargs.get("video_grid_thw"),
+        }
+
+
+__all__ = ["Lfm3VlForConditionalGeneration", "Lfm3VlPreTrainedModel"]

processing_lfm3_vl.py

@@ -0,0 +1,154 @@
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+# Licensed under the Apache License, Version 2.0
+"""Processor for LFM3-VL using Qwen visual preprocessing and LFM3 text tokenization."""
+
+import numpy as np
+
+from transformers.feature_extraction_utils import BatchFeature
+from transformers.image_utils import ImageInput
+from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack
+from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
+from transformers.utils import logging
+from transformers.video_utils import VideoInput
+
+
+logger = logging.get_logger(__name__)
+
+IMAGE_TOKEN = "<|image|>"
+VIDEO_TOKEN = "<|video|>"
+VISION_START_TOKEN = "<|vision_start|>"
+VISION_END_TOKEN = "<|vision_end|>"
+IMAGE_TOKEN_ID = 125030
+VIDEO_TOKEN_ID = 125029
+VISION_START_TOKEN_ID = 125009
+VISION_END_TOKEN_ID = 125010
+
+
+class Lfm3VlProcessorKwargs(ProcessingKwargs, total=False):
+    _defaults = {
+        "text_kwargs": {
+            "padding": False,
+            "return_token_type_ids": False,
+            "return_mm_token_type_ids": False,
+        },
+        "videos_kwargs": {"return_metadata": True},
+    }
+
+
+class Lfm3VlProcessor(ProcessorMixin):
+    """Expand Qwen-style visual spans around Qwen-preprocessed visual inputs."""
+
+    def __init__(self, image_processor=None, tokenizer=None, video_processor=None, chat_template=None, **kwargs):
+        self.image_token = IMAGE_TOKEN
+        self.video_token = VIDEO_TOKEN
+        self.vision_start_token = VISION_START_TOKEN
+        self.vision_end_token = VISION_END_TOKEN
+        self.image_token_id = tokenizer.convert_tokens_to_ids(self.image_token)
+        self.video_token_id = tokenizer.convert_tokens_to_ids(self.video_token)
+        self.vision_start_token_id = tokenizer.convert_tokens_to_ids(self.vision_start_token)
+        self.vision_end_token_id = tokenizer.convert_tokens_to_ids(self.vision_end_token)
+        actual_token_ids = (
+            self.image_token_id,
+            self.video_token_id,
+            self.vision_start_token_id,
+            self.vision_end_token_id,
+        )
+        expected_token_ids = (IMAGE_TOKEN_ID, VIDEO_TOKEN_ID, VISION_START_TOKEN_ID, VISION_END_TOKEN_ID)
+        if actual_token_ids != expected_token_ids:
+            raise ValueError(
+                "LFM3-VL requires the released multimodal tokenizer with "
+                f"{IMAGE_TOKEN}={IMAGE_TOKEN_ID}, {VIDEO_TOKEN}={VIDEO_TOKEN_ID}, "
+                f"{VISION_START_TOKEN}={VISION_START_TOKEN_ID}, and {VISION_END_TOKEN}={VISION_END_TOKEN_ID}; "
+                f"got {actual_token_ids}."
+            )
+        super().__init__(image_processor, tokenizer, video_processor, chat_template=chat_template, **kwargs)
+
+    def __call__(
+        self,
+        images: ImageInput = None,
+        text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] = None,
+        videos: VideoInput = None,
+        **kwargs: Unpack[Lfm3VlProcessorKwargs],
+    ) -> BatchFeature:
+        output_kwargs = self._merge_kwargs(
+            Lfm3VlProcessorKwargs,
+            tokenizer_init_kwargs=self.tokenizer.init_kwargs,
+            **kwargs,
+        )
+        image_inputs = (
+            self.image_processor(images=images, **output_kwargs["images_kwargs"]) if images is not None else {}
+        )
+        video_inputs = (
+            self.video_processor(videos=videos, **output_kwargs["videos_kwargs"]) if videos is not None else {}
+        )
+        image_grid_thw = image_inputs.get("image_grid_thw")
+        video_grid_thw = video_inputs.get("video_grid_thw")
+        video_metadata = video_inputs.get("video_metadata")
+
+        if not isinstance(text, list):
+            text = [text]
+        text = text.copy()
+        if image_grid_thw is not None:
+            image_index = 0
+            for sample_index in range(len(text)):
+                while self.image_token in text[sample_index]:
+                    token_count = int(image_grid_thw[image_index].prod()) // self.image_processor.merge_size**2
+                    text[sample_index] = text[sample_index].replace(
+                        self.image_token, "<|placeholder|>" * token_count, 1
+                    )
+                    image_index += 1
+                text[sample_index] = text[sample_index].replace("<|placeholder|>", self.image_token)
+
+        if video_grid_thw is not None:
+            video_index = 0
+            for sample_index in range(len(text)):
+                while self.video_token in text[sample_index]:
+                    metadata = video_metadata[video_index]
+                    if metadata.fps is None:
+                        logger.warning_once("Video FPS is missing; defaulting to 24 FPS for timestamps.")
+                        metadata.fps = 24
+                    timestamps = self._calculate_timestamps(
+                        metadata.frames_indices,
+                        metadata.fps,
+                        self.video_processor.temporal_patch_size,
+                    )
+                    frame_tokens = int(video_grid_thw[video_index][1:].prod()) // self.video_processor.merge_size**2
+                    placeholder = "".join(
+                        f"<{timestamp:.1f} seconds>"
+                        + self.vision_start_token
+                        + "<|placeholder|>" * frame_tokens
+                        + self.vision_end_token
+                        for timestamp in timestamps
+                    )
+                    text[sample_index] = text[sample_index].replace(self.video_token, placeholder, 1)
+                    video_index += 1
+                text[sample_index] = text[sample_index].replace("<|placeholder|>", self.video_token)
+
+        return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)
+        return_mm_token_type_ids = output_kwargs["text_kwargs"].pop("return_mm_token_type_ids", False)
+        text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
+        self._check_special_mm_tokens(text, text_inputs, modalities=["image", "video"])
+        if return_mm_token_type_ids:
+            input_ids = np.asarray(text_inputs["input_ids"])
+            mm_token_type_ids = np.zeros_like(input_ids)
+            mm_token_type_ids[input_ids == self.image_token_id] = 1
+            mm_token_type_ids[input_ids == self.video_token_id] = 2
+            text_inputs["mm_token_type_ids"] = mm_token_type_ids.tolist()
+        video_inputs.pop("video_metadata", None)
+        return BatchFeature(
+            data={**text_inputs, **image_inputs, **video_inputs},
+            tensor_type=return_tensors,
+        )
+
+    @staticmethod
+    def _calculate_timestamps(indices, video_fps, temporal_patch_size=2):
+        indices = indices.tolist() if not isinstance(indices, list) else indices.copy()
+        if len(indices) % temporal_patch_size:
+            indices.extend(indices[-1] for _ in range(temporal_patch_size - len(indices) % temporal_patch_size))
+        return [
+            (indices[index] + indices[index + temporal_patch_size - 1]) / (2 * video_fps)
+            for index in range(0, len(indices), temporal_patch_size)
+        ]
+
+
+__all__ = ["Lfm3VlProcessor"]

processor_config.json

@@ -0,0 +1,72 @@
+{
+  "auto_map": {
+    "AutoProcessor": "processing_lfm3_vl.Lfm3VlProcessor"
+  },
+  "image_processor": {
+    "auto_map": {
+      "AutoImageProcessor": "image_processing_lfm3_vl.Lfm3VlImageProcessor",
+      "AutoProcessor": "processing_lfm3_vl.Lfm3VlProcessor"
+    },
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_processor_type": "Lfm3VlImageProcessor",
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "merge_size": 2,
+    "patch_size": 16,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "size": {
+      "longest_edge": 16777216,
+      "shortest_edge": 65536
+    },
+    "temporal_patch_size": 2
+  },
+  "processor_class": "Lfm3VlProcessor",
+  "video_processor": {
+    "auto_map": {
+      "AutoProcessor": "processing_lfm3_vl.Lfm3VlProcessor",
+      "AutoVideoProcessor": "video_processing_lfm3_vl.Lfm3VlVideoProcessor"
+    },
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "do_sample_frames": true,
+    "fps": 2.0,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "max_frames": 768,
+    "max_video_tokens": 768,
+    "merge_size": 2,
+    "min_frames": 4,
+    "patch_size": 16,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "return_metadata": false,
+    "size": {
+      "longest_edge": 25165824,
+      "shortest_edge": 4096
+    },
+    "temporal_patch_size": 2,
+    "video_processor_type": "Lfm3VlVideoProcessor"
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a39eed58008cbcbc09e5b629b81338a654269c64cd13348aaa6090f4db75c7e7
+size 17908258

tokenizer_config.json

@@ -0,0 +1,25 @@
+{
+  "auto_map": {
+    "AutoProcessor": "processing_lfm3_vl.Lfm3VlProcessor"
+  },
+  "backend": "tokenizers",
+  "bos_token": "<|startoftext|>",
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": [],
+  "image_token": "<|image|>",
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "model_specific_special_tokens": {
+    "image_token": "<|image|>",
+    "video_token": "<|video|>",
+    "vision_end_token": "<|vision_end|>",
+    "vision_start_token": "<|vision_start|>"
+  },
+  "pad_token": "<|pad|>",
+  "processor_class": "Lfm3VlProcessor",
+  "tokenizer_class": "TokenizersBackend",
+  "video_token": "<|video|>",
+  "vision_end_token": "<|vision_end|>",
+  "vision_start_token": "<|vision_start|>"
+}

video_processing_lfm3_vl.py

@@ -0,0 +1,20 @@
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+# Licensed under the Apache License, Version 2.0
+"""Qwen-style video preprocessing for LFM3-VL."""
+
+from transformers.models.qwen3_vl.video_processing_qwen3_vl import Qwen3VLVideoProcessor
+
+
+class Lfm3VlVideoProcessor(Qwen3VLVideoProcessor):
+    patch_size = 16
+    temporal_patch_size = 2
+    merge_size = 2
+    size = {"shortest_edge": 4_096, "longest_edge": 25_165_824}
+    fps = 2.0
+    min_frames = 4
+    max_frames = 768
+    image_mean = [0.5, 0.5, 0.5]
+    image_std = [0.5, 0.5, 0.5]
+
+
+__all__ = ["Lfm3VlVideoProcessor"]