Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

.meta/create_model.py

@@ -0,0 +1,80 @@
+def create_model(source_model_id: str, source_revision: str, save_folder: str) -> str:
+    import json
+    from pathlib import Path
+
+    import torch
+    from huggingface_hub import hf_hub_download
+    from transformers import AutoConfig, AutoModelForImageTextToText, set_seed
+
+    folder = Path(save_folder)
+    folder.mkdir(parents=True, exist_ok=True)
+    # LFM3-VL is not registered in the installed Transformers release; ship its pinned remote code (text backbone,
+    # vision tower and the image/video processors) so the checkpoint loads without a newer package.
+    for name in (
+        "configuration_lfm3.py",
+        "configuration_lfm3_vl.py",
+        "modeling_lfm3.py",
+        "modeling_lfm3_vl.py",
+        "lfm3_conv.py",
+        "processing_lfm3_vl.py",
+        "image_processing_lfm3_vl.py",
+        "video_processing_lfm3_vl.py",
+        "chat_template.jinja",
+        "generation_config.json",
+        "processor_config.json",
+        "tokenizer.json",
+        "tokenizer_config.json",
+        "LICENSE",
+    ):
+        (folder / name).write_bytes(Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_bytes())
+
+    config = json.loads(Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text())
+    # Text backbone: residual/MLP widths 64/128 like tiny-random/lfm3, with at least four KV heads for TP=4 and
+    # the source 4:1 Q/KV ratio (16 Q heads, 4 KV heads, rectangular projections at head dim 32), the two-conv
+    # prefix plus two complete full/conv/sliding/conv cycles.
+    text_config = config["text_config"]
+    text_config.update(
+        hidden_size=64,
+        intermediate_size=128,
+        num_hidden_layers=10,
+        num_attention_heads=16,
+        num_key_value_heads=4,
+        head_dim=32,
+        max_position_embeddings=128000,
+    )
+    text_config["layer_types"] = text_config["layer_types"][:10]
+    # Vision tower: at least four KV heads with the source 2:1 Q/KV ratio (8 Q heads, 4 KV heads). Its head dim is
+    # derived as hidden // num_heads, so width 256 keeps head dim 32 and axial RoPE; patch 16 / temporal patch 2 /
+    # spatial merge 2. Six layers retain the conv/conv prefix and two full/conv cycles.
+    vision_config = config["vision_config"]
+    vision_config.update(
+        hidden_size=256,
+        intermediate_size=640,
+        num_hidden_layers=6,
+        num_attention_heads=8,
+        num_key_value_heads=4,
+    )
+    vision_config["layer_types"] = ["conv", "conv", "full_attention", "conv", "full_attention", "conv"]
+    # Projector: LayerNorm + GELU MLP merging 2x2 vision blocks; keep bias and the downsample contract.
+    config["projector_hidden_size"] = 128
+    (folder / "config.json").write_text(json.dumps(config, indent=2) + "\n")
+
+    set_seed(42)
+    model = AutoModelForImageTextToText.from_config(
+        AutoConfig.from_pretrained(folder, trust_remote_code=True), trust_remote_code=True, dtype=torch.bfloat16
+    )
+    model.save_pretrained(folder)
+    # `save_pretrained()` rewrites these two files with this environment's versions; restore the pinned source bytes
+    # plus the deliberate config edits so only the intended fields differ from the source.
+    (folder / "config.json").write_text(json.dumps(config, indent=2) + "\n")
+    (folder / "generation_config.json").write_bytes(
+        Path(hf_hub_download(source_model_id, "generation_config.json", revision=source_revision)).read_bytes()
+    )
+    return str(model)
+
+
+create_model(
+    source_model_id="LiquidAI/LFM3-VL-2B",
+    source_revision="ba44bd51db20a2753f28c273a1571ccaa210d954",
+    save_folder="/tmp/lfm3-vl",
+)

.meta/env.json

@@ -0,0 +1,43 @@
+{
+  "source_model": {
+    "name": "LiquidAI/LFM3-VL-2B",
+    "commit_id": "ba44bd51db20a2753f28c273a1571ccaa210d954"
+  },
+  "environments": {
+    "creation": {
+      "python": "3.12.13",
+      "packages": {
+        "huggingface_hub": "2.3.0",
+        "numpy": "2.5.2",
+        "pillow": "12.3.0",
+        "safetensors": "0.8.0",
+        "tokenizers": "0.23.3",
+        "torch": "2.13.0+cu126",
+        "torchvision": "0.28.0+cu126",
+        "transformers": "5.22.0.dev0"
+      },
+      "cuda": "12.6",
+      "hardware": {
+        "gpu_model": "NVIDIA H20"
+      }
+    },
+    "transformers": {
+      "python": "3.12.13",
+      "packages": {
+        "huggingface_hub": "2.3.0",
+        "numpy": "2.5.2",
+        "pillow": "12.3.0",
+        "safetensors": "0.8.0",
+        "tokenizers": "0.23.3",
+        "torch": "2.13.0+cu126",
+        "torchvision": "0.28.0+cu126",
+        "transformers": "5.22.0.dev0",
+        "triton": "3.7.1"
+      },
+      "cuda": "12.6",
+      "hardware": {
+        "gpu_model": "NVIDIA H20"
+      }
+    }
+  }
+}

.meta/model.log

@@ -0,0 +1,261 @@
+Lfm3VlForConditionalGeneration(
+  (model): Lfm3VlModel(
+    (vision_tower): Lfm3VlVisionModel(
+      (patchifier): Linear(in_features=1536, out_features=256, bias=False)
+      (embedding_norm): Lfm3VlRMSNorm((256,), eps=1e-05)
+      (rotary_pos_emb): Lfm3VlVisionRotaryEmbedding()
+      (layers): ModuleList(
+        (0-1): 2 x Lfm3VlVisionLayer(
+          (conv): Lfm3VlVisionConv2d(
+            (in_proj): Linear(in_features=256, out_features=768, bias=False)
+            (spatial_conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=256, bias=False)
+            (out_proj): Linear(in_features=256, out_features=256, bias=False)
+          )
+          (feed_forward): Lfm3VlMLP(
+            (w1): Linear(in_features=256, out_features=640, bias=False)
+            (w3): Linear(in_features=256, out_features=640, bias=False)
+            (w2): Linear(in_features=640, out_features=256, bias=False)
+          )
+          (operator_norm): Lfm3VlRMSNorm((256,), eps=1e-05)
+          (ffn_norm): Lfm3VlRMSNorm((256,), eps=1e-05)
+        )
+        (2): Lfm3VlVisionLayer(
+          (self_attn): Lfm3VlVisionAttention(
+            (q_proj): Linear(in_features=256, out_features=256, bias=False)
+            (k_proj): Linear(in_features=256, out_features=128, bias=False)
+            (v_proj): Linear(in_features=256, out_features=128, bias=False)
+            (out_proj): Linear(in_features=256, out_features=256, bias=False)
+            (q_layernorm): Lfm3VlRMSNorm((32,), eps=1e-05)
+            (k_layernorm): Lfm3VlRMSNorm((32,), eps=1e-05)
+          )
+          (feed_forward): Lfm3VlMLP(
+            (w1): Linear(in_features=256, out_features=640, bias=False)
+            (w3): Linear(in_features=256, out_features=640, bias=False)
+            (w2): Linear(in_features=640, out_features=256, bias=False)
+          )
+          (operator_norm): Lfm3VlRMSNorm((256,), eps=1e-05)
+          (ffn_norm): Lfm3VlRMSNorm((256,), eps=1e-05)
+        )
+        (3): Lfm3VlVisionLayer(
+          (conv): Lfm3VlVisionConv2d(
+            (in_proj): Linear(in_features=256, out_features=768, bias=False)
+            (spatial_conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=256, bias=False)
+            (out_proj): Linear(in_features=256, out_features=256, bias=False)
+          )
+          (feed_forward): Lfm3VlMLP(
+            (w1): Linear(in_features=256, out_features=640, bias=False)
+            (w3): Linear(in_features=256, out_features=640, bias=False)
+            (w2): Linear(in_features=640, out_features=256, bias=False)
+          )
+          (operator_norm): Lfm3VlRMSNorm((256,), eps=1e-05)
+          (ffn_norm): Lfm3VlRMSNorm((256,), eps=1e-05)
+        )
+        (4): Lfm3VlVisionLayer(
+          (self_attn): Lfm3VlVisionAttention(
+            (q_proj): Linear(in_features=256, out_features=256, bias=False)
+            (k_proj): Linear(in_features=256, out_features=128, bias=False)
+            (v_proj): Linear(in_features=256, out_features=128, bias=False)
+            (out_proj): Linear(in_features=256, out_features=256, bias=False)
+            (q_layernorm): Lfm3VlRMSNorm((32,), eps=1e-05)
+            (k_layernorm): Lfm3VlRMSNorm((32,), eps=1e-05)
+          )
+          (feed_forward): Lfm3VlMLP(
+            (w1): Linear(in_features=256, out_features=640, bias=False)
+            (w3): Linear(in_features=256, out_features=640, bias=False)
+            (w2): Linear(in_features=640, out_features=256, bias=False)
+          )
+          (operator_norm): Lfm3VlRMSNorm((256,), eps=1e-05)
+          (ffn_norm): Lfm3VlRMSNorm((256,), eps=1e-05)
+        )
+        (5): Lfm3VlVisionLayer(
+          (conv): Lfm3VlVisionConv2d(
+            (in_proj): Linear(in_features=256, out_features=768, bias=False)
+            (spatial_conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=256, bias=False)
+            (out_proj): Linear(in_features=256, out_features=256, bias=False)
+          )
+          (feed_forward): Lfm3VlMLP(
+            (w1): Linear(in_features=256, out_features=640, bias=False)
+            (w3): Linear(in_features=256, out_features=640, bias=False)
+            (w2): Linear(in_features=640, out_features=256, bias=False)
+          )
+          (operator_norm): Lfm3VlRMSNorm((256,), eps=1e-05)
+          (ffn_norm): Lfm3VlRMSNorm((256,), eps=1e-05)
+        )
+      )
+    )
+    (multi_modal_projector): Lfm3VlMultiModalProjector(
+      (layer_norm): LayerNorm((256,), eps=1e-06, elementwise_affine=True, bias=True)
+      (linear_1): Linear(in_features=1024, out_features=128, bias=True)
+      (act): GELUActivation()
+      (linear_2): Linear(in_features=128, out_features=64, bias=True)
+    )
+    (language_model): Lfm3Model(
+      (embed_tokens): Embedding(128000, 64, padding_idx=124893)
+      (layers): ModuleList(
+        (0-1): 2 x Lfm3DecoderLayer(
+          (conv): Lfm3ShortConv(
+            (conv): Conv1d(64, 64, kernel_size=(7,), stride=(1,), padding=(6,), groups=64, bias=False)
+            (in_proj): Linear(in_features=64, out_features=192, bias=False)
+            (out_proj): Linear(in_features=64, out_features=64, bias=False)
+          )
+          (feed_forward): Lfm3MLP(
+            (w1): Linear(in_features=64, out_features=128, bias=False)
+            (w3): Linear(in_features=64, out_features=128, bias=False)
+            (w2): Linear(in_features=128, out_features=64, bias=False)
+          )
+          (operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (post_operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (post_ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
+        )
+        (2): Lfm3DecoderLayer(
+          (self_attn): Lfm3Attention(
+            (q_proj): Linear(in_features=64, out_features=512, bias=False)
+            (k_proj): Linear(in_features=64, out_features=128, bias=False)
+            (v_proj): Linear(in_features=64, out_features=128, bias=False)
+            (o_proj): Linear(in_features=512, out_features=64, bias=False)
+            (q_norm): Lfm3RMSNorm((32,), eps=1e-05)
+            (k_norm): Lfm3RMSNorm((32,), eps=1e-05)
+            (gate_proj): Linear(in_features=64, out_features=16, bias=False)
+          )
+          (feed_forward): Lfm3MLP(
+            (w1): Linear(in_features=64, out_features=128, bias=False)
+            (w3): Linear(in_features=64, out_features=128, bias=False)
+            (w2): Linear(in_features=128, out_features=64, bias=False)
+          )
+          (operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (post_operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (post_ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
+        )
+        (3): Lfm3DecoderLayer(
+          (conv): Lfm3ShortConv(
+            (conv): Conv1d(64, 64, kernel_size=(7,), stride=(1,), padding=(6,), groups=64, bias=False)
+            (in_proj): Linear(in_features=64, out_features=192, bias=False)
+            (out_proj): Linear(in_features=64, out_features=64, bias=False)
+          )
+          (feed_forward): Lfm3MLP(
+            (w1): Linear(in_features=64, out_features=128, bias=False)
+            (w3): Linear(in_features=64, out_features=128, bias=False)
+            (w2): Linear(in_features=128, out_features=64, bias=False)
+          )
+          (operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (post_operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (post_ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
+        )
+        (4): Lfm3DecoderLayer(
+          (self_attn): Lfm3Attention(
+            (q_proj): Linear(in_features=64, out_features=512, bias=False)
+            (k_proj): Linear(in_features=64, out_features=128, bias=False)
+            (v_proj): Linear(in_features=64, out_features=128, bias=False)
+            (o_proj): Linear(in_features=512, out_features=64, bias=False)
+            (q_norm): Lfm3RMSNorm((32,), eps=1e-05)
+            (k_norm): Lfm3RMSNorm((32,), eps=1e-05)
+            (gate_proj): Linear(in_features=64, out_features=16, bias=False)
+          )
+          (feed_forward): Lfm3MLP(
+            (w1): Linear(in_features=64, out_features=128, bias=False)
+            (w3): Linear(in_features=64, out_features=128, bias=False)
+            (w2): Linear(in_features=128, out_features=64, bias=False)
+          )
+          (operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (post_operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (post_ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
+        )
+        (5): Lfm3DecoderLayer(
+          (conv): Lfm3ShortConv(
+            (conv): Conv1d(64, 64, kernel_size=(7,), stride=(1,), padding=(6,), groups=64, bias=False)
+            (in_proj): Linear(in_features=64, out_features=192, bias=False)
+            (out_proj): Linear(in_features=64, out_features=64, bias=False)
+          )
+          (feed_forward): Lfm3MLP(
+            (w1): Linear(in_features=64, out_features=128, bias=False)
+            (w3): Linear(in_features=64, out_features=128, bias=False)
+            (w2): Linear(in_features=128, out_features=64, bias=False)
+          )
+          (operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (post_operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (post_ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
+        )
+        (6): Lfm3DecoderLayer(
+          (self_attn): Lfm3Attention(
+            (q_proj): Linear(in_features=64, out_features=512, bias=False)
+            (k_proj): Linear(in_features=64, out_features=128, bias=False)
+            (v_proj): Linear(in_features=64, out_features=128, bias=False)
+            (o_proj): Linear(in_features=512, out_features=64, bias=False)
+            (q_norm): Lfm3RMSNorm((32,), eps=1e-05)
+            (k_norm): Lfm3RMSNorm((32,), eps=1e-05)
+            (gate_proj): Linear(in_features=64, out_features=16, bias=False)
+          )
+          (feed_forward): Lfm3MLP(
+            (w1): Linear(in_features=64, out_features=128, bias=False)
+            (w3): Linear(in_features=64, out_features=128, bias=False)
+            (w2): Linear(in_features=128, out_features=64, bias=False)
+          )
+          (operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (post_operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (post_ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
+        )
+        (7): Lfm3DecoderLayer(
+          (conv): Lfm3ShortConv(
+            (conv): Conv1d(64, 64, kernel_size=(7,), stride=(1,), padding=(6,), groups=64, bias=False)
+            (in_proj): Linear(in_features=64, out_features=192, bias=False)
+            (out_proj): Linear(in_features=64, out_features=64, bias=False)
+          )
+          (feed_forward): Lfm3MLP(
+            (w1): Linear(in_features=64, out_features=128, bias=False)
+            (w3): Linear(in_features=64, out_features=128, bias=False)
+            (w2): Linear(in_features=128, out_features=64, bias=False)
+          )
+          (operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (post_operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (post_ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
+        )
+        (8): Lfm3DecoderLayer(
+          (self_attn): Lfm3Attention(
+            (q_proj): Linear(in_features=64, out_features=512, bias=False)
+            (k_proj): Linear(in_features=64, out_features=128, bias=False)
+            (v_proj): Linear(in_features=64, out_features=128, bias=False)
+            (o_proj): Linear(in_features=512, out_features=64, bias=False)
+            (q_norm): Lfm3RMSNorm((32,), eps=1e-05)
+            (k_norm): Lfm3RMSNorm((32,), eps=1e-05)
+            (gate_proj): Linear(in_features=64, out_features=16, bias=False)
+          )
+          (feed_forward): Lfm3MLP(
+            (w1): Linear(in_features=64, out_features=128, bias=False)
+            (w3): Linear(in_features=64, out_features=128, bias=False)
+            (w2): Linear(in_features=128, out_features=64, bias=False)
+          )
+          (operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (post_operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (post_ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
+        )
+        (9): Lfm3DecoderLayer(
+          (conv): Lfm3ShortConv(
+            (conv): Conv1d(64, 64, kernel_size=(7,), stride=(1,), padding=(6,), groups=64, bias=False)
+            (in_proj): Linear(in_features=64, out_features=192, bias=False)
+            (out_proj): Linear(in_features=64, out_features=64, bias=False)
+          )
+          (feed_forward): Lfm3MLP(
+            (w1): Linear(in_features=64, out_features=128, bias=False)
+            (w3): Linear(in_features=64, out_features=128, bias=False)
+            (w2): Linear(in_features=128, out_features=64, bias=False)
+          )
+          (operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (post_operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
+          (post_ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
+        )
+      )
+      (rotary_emb): Lfm3RotaryEmbedding()
+      (embedding_norm): Lfm3RMSNorm((64,), eps=1e-05)
+    )
+  )
+  (lm_head): Linear(in_features=64, out_features=128000, bias=False)
+)

LICENSE

@@ -0,0 +1,71 @@
+LFM Open License v1.0
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by this document.
+
+"Licensor" shall mean Liquid AI, Inc.
+
+"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files.
+
+"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work.
+
+"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof.
+
+"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work.
+
+"Commercial Use" shall mean any use of the Work for direct or indirect commercial advantage or monetary compensation.
+
+"Qualified Non-Profit Organization" shall mean a Legal Entity that is organized and operated exclusively for religious, charitable, scientific, testing for public safety, literary, or educational purposes, and which is exempt from federal income tax under Section 501(c)(3) of the United States Internal Revenue Code of 1986, as amended, or any equivalent non-profit or charitable organization in a foreign jurisdiction.
+
+"Non-Commercial or Research Purposes" shall mean purposes that do not involve any use of the Work or a Derivative Work for Commercial Use.
+
+"Threshold" shall mean annual revenue of 10 million United States dollars ($10,000,000) or more.
+
+2. Grant of Copyright License. Subject to the terms and conditions of this License, including the Commercial Use limitation set forth in Section 5, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of this License, including the Commercial Use limitation set forth in Section 5, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions:
+
+(a) You must give any other recipients of the Work or Derivative Works a copy of this License; and
+
+(b) You must cause any modified files to carry prominent notices stating that You changed the files; and
+
+(c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and
+
+(d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License.
+
+You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License.
+
+5. Commercial Use Limitation.
+
+(a) The rights granted under this License for Commercial Use are conditioned upon You or Your Legal Entity not exceeding the Threshold.
+
+(b) Any Commercial Use of the Work or a Derivative Work by a Legal Entity that exceeds the Threshold is not licensed under this Agreement.
+
+(c) The Threshold shall not apply to a Qualified Non-Profit Organization's use of the Work or a Derivative Work for Non-Commercial or Research Purposes.
+
+6. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions.
+
+7. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except for the reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file.
+
+8. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License.
+
+9. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.
+
+10. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
+
+11. Termination. This License will terminate automatically and immediately if You fail to comply with any of its terms and conditions. Upon termination, You must cease all use of the Work and any Derivative Works and delete all copies in Your possession.
+
+END OF TERMS AND CONDITIONS

README.md

@@ -0,0 +1,106 @@
+---
+library_name: transformers
+pipeline_tag: image-text-to-text
+base_model:
+- LiquidAI/LFM3-VL-2B
+---
+
+This is a randomly initialized tiny model based on [LiquidAI/LFM3-VL-2B](https://huggingface.co/LiquidAI/LFM3-VL-2B) for testing and debugging.
+
+| Feature | Source | This repo |
+|---|---|---|
+| Checkpoint tensor elements | 1.92B BF16 | 13.81M BF16 |
+| Safetensors weight files | 3.84 GB (model.safetensors) | 27.64 MB (model.safetensors) |
+| Text layers / layer-type cycle | 24 / two-conv prefix, then full/conv/sliding/conv | 10 / same prefix and two complete cycles |
+| Text hidden width / MLP width | 2,048 / 7,680 | 64 / 128 |
+| Text Q heads / KV heads / head dim | 16 / 4 / 128 | 16 / 4 / 32 |
+| Vision layers / layer-type cycle | 14 / two-conv prefix, then six full/conv pairs | 6 / same prefix and two complete full/conv cycles |
+| Vision hidden width / MLP width | 1,024 / 2,560 | 256 / 640 |
+| Vision Q heads / KV heads / head dim | 16 / 8 / 64 | 8 / 4 / 32 |
+| Patch / temporal patch / spatial merge | 16 / 2 / 2 | Same |
+| Vision depthwise conv kernel | 3 x 3 | Same |
+| RoPE | vision axial theta 1e6; text full/sliding dual theta | Same |
+| Projector MLP hidden / downsample factor | 4,096 / 2 | 128 / 2 |
+| Sandwich norms / QK RMSNorm / sigmoid head gate | present | Same |
+| Vocabulary rows / embedding tying | 128,000 / tied | Same |
+| Text maximum context length | 128,000 | Same |
+| Stored precision | BF16 | Same |
+
+Text-backbone widths follow [tiny-random/lfm3](https://huggingface.co/tiny-random/lfm3), which follows [tiny-random/lfm2](https://huggingface.co/tiny-random/lfm2). The pinned upstream implementation is included for Transformers loading. This repo provides a Transformers example with image and video inputs; no serving-backend validation is claimed.
+
+| File path | Size |
+|------|------|
+| model.safetensors | 27.6MB |
+
+### Example usage
+
+#### Transformers (✅ verified)
+
+Load the included pinned upstream implementation with `trust_remote_code=True`. The example sends one image and four predecoded video frames with complete `VideoMetadata`; the Triton short-conv path requires CUDA.
+
+```python
+def run_transformers(model_id: str):
+    import numpy as np
+    import torch
+    from PIL import Image
+    from transformers import AutoModelForImageTextToText, AutoProcessor
+    from transformers.video_utils import VideoMetadata
+
+    model_id = str(model_id)
+    processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True)
+    model = (
+        AutoModelForImageTextToText.from_pretrained(
+            model_id, trust_remote_code=True, dtype=torch.bfloat16, attn_implementation="sdpa"
+        )
+        .eval()
+        .cuda()
+    )
+    image = Image.fromarray(np.random.randint(0, 255, (256, 256, 3), dtype=np.uint8), "RGB")
+    frames = np.stack(
+        [
+            np.asarray(Image.new("RGB", (256, 256), color))
+            for color in ((220, 30, 30), (30, 220, 30), (30, 30, 220), (220, 220, 30))
+        ]
+    )
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "image", "image": image},
+                {"type": "video", "video": frames},
+                {"type": "text", "text": "Describe the image and the video."},
+            ],
+        }
+    ]
+    inputs = processor.apply_chat_template(
+        messages,
+        add_generation_prompt=True,
+        tokenize=True,
+        return_dict=True,
+        return_tensors="pt",
+        enable_thinking=False,
+        processor_kwargs={"video_metadata": [VideoMetadata(total_num_frames=4, fps=2, frames_indices=[0, 1, 2, 3])]},
+    ).to(model.device)
+    assert inputs["pixel_values"].shape[0] > 0 and inputs["pixel_values_videos"].shape[0] > 0
+    with torch.no_grad():
+        logits = model(**inputs).logits
+        assert torch.isfinite(logits).all()
+        output = model.generate(**inputs, do_sample=False, max_new_tokens=32)
+    print(processor.decode(output[0, inputs["input_ids"].shape[1] :], skip_special_tokens=False))
+
+
+if __name__ == "__main__":
+    run_transformers(model_id="tiny-random/lfm3-vl")
+```
+
+### Creating this repo
+
+- Code: [.meta/create_model.py](.meta/create_model.py)
+- Model structure: [.meta/model.log](.meta/model.log)
+- Environment: [.meta/env.json](.meta/env.json)
+
+### Change log
+
+- 2026-11-01: Initial version. Text backbone matches tiny-random/lfm3's residual/MLP widths (64/128, ten layers with the two-conv prefix and two full/conv/sliding/conv cycles) but keeps the source 4:1 Q/KV ratio with 16 Q heads and at least four KV heads (4) at head dim 32 for TP=4. The vision tower keeps the source 2:1 Q/KV ratio with 8 Q heads / 4 KV heads, which requires width 256 for its derived head dim of 32; it retains the conv/conv prefix, two full/conv cycles, patch 16, temporal patch 2, merge size 2 and axial RoPE.
+
+*This repo is generated by GPT-6.1-Sol.*

chat_template.jinja

@@ -0,0 +1,280 @@
+{{- bos_token -}}
+{%- set preserve_thinking = preserve_thinking | default(false) -%}
+{%- set enable_thinking = enable_thinking | default(true) -%}
+
+{%- macro render_value(v) -%}
+    {%- if v is string -%}
+        {{- v -}}
+    {%- elif v is boolean -%}
+        {{- "true" if v else "false" -}}
+    {%- elif v is none -%}
+        {{- "null" -}}
+    {%- elif v is mapping or (v is sequence and v is not string) -%}
+        {{- v | tojson -}}
+    {%- else -%}
+        {{- v -}}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro parse_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- elif content is mapping and content.get("type") == "image" -%}
+        {{- "<|vision_start|><|image|><|vision_end|>" -}}
+    {%- elif content is mapping and content.get("type") == "video" -%}
+        {{- "<|vision_start|><|video|><|vision_end|>" -}}
+    {%- elif content is mapping and content.get("type") == "text" -%}
+        {{- (content.get("text") or "") | string -}}
+    {%- elif content is mapping -%}
+        {{- content | tojson -}}
+    {%- elif content is iterable -%}
+        {%- set _ns = namespace(result="") -%}
+        {%- for item in content -%}
+            {%- if item is string -%}
+                {%- set _ns.result = _ns.result + item -%}
+            {%- elif item is mapping and item.get("type") == "image" -%}
+                {%- set _ns.result = _ns.result + "<|vision_start|><|image|><|vision_end|>" -%}
+            {%- elif item is mapping and item.get("type") == "video" -%}
+                {%- set _ns.result = _ns.result + "<|vision_start|><|video|><|vision_end|>" -%}
+            {%- elif item is mapping and item.get("type") == "text" -%}
+                {%- set _ns.result = _ns.result + ((item.get("text") or "") | string) -%}
+            {%- else -%}
+                {%- set _ns.result = _ns.result + (item | tojson) -%}
+            {%- endif -%}
+        {%- endfor -%}
+        {{- _ns.result -}}
+    {%- elif content is not none -%}
+        {{- content | tojson -}}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro render_tool_calls(tool_calls) -%}
+    {%- for tool_call in tool_calls -%}
+        {%- if not loop.first -%}{{- "\n\n" -}}{%- endif -%}
+        {%- set func = tool_call["function"] if "function" in tool_call else tool_call -%}
+        {%- set func_args = func.get("arguments") -%}
+        {%- if func_args is string and (func_args | trim) not in ["", "{}", "null"] -%}
+            {{- raise_exception("Tool call arguments must be a mapping, got a JSON-encoded string: parse arguments with json.loads() before applying the chat template") -}}
+        {%- endif -%}
+        {{- "<tool_call>\n<function=" + func["name"] + ">\n" -}}
+        {%- if func_args is mapping -%}
+            {%- for arg_name, arg_value in func_args.items() -%}
+                {{- "<parameter=" + arg_name + ">\n" + render_value(arg_value) + "\n</parameter>\n" -}}
+            {%- endfor -%}
+        {%- endif -%}
+        {{- "</function>\n</tool_call>" -}}
+    {%- endfor -%}
+{%- endmacro -%}
+
+{%- macro tool_to_json(tool) -%}
+    {%- set fn = tool.function if tool.function is defined else tool -%}
+    {%- set ns_t = namespace(first=true) -%}
+    {{- "{" -}}
+    {%- for k, v in fn.items() -%}
+        {%- if k != "defer_loading" -%}
+            {%- if not ns_t.first -%}{{- ", " -}}{%- endif -%}
+            {%- set ns_t.first = false -%}
+            {{- k | tojson -}}{{- ": " -}}{{- v | tojson -}}
+        {%- endif -%}
+    {%- endfor -%}
+    {{- "}" -}}
+{%- endmacro -%}
+
+{%- macro is_tool_reference(item) -%}
+    {{- "1" if item is mapping and item.get("type") == "tool_reference" else "" -}}
+{%- endmacro -%}
+
+{%- macro referenced_tool_to_json(name) -%}
+    {%- set _ns = namespace(found=none) -%}
+    {%- for tool in tools or [] -%}
+        {%- set fn = tool.function if tool.function is defined else tool -%}
+        {%- if fn.name == name -%}
+            {%- set _ns.found = fn -%}
+        {%- endif -%}
+    {%- endfor -%}
+    {%- if _ns.found is none -%}
+        {{- raise_exception("Tool reference '" + name | string + "' not found in available tools. Every tool referenced by a tool_reference must be in `tools` on every request.") -}}
+    {%- endif -%}
+    {{- tool_to_json(_ns.found) -}}
+{%- endmacro -%}
+
+{%- macro render_tool_response(content) -%}
+    {%- if content is string and "<tool_response>" in content -%}
+        {{- content -}}
+    {%- else -%}
+    {{- "<tool_response>\n" -}}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- elif content is mapping -%}
+        {{- parse_content(content) -}}
+    {%- elif content is iterable -%}
+        {%- for item in content -%}
+            {%- if not loop.first -%}{{- "\n" -}}{%- endif -%}
+            {%- if is_tool_reference(item) -%}
+                {%- if loop.first or not is_tool_reference(loop.previtem) -%}
+                    {{- "<additional_tools>\n" -}}
+                {%- endif -%}
+                {{- referenced_tool_to_json(item.name) -}}
+                {%- if loop.last or not is_tool_reference(loop.nextitem) -%}
+                    {{- "\n</additional_tools>" -}}
+                {%- endif -%}
+            {%- elif item is mapping and item.result is defined and item.result is mapping and item.result.tools is defined -%}
+                {{- "<additional_tools>\n" -}}
+                {%- set _tl = item.result.tools -%}
+                {%- if _tl is string -%}
+                    {{- _tl -}}
+                {%- elif _tl is mapping -%}
+                    {{- tool_to_json(_tl) -}}
+                {%- else -%}
+                    {%- for t in _tl -%}
+                        {%- if not loop.first -%}{{- "\n" -}}{%- endif -%}
+                        {{- tool_to_json(t) -}}
+                    {%- endfor -%}
+                {%- endif -%}
+                {{- "\n</additional_tools>" -}}
+            {%- else -%}
+                {{- parse_content(item) -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- elif content is not none -%}
+        {{- content | tojson -}}
+    {%- endif -%}
+    {{- "\n</tool_response>" -}}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- set ns = namespace(system_prompt="", last_user_index=-1) -%}
+{%- if messages and messages[0]["role"] == "system" -%}
+    {%- if messages[0].get("content") -%}
+        {%- set ns.system_prompt = parse_content(messages[0]["content"]) -%}
+    {%- endif -%}
+    {%- set messages = messages[1:] -%}
+{%- endif -%}
+{%- if tools -%}
+    {%- set tools_ns = namespace(lines=[]) -%}
+    {%- for tool in tools -%}
+        {%- set fn = tool.function if tool.function is defined else tool -%}
+        {%- set _dyn = (fn.defer_loading is defined and fn.defer_loading) or (tool.defer_loading is defined and tool.defer_loading) -%}
+        {%- if not _dyn -%}
+            {%- set tools_ns.lines = tools_ns.lines + [tool_to_json(fn)] -%}
+        {%- endif -%}
+    {%- endfor -%}
+    {%- if tools_ns.lines -%}
+        {%- set ns.system_prompt = ns.system_prompt + ("\n\n" if ns.system_prompt else "") + "<tools>\n" + (tools_ns.lines | join("\n")) + "\n</tools>" -%}
+    {%- endif -%}
+{%- endif -%}
+{%- if ns.system_prompt -%}
+    {{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}}
+{%- endif -%}
+{%- for message in messages -%}
+    {%- if message["role"] == "user" -%}
+        {%- set ns.last_user_index = loop.index0 -%}
+    {%- endif -%}
+{%- endfor -%}
+{%- set carry = namespace(pending=[]) -%}
+{%- for message in messages -%}
+    {%- set _next = messages[loop.index0 + 1] if not loop.last else none -%}
+    {%- set _merge = message.role == "tool" and not message.get("additional_tools")
+        and _next is not none and _next.role == "tool" and not _next.get("additional_tools")
+        and message.get("tool_call_id") and _next.get("tool_call_id") == message.get("tool_call_id") -%}
+    {%- set _items = [] -%}
+    {%- if message.role == "tool" and not message.get("additional_tools") -%}
+        {%- set _c = message.get("content") -%}
+        {%- if _c is string -%}
+            {%- set _items = [{"type": "text", "text": _c}] if _c else [] -%}
+        {%- elif _c is mapping -%}
+            {%- set _items = [_c] -%}
+        {%- elif _c is iterable -%}
+            {%- set _items = _c | list -%}
+        {%- elif _c is not none -%}
+            {%- set _items = [_c] -%}
+        {%- endif -%}
+    {%- endif -%}
+    {%- if _merge -%}
+        {%- set carry.pending = carry.pending + _items -%}
+    {%- else -%}
+    {{- "<|im_start|>" + message.role + "\n" -}}
+    {%- if message.role == "assistant" -%}
+        {%- generation -%}
+        {%- set thinking = message.thinking if message.thinking is defined else message.reasoning if message.reasoning is defined else message.reasoning_content -%}
+        {%- set keep_thinking = preserve_thinking or loop.index0 > ns.last_user_index -%}
+        {%- if thinking is defined and keep_thinking -%}
+            {{- "<think>" + (thinking or "") + "</think>" -}}
+        {%- elif keep_thinking and "</think>" not in (message.get("content") if message.get("content") is string else "") -%}
+            {{- "<think></think>" -}}
+        {%- endif -%}
+        {%- set _cfm_tag = "CONTINUE_FINAL_MESSAGE_TAG " -%}
+        {%- set _has_cfm = false -%}
+        {%- set content = "" -%}
+        {%- if message.get("content") -%}
+            {%- set content = parse_content(message.content) -%}
+        {%- endif -%}
+        {%- if not keep_thinking -%}
+            {%- if "</think>" in content -%}
+                {%- set content = content.split("</think>")[-1] | trim -%}
+            {%- endif -%}
+            {%- set content = "<think></think>" + content -%}
+        {%- endif -%}
+        {%- if content.endswith(_cfm_tag) -%}
+            {%- set _has_cfm = true -%}
+            {%- set _trunc_len = (content | length) - (_cfm_tag | length) -%}
+            {%- set content = content[:_trunc_len] -%}
+        {%- endif -%}
+        {{- content -}}
+        {%- if message.tool_calls -%}
+            {{- render_tool_calls(message.tool_calls) -}}
+        {%- endif -%}
+        {%- if _has_cfm -%}
+            {{- _cfm_tag -}}
+        {%- endif -%}
+        {{- "<|im_end|>\n" -}}
+        {%- endgeneration -%}
+    {%- elif message.role == "tool" -%}
+        {%- if message.get("additional_tools") -%}
+            {%- set _at = message["additional_tools"] -%}
+            {%- if _at is mapping or (_at is sequence and _at is not string) -%}
+                {%- set _src = _at -%}
+            {%- else -%}
+                {%- set _src = message.get("content") -%}
+            {%- endif -%}
+            {%- if _src is string and "<tool_response>" in _src -%}
+                {{- _src -}}
+            {%- elif _src is string and "<additional_tools>" in _src -%}
+                {{- "<tool_response>\n" + _src + "\n</tool_response>" -}}
+            {%- elif _src -%}
+                {{- "<tool_response>\n<additional_tools>\n" -}}
+                {%- if _src is string -%}
+                    {{- _src -}}
+                {%- elif _src is mapping -%}
+                    {{- tool_to_json(_src) -}}
+                {%- else -%}
+                    {%- for t in _src -%}
+                        {%- if not loop.first -%}{{- "\n" -}}{%- endif -%}
+                        {{- tool_to_json(t) -}}
+                    {%- endfor -%}
+                {%- endif -%}
+                {{- "\n</additional_tools>\n</tool_response>" -}}
+            {%- endif -%}
+        {%- elif carry.pending -%}
+            {{- render_tool_response(carry.pending + _items) -}}
+            {%- set carry.pending = [] -%}
+        {%- else -%}
+            {{- render_tool_response("" if message.get("content") is none else message.get("content")) -}}
+        {%- endif -%}
+        {{- "<|im_end|>\n" -}}
+    {%- else %}
+        {%- if message.get("content") -%}
+            {{- parse_content(message["content"]) -}}
+        {%- endif -%}
+        {{- "<|im_end|>\n" -}}
+    {%- endif %}
+    {%- endif -%}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{- "<|im_start|>assistant\n" -}}
+    {%- if enable_thinking -%}
+        {{- "<think>" -}}
+    {%- else -%}
+        {{- "<think></think>" -}}
+    {%- endif -%}
+{%- endif -%}

config.json

@@ -0,0 +1,100 @@
+{
+  "architectures": [
+    "Lfm3VlForConditionalGeneration"
+  ],
+  "bos_token_id": 124894,
+  "downsample_factor": 2,
+  "dtype": "bfloat16",
+  "eos_token_id": 124900,
+  "image_token_id": 125030,
+  "model_type": "lfm3_vl",
+  "pad_token_id": 124893,
+  "projector_bias": true,
+  "projector_hidden_act": "gelu",
+  "projector_hidden_size": 128,
+  "projector_layer_norm_eps": 1e-06,
+  "text_config": {
+    "bos_token_id": 124894,
+    "conv_L_cache": 7,
+    "conv_bias": false,
+    "dtype": "bfloat16",
+    "eos_token_id": 124900,
+    "head_dim": 32,
+    "hidden_size": 64,
+    "initializer_range": 0.02,
+    "intermediate_size": 128,
+    "layer_types": [
+      "conv",
+      "conv",
+      "full_attention",
+      "conv",
+      "sliding_attention",
+      "conv",
+      "full_attention",
+      "conv",
+      "sliding_attention",
+      "conv"
+    ],
+    "max_position_embeddings": 128000,
+    "model_type": "lfm3",
+    "norm_eps": 1e-05,
+    "num_attention_heads": 16,
+    "num_hidden_layers": 10,
+    "num_key_value_heads": 4,
+    "pad_token_id": 124893,
+    "rope_parameters": {
+      "full_attention": {
+        "partial_rotary_factor": 0.5,
+        "rope_theta": 1000000.0,
+        "rope_type": "default"
+      },
+      "sliding_attention": {
+        "rope_theta": 10000,
+        "rope_type": "default"
+      }
+    },
+    "sliding_window": 256,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_sandwich_norm": true,
+    "vocab_size": 128000
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.19.0.dev0",
+  "video_token_id": 125029,
+  "vision_config": {
+    "conv_kernel_size": 3,
+    "dtype": "bfloat16",
+    "hidden_size": 256,
+    "in_channels": 3,
+    "initializer_range": 0.02,
+    "intermediate_size": 640,
+    "layer_types": [
+      "conv",
+      "conv",
+      "full_attention",
+      "conv",
+      "full_attention",
+      "conv"
+    ],
+    "model_type": "lfm3_vl_vision",
+    "norm_eps": 1e-05,
+    "num_attention_heads": 8,
+    "num_hidden_layers": 6,
+    "num_key_value_heads": 4,
+    "patch_size": 16,
+    "rope_parameters": {
+      "rope_theta": 1000000.0,
+      "rope_type": "axial"
+    },
+    "spatial_merge_size": 2,
+    "temporal_patch_size": 2
+  },
+  "vision_end_token_id": 125010,
+  "vision_start_token_id": 125009,
+  "auto_map": {
+    "AutoConfig": "configuration_lfm3_vl.Lfm3VlConfig",
+    "AutoModel": "modeling_lfm3_vl.Lfm3VlModel",
+    "AutoModelForImageTextToText": "modeling_lfm3_vl.Lfm3VlForConditionalGeneration"
+  }
+}

configuration_lfm3.py

@@ -0,0 +1,132 @@
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from huggingface_hub.dataclasses import strict
+
+from transformers.configuration_utils import PreTrainedConfig
+from transformers.utils import auto_docstring
+
+
+@auto_docstring(checkpoint="LiquidAI/LFM3-1.7B")
+@strict
+class Lfm3Config(PreTrainedConfig):
+    r"""
+    rope_parameters (`dict[str, RopeParameters]`, *optional*):
+        RoPE parameters keyed by attention layer type, for example `{"full_attention": {"rope_type": "default",
+        "rope_theta": 1000000.0, "partial_rotary_factor": 0.5}, "sliding_attention": {"rope_type": "default",
+        "rope_theta": 10000.0}}`. `conv` layers do not use RoPE and have no entry. Full-attention layers rotate the
+        first `partial_rotary_factor * head_dim` dimensions and leave the rest unrotated.
+    conv_bias (`bool`, *optional*, defaults to `False`):
+        Whether to use bias in the conv layers.
+    conv_L_cache (`int`, *optional*, defaults to 7):
+        L_cache dim in the conv layers.
+    layer_types (`list[str]`, *optional*):
+        Per-layer operator type, each one of `"full_attention"`, `"sliding_attention"` or `"conv"`. Defaults to the
+        LFM3 pattern of a full-attention layer every fourth layer starting from the third, a sliding-attention layer
+        in between, and gated short convolutions elsewhere.
+    sliding_window (`int`, *optional*, defaults to 256):
+        Sliding-window size of the `sliding_attention` layers.
+    use_sandwich_norm (`bool`, *optional*, defaults to `True`):
+        Whether to apply an RMSNorm to the operator and feed-forward outputs before they enter the residual stream.
+
+    ```python
+    >>> from transformers import Lfm3Model, Lfm3Config
+
+    >>> # Initializing a LFM3 model
+    >>> configuration = Lfm3Config()
+
+    >>> # Initializing a model from the LFM3-1.7B style configuration
+    >>> model = Lfm3Model(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```
+    """
+
+    model_type = "lfm3"
+    keys_to_ignore_at_inference = ["past_key_values"]
+    default_theta = 1000000.0
+
+    vocab_size: int = 128_000
+    hidden_size: int = 2048
+    intermediate_size: int = 7680
+    num_hidden_layers: int = 24
+    num_attention_heads: int = 16
+    num_key_value_heads: int = 4
+    max_position_embeddings: int = 128_000
+    initializer_range: float = 0.02
+    norm_eps: float = 0.00001
+    use_cache: bool = True
+    pad_token_id: int | None = 124_893
+    bos_token_id: int | None = 124_894
+    eos_token_id: int | list[int] | None = 124_900
+    tie_word_embeddings: bool = True
+    rope_parameters: dict | None = None
+    conv_bias: bool = False
+    conv_L_cache: int = 7
+    layer_types: list[str] | None = None
+    _attention_layer_types = ("full_attention", "sliding_attention")
+    head_dim: int | None = None
+    sliding_window: int | None = 256
+    use_sandwich_norm: bool = True
+
+    def __post_init__(self, **kwargs):
+        if self.head_dim is None:
+            self.head_dim = self.hidden_size // self.num_attention_heads
+        if self.rope_parameters is None:
+            self.rope_parameters = {
+                "full_attention": {"rope_type": "default", "rope_theta": 1_000_000.0, "partial_rotary_factor": 0.5},
+                "sliding_attention": {"rope_type": "default", "rope_theta": 10_000.0},
+            }
+        if self.layer_types is None:
+            self.layer_types = [
+                "full_attention" if i % 4 == 2 else "sliding_attention" if i % 4 == 0 and i > 0 else "conv"
+                for i in range(self.num_hidden_layers)
+            ]
+
+        self.tie_word_embeddings = kwargs.pop("tie_embedding", self.tie_word_embeddings)
+        super().__post_init__(**kwargs)
+
+    def validate_architecture(self):
+        """Part of ``@strict``-powered validation."""
+        if len(self.layer_types) != self.num_hidden_layers:
+            raise ValueError(
+                f"`layer_types` has {len(self.layer_types)} entries but `num_hidden_layers` is {self.num_hidden_layers}."
+            )
+        allowed_layer_types = (*self._attention_layer_types, "conv")
+        invalid_layer_types = [layer_type for layer_type in self.layer_types if layer_type not in allowed_layer_types]
+        if invalid_layer_types:
+            raise ValueError(f"`layer_types` entries must be one of {allowed_layer_types}; got {invalid_layer_types}.")
+        if "sliding_attention" in self.layer_types and self.sliding_window is None:
+            raise ValueError("`layer_types` has `sliding_attention` layers but `sliding_window` is None.")
+        missing_rope = [
+            layer_type
+            for layer_type in self._attention_layer_types
+            if layer_type in self.layer_types and layer_type not in self.rope_parameters
+        ]
+        if missing_rope:
+            raise ValueError(f"`rope_parameters` has no entry for the {missing_rope} layers in `layer_types`.")
+        for layer_type, rope_params in self.rope_parameters.items():
+            if not isinstance(rope_params, dict):
+                continue
+            if (
+                rope_params.get("rope_type", "default") in ("dynamic", "longrope")
+                and rope_params.get("partial_rotary_factor", 1.0) < 1
+            ):
+                raise ValueError(
+                    f"`{rope_params['rope_type']}` RoPE recomputes its frequencies during the forward pass over the "
+                    f"rotated dimension only and cannot keep LFM3's partial rotation on `{layer_type}` layers."
+                )
+
+
+__all__ = ["Lfm3Config"]

configuration_lfm3_vl.py

@@ -0,0 +1,211 @@
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from huggingface_hub.dataclasses import strict
+
+from transformers.configuration_utils import PreTrainedConfig
+from transformers.models.auto import CONFIG_MAPPING, AutoConfig
+from transformers.utils import auto_docstring
+
+from .configuration_lfm3 import Lfm3Config
+
+
+def _config_class(model_type):
+    # the LFM3 configs ship with the checkpoint instead of the transformers registry
+    remote = {"lfm3": Lfm3Config, "lfm3_vl_vision": Lfm3VlVisionConfig}
+    return remote.get(model_type) or CONFIG_MAPPING[model_type]
+
+
+@auto_docstring(checkpoint="LiquidAI/LFM3-VL-2B")
+@strict
+class Lfm3VlVisionConfig(PreTrainedConfig):
+    r"""
+    layer_types (`list[str]`, *optional*):
+        Per-layer operator type, each one of `"full_attention"` or `"conv"`. Defaults to two gated convolution
+        layers followed by alternating attention and convolution layers.
+    patch_size (`int`, *optional*, defaults to 16):
+        The spatial size of each image patch.
+    temporal_patch_size (`int`, *optional*, defaults to 2):
+        The number of video frames packed into one patch.
+    spatial_merge_size (`int`, *optional*, defaults to 2):
+        The side of the square block of patches merged into one language token.
+    conv_kernel_size (`int`, *optional*, defaults to 3):
+        The kernel size of the depthwise 2D convolution layers.
+    norm_eps (`float`, *optional*, defaults to 1e-05):
+        The epsilon used by the RMS normalization layers.
+    """
+
+    model_type = "lfm3_vl_vision"
+    base_config_key = "vision_config"
+    default_theta = 1_000_000.0
+    default_rope_type = "axial"
+    _attention_layer_types = ("full_attention",)
+
+    hidden_size: int = 1024
+    intermediate_size: int = 2560
+    num_hidden_layers: int = 14
+    num_attention_heads: int = 16
+    num_key_value_heads: int = 8
+    layer_types: list[str] | None = None
+    patch_size: int = 16
+    temporal_patch_size: int = 2
+    in_channels: int = 3
+    spatial_merge_size: int = 2
+    conv_kernel_size: int = 3
+    norm_eps: float = 1e-5
+    initializer_range: float = 0.02
+    rope_parameters: dict | None = None
+
+    def __post_init__(self, **kwargs):
+        if self.layer_types is None:
+            self.layer_types = ["conv" if i < 2 or i % 2 else "full_attention" for i in range(self.num_hidden_layers)]
+        super().__post_init__(**kwargs)
+
+    def validate_architecture(self):
+        """Part of ``@strict``-powered validation."""
+        if len(self.layer_types) != self.num_hidden_layers:
+            raise ValueError(
+                f"`layer_types` has {len(self.layer_types)} entries but `num_hidden_layers` is {self.num_hidden_layers}."
+            )
+        allowed_layer_types = (*self._attention_layer_types, "conv")
+        invalid_layer_types = [layer_type for layer_type in self.layer_types if layer_type not in allowed_layer_types]
+        if invalid_layer_types:
+            raise ValueError(f"`layer_types` entries must be one of {allowed_layer_types}; got {invalid_layer_types}.")
+        if self.hidden_size % self.num_attention_heads:
+            raise ValueError("`hidden_size` must be a multiple of `num_attention_heads`.")
+
+
+# The first checkpoints spell every text and `vision_*` field at the top level of the config
+_LEGACY_TEXT_KEYS = (
+    "vocab_size",
+    "hidden_size",
+    "intermediate_size",
+    "num_hidden_layers",
+    "num_attention_heads",
+    "num_key_value_heads",
+    "head_dim",
+    "layer_types",
+    "conv_bias",
+    "conv_L_cache",
+    "sliding_window",
+    "max_position_embeddings",
+    "use_sandwich_norm",
+    "use_cache",
+)
+_LEGACY_VISION_KEYS = {
+    "vision_hidden_size": "hidden_size",
+    "vision_intermediate_size": "intermediate_size",
+    "vision_num_hidden_layers": "num_hidden_layers",
+    "vision_num_attention_heads": "num_attention_heads",
+    "vision_num_key_value_heads": "num_key_value_heads",
+    "vision_layer_types": "layer_types",
+    "vision_patch_size": "patch_size",
+    "vision_temporal_patch_size": "temporal_patch_size",
+    "vision_num_channels": "in_channels",
+    "vision_spatial_merge_size": "spatial_merge_size",
+}
+_LEGACY_DROPPED_KEYS = ("vision_patch_dim", "vision_max_position_embeddings")
+
+
+@auto_docstring(checkpoint="LiquidAI/LFM3-VL-2B")
+@strict
+class Lfm3VlConfig(PreTrainedConfig):
+    r"""
+    video_token_id (`int`, *optional*, defaults to 125029):
+        The id of the placeholder token for video patches.
+    vision_start_token_id (`int`, *optional*, defaults to 125009):
+        The id of the token opening an image or a video frame.
+    vision_end_token_id (`int`, *optional*, defaults to 125010):
+        The id of the token closing an image or a video frame.
+    projector_layer_norm_eps (`float`, *optional*, defaults to 1e-06):
+        The epsilon of the layer normalization applied to the vision features before projection.
+    downsample_factor (`int`, *optional*, defaults to 2):
+        The side of the square block of vision tokens merged by the projector, equal to the vision encoder's
+        `spatial_merge_size`.
+    """
+
+    model_type = "lfm3_vl"
+    sub_configs = {"text_config": AutoConfig, "vision_config": Lfm3VlVisionConfig}
+
+    text_config: dict | PreTrainedConfig | None = None
+    vision_config: dict | PreTrainedConfig | None = None
+    image_token_id: int = 125_030
+    video_token_id: int = 125_029
+    vision_start_token_id: int = 125_009
+    vision_end_token_id: int = 125_010
+    projector_hidden_act: str = "gelu"
+    projector_hidden_size: int = 4096
+    projector_bias: bool = True
+    projector_layer_norm_eps: float = 1e-6
+    downsample_factor: int = 2
+    tie_word_embeddings: bool = True
+
+    def __post_init__(self, **kwargs):
+        if "vision_hidden_size" in kwargs:
+            kwargs = self._convert_legacy_flat_config(kwargs)
+
+        if isinstance(self.vision_config, dict):
+            self.vision_config = self.sub_configs["vision_config"](**self.vision_config)
+        elif self.vision_config is None:
+            self.vision_config = self.sub_configs["vision_config"]()
+
+        if isinstance(self.text_config, dict):
+            self.text_config = {"model_type": "lfm3", **self.text_config}
+            self.text_config = _config_class(self.text_config["model_type"])(**self.text_config)
+        elif self.text_config is None:
+            self.text_config = _config_class("lfm3")()
+        self.validate_vision_fields()
+
+        super().__post_init__(**kwargs)
+
+    def validate_vision_fields(self):
+        """Part of `@strict`-powered validation, so `validate()` and `save_pretrained` refuse later edits too."""
+        if self.downsample_factor != self.vision_config.spatial_merge_size:
+            raise ValueError("downsample_factor must match vision_config.spatial_merge_size")
+        # an `inputs_embeds` forward embeds both placeholder ids to find them
+        vocab_size = self.text_config.vocab_size
+        for name in ("image_token_id", "video_token_id"):
+            if not 0 <= getattr(self, name) < vocab_size:
+                raise ValueError(f"{name}={getattr(self, name)} must lie in [0, text_config.vocab_size={vocab_size})")
+
+    def _convert_legacy_flat_config(self, kwargs: dict) -> dict:
+        text_config = {key: kwargs.pop(key) for key in _LEGACY_TEXT_KEYS if key in kwargs}
+        text_config["norm_eps"] = kwargs.pop("rms_norm_eps", 1e-5)
+        text_config["rope_parameters"] = {
+            "full_attention": {
+                "rope_type": "default",
+                "rope_theta": kwargs.pop("full_attention_rope_theta", 1_000_000.0),
+                "partial_rotary_factor": kwargs.pop("partial_rotary_factor", 0.5),
+            },
+            "sliding_attention": {
+                "rope_type": "default",
+                "rope_theta": kwargs.pop("sliding_attention_rope_theta", 10_000.0),
+            },
+        }
+        for key in ("pad_token_id", "bos_token_id", "eos_token_id"):
+            if kwargs.get(key) is not None:
+                text_config[key] = kwargs[key]
+        text_config["tie_word_embeddings"] = self.tie_word_embeddings
+
+        vision_config = {new: kwargs.pop(old) for old, new in _LEGACY_VISION_KEYS.items() if old in kwargs}
+        vision_config["norm_eps"] = text_config["norm_eps"]
+        vision_config["rope_parameters"] = {"rope_type": "axial", "rope_theta": kwargs.pop("vision_rope_theta", 1e6)}
+        for key in _LEGACY_DROPPED_KEYS:
+            kwargs.pop(key, None)
+
+        self.text_config = text_config
+        self.vision_config = vision_config
+        return kwargs
+
+
+__all__ = ["Lfm3VlConfig", "Lfm3VlVisionConfig"]

generation_config.json

@@ -0,0 +1,16 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 124894,
+  "do_sample": true,
+  "eos_token_id": 124900,
+  "min_p": 0.0,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 124893,
+  "repetition_penalty": 1.1,
+  "temperature": 0.1,
+  "top_k": 50,
+  "top_p": 1.0,
+  "transformers_version": "5.19.0.dev0",
+  "use_cache": true
+}

image_processing_lfm3_vl.py

@@ -0,0 +1,293 @@
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import math
+import warnings
+from collections.abc import Iterable
+
+import torch
+from torchvision.transforms.v2 import functional as tvF
+
+from transformers.image_processing_backends import TorchvisionBackend
+from transformers.image_processing_utils import BatchFeature
+from transformers.image_transforms import group_images_by_shape, reorder_images
+from transformers.image_utils import ImageInput, PILImageResampling, SizeDict
+from transformers.processing_utils import ImagesKwargs, Unpack
+from transformers.utils import TensorType, auto_docstring
+
+
+class Lfm3VlImageProcessorKwargs(ImagesKwargs, total=False):
+    r"""
+    min_pixels (`int`, *optional*, defaults to `56 * 56`):
+        The min pixels of the image to resize the image.
+    max_pixels (`int`, *optional*, defaults to `28 * 28 * 1280`):
+        The max pixels of the image to resize the image.
+    patch_size (`int`, *optional*, defaults to 14):
+        The spatial patch size of the vision encoder.
+    temporal_patch_size (`int`, *optional*, defaults to 2):
+        The temporal patch size of the vision encoder.
+    merge_size (`int`, *optional*, defaults to 2):
+        The merge size of the vision encoder to llm encoder.
+    """
+
+    min_pixels: int
+    max_pixels: int
+    patch_size: int
+    temporal_patch_size: int
+    merge_size: int
+
+
+def smart_resize(
+    height: int, width: int, factor: int = 28, min_pixels: int = 56 * 56, max_pixels: int = 14 * 14 * 4 * 1280
+):
+    """Rescales the image so that the following conditions are met:
+
+    1. Both dimensions (height and width) are divisible by 'factor'.
+
+    2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
+
+    3. The aspect ratio of the image is maintained as closely as possible.
+
+    """
+    if max(height, width) / min(height, width) > 200:
+        raise ValueError(
+            f"absolute aspect ratio must be smaller than 200, got {max(height, width) / min(height, width)}"
+        )
+    h_bar = round(height / factor) * factor
+    w_bar = round(width / factor) * factor
+    if h_bar * w_bar > max_pixels:
+        beta = math.sqrt((height * width) / max_pixels)
+        h_bar = max(factor, math.floor(height / beta / factor) * factor)
+        w_bar = max(factor, math.floor(width / beta / factor) * factor)
+    elif h_bar * w_bar < min_pixels:
+        beta = math.sqrt(min_pixels / (height * width))
+        h_bar = math.ceil(height * beta / factor) * factor
+        w_bar = math.ceil(width * beta / factor) * factor
+    return h_bar, w_bar
+
+
+@auto_docstring
+class Lfm3VlImageProcessor(TorchvisionBackend):
+    do_resize = True
+    resample = PILImageResampling.BICUBIC
+    size = {"shortest_edge": 65_536, "longest_edge": 16_777_216}
+    default_to_square = False
+    do_rescale = True
+    do_normalize = True
+    image_mean = [0.5, 0.5, 0.5]
+    image_std = [0.5, 0.5, 0.5]
+    do_convert_rgb = True
+    patch_size = 16
+    temporal_patch_size = 2
+    merge_size = 2
+    valid_kwargs = Lfm3VlImageProcessorKwargs
+    model_input_names = ["pixel_values", "image_grid_thw"]
+
+    def __init__(self, **kwargs: Unpack[Lfm3VlImageProcessorKwargs]):
+        # backward compatibility: override size with min_pixels and max_pixels if they are provided
+        size = kwargs.pop("size", None)
+        size = dict(self.size) if size is None else size
+        if (min_pixels := kwargs.pop("min_pixels", None)) is not None:
+            size["shortest_edge"] = min_pixels
+            size.pop("min_pixels", None)
+        if (max_pixels := kwargs.pop("max_pixels", None)) is not None:
+            size["longest_edge"] = max_pixels
+            size.pop("max_pixels", None)
+        super().__init__(size=size, **kwargs)
+
+    def _standardize_kwargs(
+        self,
+        size: int | Iterable[int] | dict[str, int] | SizeDict | None = None,
+        min_pixels: int | None = None,
+        max_pixels: int | None = None,
+        **kwargs,
+    ) -> dict:
+        if min_pixels is not None or max_pixels is not None:
+            warnings.warn(
+                "Passing `min_pixels` and `max_pixels` to a processor call is deprecated and will be removed in v5.23. "
+                "Pass in `size={'longest_edge': xxx, 'shortest_edge': xxx} to override the target size.`",
+                FutureWarning,
+            )
+
+            size_dict = dict(size) if isinstance(size, (dict, SizeDict)) else {}
+            if min_pixels is not None:
+                size_dict["shortest_edge"] = min_pixels
+            if max_pixels is not None:
+                size_dict["longest_edge"] = max_pixels
+            size = SizeDict(**size_dict)
+        return super()._standardize_kwargs(size=size, **kwargs)
+
+    @auto_docstring
+    def preprocess(
+        self,
+        images: ImageInput,
+        **kwargs: Unpack[Lfm3VlImageProcessorKwargs],
+    ) -> BatchFeature:
+        return super().preprocess(images, **kwargs)
+
+    def resize(
+        self,
+        images: "torch.Tensor",
+        size: SizeDict,
+        resample: "PILImageResampling | tvF.InterpolationMode | int | None",
+        factor: int,
+        **kwargs,
+    ) -> "torch.Tensor":
+        """Resize dynamically based on input image aspect ratio."""
+        if not size.shortest_edge or not size.longest_edge:
+            raise ValueError(f"`size` dict must contain 'shortest_edge' and 'longest_edge' keys but got {size}.")
+
+        height, width = images.shape[-2:]
+        resized_height, resized_width = smart_resize(
+            height,
+            width,
+            factor=factor,
+            min_pixels=size.shortest_edge,
+            max_pixels=size.longest_edge,
+        )
+        return super().resize(
+            image=images,
+            size=SizeDict(height=resized_height, width=resized_width),
+            resample=resample,
+        )
+
+    def patchify(
+        self,
+        images: "torch.Tensor",
+        patch_size: int,
+        merge_size: int,
+        temporal_patch_size: int,
+    ) -> tuple["torch.Tensor", int, int]:
+        """Patchifies each image into flat layout of shape (`seq_len`, `patch_dim`) so we can concat dynamically shaped pixels."""
+        batch_size, channel, resized_height, resized_width = images.shape
+        grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
+        patches = images.reshape(
+            batch_size,
+            channel,
+            grid_h // merge_size,
+            merge_size,
+            patch_size,
+            grid_w // merge_size,
+            merge_size,
+            patch_size,
+        )
+        # Reorder dimensions to group grid and patch information for subsequent flattening.
+        # [batch, grid_h/merge, grid_w/merge, merge, merge, channel, patch, patch]
+        patches = patches.permute(0, 2, 5, 3, 6, 1, 4, 7)
+
+        flatten_patches = (
+            patches.unsqueeze(6)
+            .expand(-1, -1, -1, -1, -1, -1, temporal_patch_size, -1, -1)
+            .reshape(
+                batch_size,
+                grid_h * grid_w,
+                channel * temporal_patch_size * patch_size * patch_size,
+            )
+        )
+        return flatten_patches, grid_h, grid_w
+
+    def _preprocess(
+        self,
+        images: list["torch.Tensor"],
+        do_resize: bool,
+        size: SizeDict,
+        resample: "PILImageResampling | tvF.InterpolationMode | int | None",
+        do_rescale: bool,
+        rescale_factor: float,
+        do_normalize: bool,
+        image_mean: float | list[float] | None,
+        image_std: float | list[float] | None,
+        patch_size: int,
+        temporal_patch_size: int,
+        merge_size: int,
+        disable_grouping: bool | None,
+        return_tensors: str | TensorType | None,
+        **kwargs,
+    ) -> BatchFeature:
+        grouped_images, grouped_images_index = group_images_by_shape(images, disable_grouping=disable_grouping)
+        resized_images_grouped = {}
+        for shape, stacked_images in grouped_images.items():
+            if do_resize:
+                stacked_images = self.resize(
+                    images=stacked_images,
+                    size=size,
+                    resample=resample,
+                    factor=patch_size * merge_size,
+                )
+            resized_images_grouped[shape] = stacked_images
+        resized_images = reorder_images(resized_images_grouped, grouped_images_index)
+
+        grouped_images, grouped_images_index = group_images_by_shape(resized_images, disable_grouping=disable_grouping)
+        processed_images_grouped = {}
+        processed_grids = {}
+        for shape, stacked_images in grouped_images.items():
+            stacked_images = self.rescale_and_normalize(
+                stacked_images, do_rescale, rescale_factor, do_normalize, image_mean, image_std
+            )
+            patches, grid_h, grid_w = self.patchify(
+                stacked_images,
+                patch_size=patch_size,
+                merge_size=merge_size,
+                temporal_patch_size=temporal_patch_size,
+            )
+
+            processed_images_grouped[shape] = patches
+            processed_grids[shape] = [[1, grid_h, grid_w]] * len(stacked_images)
+
+        processed_images = reorder_images(processed_images_grouped, grouped_images_index)
+        processed_grids_ordered = reorder_images(processed_grids, grouped_images_index)
+        pixel_values = processed_images[0] if len(processed_images) == 1 else torch.cat(processed_images, dim=0)
+        image_grid_thw = torch.tensor(processed_grids_ordered, dtype=torch.long)
+
+        return BatchFeature(
+            data={"pixel_values": pixel_values, "image_grid_thw": image_grid_thw}, tensor_type=return_tensors
+        )
+
+    def get_number_of_image_patches(self, height: int, width: int, images_kwargs: dict | None = None) -> int:
+        """
+        A utility that returns number of image patches for a given image size.
+
+        Note: Do not remove this method! It is used by vLLM to infer the number of patches and placeholders
+        without an image input.
+
+        Args:
+            height (`int`):
+                Height of the input image.
+            width (`int`):
+                Width of the input image.
+            images_kwargs (`dict`, *optional*)
+                Any kwargs to override defaults of the image processor.
+        Returns:
+            `int`: Number of image patches per image.
+        """
+        # `size` is the supported override (`min_pixels` / `max_pixels` are deprecated): count with the budget the
+        # preprocessing applies
+        images_kwargs = dict(images_kwargs or {})
+        size = images_kwargs.pop("size", None) or self.size
+        images_kwargs.setdefault("min_pixels", size["shortest_edge"])
+        images_kwargs.setdefault("max_pixels", size["longest_edge"])
+        images_kwargs = images_kwargs or {}
+        min_pixels = images_kwargs["min_pixels"] if "min_pixels" in images_kwargs else self.size["shortest_edge"]
+        max_pixels = images_kwargs["max_pixels"] if "max_pixels" in images_kwargs else self.size["longest_edge"]
+        patch_size = images_kwargs.get("patch_size", self.patch_size)
+        merge_size = images_kwargs.get("merge_size", self.merge_size)
+
+        factor = patch_size * merge_size
+        resized_height, resized_width = smart_resize(
+            height, width, factor, min_pixels=min_pixels, max_pixels=max_pixels
+        )
+        grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
+        return grid_h * grid_w
+
+
+__all__ = ["Lfm3VlImageProcessor"]

lfm3_conv.py

@@ -0,0 +1,202 @@
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Inference kernels for LFM3's width-7 depthwise convolution."""
+
+import torch
+
+from transformers.utils import is_triton_available
+
+
+if is_triton_available():
+    import triton
+    import triton.language as tl
+
+    # the length and strides are runtime arguments, or every prompt length would compile its own kernel
+    @triton.jit(do_not_specialize=["T", "X0", "X1", "X2", "S0", "S1", "ST"])
+    def _prefill(
+        X,
+        W,
+        BIAS,
+        SEQ,
+        Y,
+        T,
+        D: tl.constexpr,
+        X0,
+        X1,
+        X2,
+        S0,
+        S1,
+        ST,
+        HAS_BIAS: tl.constexpr,
+        PACKED: tl.constexpr,
+        BLOCK: tl.constexpr,
+    ):
+        # int64 offsets: batch * channels * length can exceed 2**31 elements
+        b, d = tl.program_id(0).to(tl.int64), tl.program_id(1).to(tl.int64)
+        t = tl.program_id(2).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
+        acc = tl.full((BLOCK,), 0, tl.float32)
+        if PACKED:
+            doc = tl.load(SEQ + b * S0 + (t - T + ST) * S1, (t < T) & (t >= T - ST), other=-1)
+        for k in tl.static_range(7):
+            pos = t + k - 6
+            valid = (t < T) & (pos >= 0)
+            if PACKED:
+                prev = tl.load(SEQ + b * S0 + (pos - T + ST) * S1, valid & (pos >= T - ST), other=-1)
+                valid = valid & (doc == prev)
+            x = tl.load(X + b * X0 + d * X1 + pos * X2, valid, other=0).to(tl.float32)
+            w = tl.load(W + d * 7 + k).to(tl.float32)
+            acc += x * w
+        if HAS_BIAS:
+            acc += tl.load(BIAS + d).to(tl.float32)
+        tl.store(Y + (b * D + d) * T + t, acc, t < T)
+
+    @triton.jit
+    def _decode(
+        X,
+        W,
+        BIAS,
+        STATE,
+        Y,
+        D: tl.constexpr,
+        K: tl.constexpr,
+        X0: tl.constexpr,
+        X1: tl.constexpr,
+        S0: tl.constexpr,
+        S1: tl.constexpr,
+        S2: tl.constexpr,
+        HAS_BIAS: tl.constexpr,
+        BLOCK: tl.constexpr,
+    ):
+        b = tl.program_id(0).to(tl.int64)
+        d = tl.program_id(1).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
+        k = tl.arange(0, 8)
+        # Load the complete state before writing the shifted window.
+        old = tl.load(STATE + b * S0 + d[:, None] * S1 + k[None, :] * S2, (d[:, None] < D) & (k[None, :] < K), other=0)
+        x = tl.load(X + b * X0 + d * X1, d < D, other=0)
+        shifted = tl.gather(old, tl.broadcast_to(tl.minimum(k + 1, 7)[None, :], (BLOCK, 8)), 1)
+        shifted = tl.where(k[None, :] == K - 1, x[:, None], shifted)
+        # A cache may store either K=6 previous inputs or K=7 including the oldest discarded input.
+        positions = k + K - 6
+        values = tl.gather(old, tl.broadcast_to(tl.minimum(positions, 7)[None, :], (BLOCK, 8)), 1)
+        values = tl.where(k[None, :] == 6, x[:, None], values)
+        weights = tl.load(W + d[:, None] * 7 + k[None, :], (d[:, None] < D) & (k[None, :] < 7), other=0)
+        acc = tl.sum(values.to(tl.float32) * weights.to(tl.float32), 1)
+        if HAS_BIAS:
+            acc += tl.load(BIAS + d, d < D, other=0).to(tl.float32)
+        tl.store(STATE + b * S0 + d[:, None] * S1 + k[None, :] * S2, shifted, (d[:, None] < D) & (k[None, :] < K))
+        tl.store(Y + b * D + d, acc, d < D)
+
+
+def can_use_lfm3_conv(hidden_states, weight):
+    return (
+        is_triton_available()
+        and hasattr(tl, "gather")  # the decode kernel needs Triton >= 3.3
+        and hidden_states.is_cuda
+        and not torch.is_grad_enabled()
+        and hidden_states.dtype == weight.dtype
+        and hidden_states.dtype in (torch.float16, torch.bfloat16, torch.float32)
+        and weight.shape[-1] == 7
+    )
+
+
+def lfm3_causal_conv1d(hidden_states, weight, bias=None, seq_idx=None):
+    batch, channels, length = hidden_states.shape
+    # the kernels index every tensor by the batch and channel of `hidden_states` and cannot bound the others
+    if weight.shape != (channels, 7) or (bias is not None and bias.shape != (channels,)):
+        raise ValueError(
+            f"`weight` must have shape ({channels}, 7) and `bias` ({channels},), got {tuple(weight.shape)}"
+        )
+    if seq_idx is not None and (
+        seq_idx.ndim != 2 or seq_idx.shape[0] not in (1, batch) or seq_idx.dtype not in (torch.int32, torch.int64)
+    ):
+        raise ValueError(
+            f"`seq_idx` must be an int32 or int64 tensor of shape (1, length) or ({batch}, length), got "
+            f"{seq_idx.dtype} {tuple(seq_idx.shape)}"
+        )
+    if any(t is not None and t.device != hidden_states.device for t in (weight, bias, seq_idx)):
+        raise ValueError(
+            f"`weight`, `bias` and `seq_idx` must be on the device of `hidden_states` ({hidden_states.device})"
+        )
+    output = torch.empty_like(hidden_states, memory_format=torch.contiguous_format)
+    # Triton launches on the current device; the model may live on another one (`device_map`, `.to("cuda:1")`).
+    with torch.cuda.device(hidden_states.device):
+        _prefill[(batch, channels, triton.cdiv(length, 128))](
+            hidden_states,
+            # the kernels index `bias` and `weight` densely
+            weight.contiguous(),
+            bias.contiguous() if bias is not None else None,
+            seq_idx,
+            output,
+            length,
+            channels,
+            *hidden_states.stride(),
+            *(
+                (0 if seq_idx.shape[0] == 1 else seq_idx.stride(0), seq_idx.stride(1))
+                if seq_idx is not None
+                else (0, 0)
+            ),
+            seq_idx.shape[-1] if seq_idx is not None else 0,
+            bias is not None,
+            seq_idx is not None,
+            128,
+            enable_fp_fusion=False,
+        )
+    return output
+
+
+def lfm3_causal_conv1d_update(hidden_states, conv_state, weight, bias=None):
+    batch, channels, length = hidden_states.shape
+    # the kernel addresses one position per row and bounds nothing by the other tensors' shapes
+    if length != 1:
+        raise ValueError(f"`hidden_states` must hold one position per row, got {tuple(hidden_states.shape)}")
+    # the 8-wide tile holds a 6- or 7-wide state (see `positions` in `_decode`)
+    if (
+        conv_state.ndim != 3
+        or conv_state.shape[:2] != (batch, channels)
+        or conv_state.shape[-1] not in (6, 7)
+        or weight.shape != (channels, 7)
+    ):
+        raise ValueError(
+            f"`conv_state` must have shape ({batch}, {channels}, 6 or 7) and `weight` ({channels}, 7), got "
+            f"{tuple(conv_state.shape)} and {tuple(weight.shape)}"
+        )
+    if bias is not None and bias.shape != (channels,):
+        raise ValueError(f"`bias` must have shape ({channels},), got {tuple(bias.shape)}")
+    # the kernel writes the state in place: rows or channels sharing memory (an `expand`ed cache) would race
+    extent = 0
+    for stride, size in sorted(zip(conv_state.stride(), conv_state.shape)):
+        if size > 1 and stride <= extent:
+            raise ValueError(f"`conv_state` must not overlap itself, got strides {conv_state.stride()}")
+        extent += stride * (size - 1)
+    if any(t is not None and t.device != hidden_states.device for t in (conv_state, weight, bias)):
+        raise ValueError(
+            f"`conv_state`, `weight` and `bias` must be on the device of `hidden_states` ({hidden_states.device})"
+        )
+    output = torch.empty_like(hidden_states, memory_format=torch.contiguous_format)
+    with torch.cuda.device(hidden_states.device):
+        _decode[(batch, triton.cdiv(channels, 32))](
+            hidden_states,
+            weight.contiguous(),
+            bias.contiguous() if bias is not None else None,
+            conv_state,
+            output,
+            channels,
+            conv_state.shape[-1],
+            *hidden_states.stride()[:2],
+            *conv_state.stride(),
+            bias is not None,
+            32,
+            enable_fp_fusion=False,
+        )
+    return output

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb8f648b14c0a3cdb2343315dfa89bb843a020fdf01da71deb7d82ee7812d4ff
+size 27642136

modeling_lfm3.py

@@ -0,0 +1,881 @@
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import copy
+from collections.abc import Callable
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+from transformers import initialization as init
+from transformers.activations import ACT2FN
+from transformers.cache_utils import Cache, DynamicCache
+from transformers.configuration_utils import PreTrainedConfig
+from transformers.generation import GenerationMixin
+from transformers.integrations import (
+    use_kernel_forward_from_hub,
+    use_kernel_func_from_hub_with_fallback,
+    use_kernelized_func,
+)
+from transformers.integrations.accelerate import force_accelerate_hooks
+from transformers.masking_utils import (
+    create_causal_mask,
+    create_recurrent_attention_mask,
+    create_sliding_window_causal_mask,
+)
+from transformers.modeling_layers import GradientCheckpointingLayer
+from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
+from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
+from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
+from transformers.processing_utils import Unpack
+from transformers.utils import TransformersKwargs, auto_docstring, can_return_tuple, torch_compilable_check
+from transformers.utils.generic import merge_with_config_defaults
+from transformers.utils.import_utils import is_tracing
+from transformers.utils.output_capturing import capture_outputs
+
+from .configuration_lfm3 import Lfm3Config
+from .lfm3_conv import can_use_lfm3_conv, lfm3_causal_conv1d, lfm3_causal_conv1d_update
+
+
+@use_kernel_forward_from_hub("RMSNorm")
+class Lfm3RMSNorm(nn.Module):
+    def __init__(self, hidden_size, eps: float = 1e-6) -> None:
+        """
+        Lfm3RMSNorm is equivalent to T5LayerNorm
+        """
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+        return self.weight * hidden_states.to(input_dtype)
+
+    def extra_repr(self):
+        return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
+
+
+class Lfm3RotaryEmbedding(nn.Module):
+    """
+    Rotary embedding per attention layer type, like Gemma 3; `conv` layers register no buffers. Partial RoPE keeps the
+    frequency progression of the full head dimension and zeroes the tail instead of shrinking the rotated dimension.
+    """
+
+    def __init__(self, config: Lfm3Config, device=None):
+        super().__init__()
+        self.max_seq_len_cached = config.max_position_embeddings
+        self.original_max_seq_len = config.max_position_embeddings
+        self.config = config
+        self.layer_types = sorted(set(config.layer_types) & set(config.rope_parameters))
+        self.rope_type = {}
+        for layer_type in self.layer_types:
+            rope_params = self.config.rope_parameters[layer_type]
+            self.rope_type[layer_type] = rope_params["rope_type"]
+            curr_inv_freq, curr_attention_scaling = self.compute_rope_parameters(config, device, layer_type=layer_type)
+            setattr(self, f"{layer_type}_inv_freq", nn.Buffer(curr_inv_freq, persistent=False))
+            setattr(self, f"{layer_type}_original_inv_freq", nn.Buffer(curr_inv_freq.clone(), persistent=False))
+            setattr(self, f"{layer_type}_attention_scaling", curr_attention_scaling)
+
+    @staticmethod
+    def compute_default_rope_parameters(
+        config: Lfm3Config, device=None, layer_type: str | None = None, **kwargs
+    ) -> tuple[torch.Tensor, float]:
+        """
+        Inverse frequencies of the default RoPE with LFM3's zero-tail partial rotation: the first
+        `partial_rotary_factor * head_dim // 2` frequencies follow the progression of the full head dimension and the
+        rest are zero, leaving those dimensions unrotated.
+
+        Returns:
+            Tuple of (`torch.Tensor`, `float`): the inverse frequencies and the attention scaling factor (always 1).
+        """
+        base = config.rope_parameters[layer_type]["rope_theta"]
+        partial_rotary_factor = config.rope_parameters[layer_type].get("partial_rotary_factor", 1.0)
+        dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads
+        rope_angles = int(partial_rotary_factor * dim) // 2
+        inv_freq = 1.0 / (base ** (torch.arange(0, 2 * rope_angles, 2, dtype=torch.float) / dim))
+        inv_freq = torch.cat((inv_freq, torch.zeros(dim // 2 - rope_angles, dtype=torch.float)))
+        return inv_freq.to(device), 1.0
+
+    @torch.no_grad()
+    @dynamic_rope_update  # power user: used with advanced RoPE types (e.g. dynamic rope)
+    def forward(self, x, position_ids, layer_type):
+        inv_freq = getattr(self, f"{layer_type}_inv_freq")
+        attention_scaling = getattr(self, f"{layer_type}_attention_scaling")
+
+        freqs = position_ids[..., None].float() * inv_freq.to(device=x.device, dtype=torch.float)
+        emb = torch.cat((freqs, freqs), dim=-1)
+        cos = emb.cos() * attention_scaling
+        sin = emb.sin() * attention_scaling
+
+        return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
+
+    @classmethod
+    def compute_rope_parameters(
+        cls, config: Lfm3Config, device=None, layer_type: str | None = None
+    ) -> tuple[torch.Tensor, float]:
+        """Inverse frequencies of any RoPE type in the zero-tail layout: scaled types are evaluated over the full head
+        dimension and truncated to the rotated prefix."""
+        rope_params = config.rope_parameters[layer_type]
+        if rope_params["rope_type"] == "default":
+            return cls.compute_default_rope_parameters(config, device, layer_type=layer_type)
+        full_config = copy.copy(config)
+        full_config.rope_parameters = {
+            **config.rope_parameters,
+            layer_type: {**rope_params, "partial_rotary_factor": 1.0},
+        }
+        inv_freq, attention_factor = ROPE_INIT_FUNCTIONS[rope_params["rope_type"]](
+            full_config, device, layer_type=layer_type
+        )
+        dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads
+        rope_angles = int(rope_params.get("partial_rotary_factor", 1.0) * dim) // 2
+        inv_freq = torch.cat((inv_freq[:rope_angles], inv_freq.new_zeros(dim // 2 - rope_angles)))
+        return inv_freq, attention_factor
+
+
+class Lfm3MLP(nn.Module):
+    def __init__(self, config: Lfm3Config):
+        super().__init__()
+        self.w1 = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
+        self.w3 = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
+        self.w2 = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
+
+    def forward(self, x):
+        return self.w2(F.silu(self.w1(x)) * self.w3(x))
+
+
+def rotate_half(x):
+    """Rotates half the hidden dims of the input."""
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2 :]
+    return torch.cat((-x2, x1), dim=-1)
+
+
+def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
+    """
+    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
+    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
+    """
+    batch, num_key_value_heads, slen, head_dim = hidden_states.shape
+    if n_rep == 1:
+        return hidden_states
+    hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
+    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
+
+
+def eager_attention_forward(
+    module: nn.Module,
+    query: torch.Tensor,
+    key: torch.Tensor,
+    value: torch.Tensor,
+    attention_mask: torch.Tensor | None,
+    scaling: float,
+    dropout: float = 0.0,
+    **kwargs: Unpack[TransformersKwargs],
+):
+    key_states = repeat_kv(key, module.num_key_value_groups)
+    value_states = repeat_kv(value, module.num_key_value_groups)
+
+    attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling
+    if attention_mask is not None:
+        attn_weights = attn_weights + attention_mask
+
+    attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)
+    attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
+    attn_output = torch.matmul(attn_weights, value_states)
+    attn_output = attn_output.transpose(1, 2).contiguous()
+
+    return attn_output, attn_weights
+
+
+def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1):
+    """Applies RoPE to the query and key tensors in float32, like the reference implementation."""
+    cos = cos.unsqueeze(unsqueeze_dim).float()
+    sin = sin.unsqueeze(unsqueeze_dim).float()
+    q_embed = (q.float() * cos) + (rotate_half(q.float()) * sin)
+    k_embed = (k.float() * cos) + (rotate_half(k.float()) * sin)
+    return q_embed.to(q.dtype), k_embed.to(k.dtype)
+
+
+class Lfm3Attention(nn.Module):
+    """Llama attention with QK RMSNorm, a sigmoid output gate per head, a sliding window on `sliding_attention`
+    layers and RoPE in float32."""
+
+    def __init__(self, config: Lfm3Config, layer_idx: int):
+        super().__init__()
+        self.config = config
+        self.layer_idx = layer_idx
+        self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
+        self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads
+        self.scaling = self.head_dim**-0.5
+        self.is_causal = True
+        self.q_proj = nn.Linear(config.hidden_size, config.num_attention_heads * self.head_dim, bias=False)
+        self.k_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=False)
+        self.v_proj = nn.Linear(config.hidden_size, config.num_key_value_heads * self.head_dim, bias=False)
+        self.o_proj = nn.Linear(config.num_attention_heads * self.head_dim, config.hidden_size, bias=False)
+        self.q_norm = Lfm3RMSNorm(self.head_dim, eps=config.norm_eps)
+        self.k_norm = Lfm3RMSNorm(self.head_dim, eps=config.norm_eps)
+        self.gate_proj = nn.Linear(config.hidden_size, config.num_attention_heads, bias=False)
+        self.sliding_window = config.sliding_window if config.layer_types[layer_idx] == "sliding_attention" else None
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor],
+        attention_mask: torch.Tensor | None,
+        past_key_values: Cache | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple[torch.Tensor, torch.Tensor | None]:
+        input_shape = hidden_states.shape[:-1]
+        hidden_shape = (*input_shape, -1, self.head_dim)
+
+        query_states = self.q_norm(self.q_proj(hidden_states).view(*hidden_shape)).transpose(1, 2)
+        key_states = self.k_norm(self.k_proj(hidden_states).view(*hidden_shape)).transpose(1, 2)
+        value_states = self.v_proj(hidden_states).view(*hidden_shape).transpose(1, 2)
+
+        cos, sin = position_embeddings
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
+
+        if past_key_values is not None:
+            key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx)
+
+        attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(
+            self.config._attn_implementation, eager_attention_forward
+        )
+
+        attn_output, attn_weights = attention_interface(
+            self,
+            query_states,
+            key_states,
+            value_states,
+            attention_mask,
+            dropout=0.0,
+            scaling=self.scaling,
+            sliding_window=self.sliding_window,
+            **kwargs,
+        )
+        gate = torch.sigmoid(self.gate_proj(hidden_states)).unsqueeze(-1)
+        attn_output = (attn_output * gate).reshape(*input_shape, -1).contiguous()
+        attn_output = self.o_proj(attn_output)
+        return attn_output, attn_weights
+
+
+def apply_mask_to_padding_states(hidden_states, attention_mask):
+    """
+    Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/mamba/issues/66
+    """
+    # NOTE: attention mask is a 2D boolean tensor
+    if attention_mask is not None:
+        dtype = hidden_states.dtype
+        hidden_states = (hidden_states * attention_mask[:, :, None]).to(dtype)
+
+    return hidden_states
+
+
+@use_kernel_func_from_hub_with_fallback("causal_conv1d_update", "causal_conv1d")
+def causal_conv1d_update(
+    hidden_states: torch.Tensor,
+    conv_state: torch.Tensor,
+    weight: nn.Parameter,
+    bias: nn.Parameter | None = None,
+    activation: str | None = None,
+):
+    _, hidden_size, seq_len = hidden_states.shape
+    state_len = conv_state.shape[-1]
+
+    hidden_states_new = torch.cat([conv_state, hidden_states], dim=-1).to(weight.dtype)
+    conv_state.copy_(hidden_states_new[:, :, -state_len:])
+    out = F.conv1d(hidden_states_new, weight.unsqueeze(1), bias, padding=0, groups=hidden_size)
+    out = out[:, :, -seq_len:]
+    if activation is not None:
+        out = ACT2FN[activation](out)
+    return out.to(hidden_states.dtype)
+
+
+@use_kernel_func_from_hub_with_fallback("causal_conv1d_fn", "causal_conv1d")
+def causal_conv1d_fn(
+    hidden_states: torch.Tensor,
+    weight: nn.Parameter,
+    bias: nn.Parameter | None = None,
+    activation: str | None = None,
+    **kwargs,
+):
+    _, hidden_size, seq_len = hidden_states.shape
+    padding = weight.shape[-1] - 1
+
+    out = F.conv1d(
+        hidden_states.to(weight.dtype),
+        weight=weight.unsqueeze(1),
+        bias=bias,
+        padding=padding,
+        groups=hidden_size,
+    )[:, :, :seq_len]
+    if activation is not None:
+        out = ACT2FN[activation](out)
+    return out.to(hidden_states.dtype)
+
+
+@use_kernelized_func([causal_conv1d_update, causal_conv1d_fn])
+class Lfm3ShortConv(nn.Module):
+    """LFM2 gated short convolution with packed-document boundaries (`seq_idx`), the padding mask re-applied after a
+    biased `in_proj`, and width-7 Triton inference kernels with a PyTorch fallback."""
+
+    def __init__(
+        self,
+        config: Lfm3Config,
+        layer_idx: int,
+    ):
+        super().__init__()
+        self.config = config
+        self.layer_idx = layer_idx
+        self.conv_kernel_size = config.conv_L_cache
+        self.bias = config.conv_bias
+
+        self.conv = nn.Conv1d(
+            in_channels=config.hidden_size,
+            out_channels=config.hidden_size,
+            kernel_size=self.conv_kernel_size,
+            groups=config.hidden_size,
+            bias=self.bias,
+            padding=self.conv_kernel_size - 1,
+        )
+        self.in_proj = nn.Linear(config.hidden_size, 3 * config.hidden_size, bias=self.bias)
+        self.out_proj = nn.Linear(config.hidden_size, config.hidden_size, bias=self.bias)
+
+        self.layer_type = config.layer_types[layer_idx]
+
+    @force_accelerate_hooks("conv")
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        past_key_values: Cache | None = None,
+        attention_mask: torch.Tensor | None = None,
+        seq_idx: torch.IntTensor | None = None,
+    ):
+        seq_len = hidden_states.shape[1]
+
+        hidden_states = apply_mask_to_padding_states(hidden_states, attention_mask)
+        BCx = self.in_proj(hidden_states).transpose(-1, -2)
+        B, C, x = BCx.chunk(3, dim=-2)
+        hidden_states = B * x
+        if attention_mask is not None:
+            # with `conv_bias` the projection makes the padded positions non-zero again
+            hidden_states = hidden_states * attention_mask[:, None, :].to(hidden_states.dtype)
+
+        use_precomputed_states = past_key_values is not None and past_key_values.has_previous_state(self.layer_idx)
+        use_kernels = 2 <= self.conv_kernel_size <= 4
+        use_wide_kernel = can_use_lfm3_conv(hidden_states, self.conv.weight)
+
+        if use_precomputed_states and seq_len == 1 and not past_key_values.layers[self.layer_idx].record_past:
+            conv_state = past_key_values.layers[self.layer_idx].conv_states[0]
+            if use_wide_kernel and conv_state.shape[-1] in (6, 7):
+                hidden_states = lfm3_causal_conv1d_update(
+                    hidden_states, conv_state, self.conv.weight.squeeze(1), self.conv.bias
+                )
+            elif use_kernels:
+                hidden_states = causal_conv1d_update(
+                    hidden_states, conv_state, self.conv.weight.squeeze(1), self.conv.bias
+                )
+            else:
+                hidden_states = torch.cat([conv_state, hidden_states], dim=-1)
+                conv_state.copy_(hidden_states[:, :, -conv_state.shape[-1] :])
+                hidden_states = self._conv(hidden_states)[:, :, -seq_len:]
+        else:
+            if past_key_values is not None and seq_idx is not None:
+                # `seq_idx` is only set on an empty cache, and the decode steps read the cached window without it:
+                # only the last document (after any positional left padding) may enter the window
+                last_document = (seq_idx == seq_idx[:, -1:])[:, None].to(hidden_states.dtype)
+                past_key_values.update_conv_state(
+                    hidden_states * last_document, self.layer_idx, conv_kernel_size=self.conv_kernel_size
+                )
+            elif past_key_values is not None:
+                hidden_states = past_key_values.update_conv_state(
+                    hidden_states, self.layer_idx, conv_kernel_size=self.conv_kernel_size
+                )
+
+            if use_wide_kernel:
+                hidden_states = lfm3_causal_conv1d(hidden_states, self.conv.weight.squeeze(1), self.conv.bias, seq_idx)
+            elif seq_idx is not None:
+                hidden_states = self._segmented_conv(hidden_states, seq_idx)
+            elif use_kernels:
+                hidden_states = causal_conv1d_fn(hidden_states, self.conv.weight.squeeze(1), self.conv.bias)
+            else:
+                hidden_states = self._conv(hidden_states)
+
+            # Drop the additional previous states
+            if past_key_values is not None:
+                hidden_states = hidden_states[:, :, -seq_len:]
+
+        y = C * hidden_states
+        y = y.transpose(-1, -2).contiguous()
+        y = self.out_proj(y)
+        return y
+
+    def _conv(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        """Causal depthwise conv in torch: the `causal_conv1d` kernels only accept widths 2 to 4."""
+        conv_out = self.conv(hidden_states.to(self.conv.weight.dtype))[..., : hidden_states.shape[-1]]
+        return conv_out.to(hidden_states.dtype)
+
+    def _segmented_conv(self, hidden_states: torch.Tensor, seq_idx: torch.Tensor) -> torch.Tensor:
+        """Causal depthwise conv that only reads positions sharing the current token's document index."""
+        kernel_size = self.conv_kernel_size
+        seq_idx = F.pad(seq_idx, (hidden_states.shape[-1] - seq_idx.shape[-1] + kernel_size - 1, 0), value=-1)
+        windows = F.pad(hidden_states, (kernel_size - 1, 0)).unfold(-1, kernel_size, 1)
+        seq_idx_windows = seq_idx.unfold(-1, kernel_size, 1)
+        same_document = (seq_idx_windows == seq_idx_windows[..., -1:]).unsqueeze(1)
+        conv_out = torch.einsum("bdtk,dk->bdt", windows * same_document.to(windows.dtype), self.conv.weight.squeeze(1))
+        if self.bias:
+            conv_out = conv_out + self.conv.bias[None, :, None]
+        return conv_out.to(hidden_states.dtype)
+
+
+class Lfm3DecoderLayer(GradientCheckpointingLayer):
+    def __init__(self, config: Lfm3Config, layer_idx: int):
+        super().__init__()
+        self.is_attention_layer = config.layer_types[layer_idx] in config._attention_layer_types
+        self.use_sandwich_norm = config.use_sandwich_norm
+
+        if self.is_attention_layer:
+            self.self_attn = Lfm3Attention(config, layer_idx)
+        else:
+            self.conv = Lfm3ShortConv(config, layer_idx)
+        self.feed_forward = Lfm3MLP(config)
+        self.operator_norm = Lfm3RMSNorm(config.hidden_size, eps=config.norm_eps)
+        self.ffn_norm = Lfm3RMSNorm(config.hidden_size, eps=config.norm_eps)
+        if self.use_sandwich_norm:
+            self.post_operator_norm = Lfm3RMSNorm(config.hidden_size, eps=config.norm_eps)
+            self.post_ffn_norm = Lfm3RMSNorm(config.hidden_size, eps=config.norm_eps)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        seq_idx: torch.IntTensor | None = None,
+        **kwargs,
+    ) -> torch.Tensor:
+        residual = hidden_states
+        if self.is_attention_layer:
+            hidden_states, _ = self.self_attn(
+                hidden_states=self.operator_norm(hidden_states),
+                position_embeddings=position_embeddings,
+                attention_mask=attention_mask,
+                position_ids=position_ids,
+                past_key_values=past_key_values,
+                **kwargs,
+            )
+        else:
+            hidden_states = self.conv(
+                hidden_states=self.operator_norm(hidden_states),
+                past_key_values=past_key_values,
+                attention_mask=attention_mask,
+                seq_idx=seq_idx,
+            )
+        if self.use_sandwich_norm:
+            hidden_states = self.post_operator_norm(hidden_states)
+        hidden_states = hidden_states + residual
+
+        ffn_out = self.feed_forward(self.ffn_norm(hidden_states))
+        if self.use_sandwich_norm:
+            ffn_out = self.post_ffn_norm(ffn_out)
+        hidden_states = hidden_states + ffn_out
+
+        return hidden_states
+
+
+def create_masks_by_layer_type(
+    config: PreTrainedConfig,
+    inputs_embeds: torch.Tensor,
+    attention_mask: torch.Tensor | None,
+    past_key_values: Cache | None,
+    position_ids: torch.Tensor | None = None,
+) -> dict:
+    """The cache sizes each mask from the layer it is asked about, and the generic mask utilities ask about the first
+    non-sliding layer, which for a leading `conv` layer redirects to the first attention layer of any type: name the
+    first layer of each mask's own type instead."""
+    config = config.get_text_config()
+    mask_kwargs = {
+        "config": config,
+        "inputs_embeds": inputs_embeds,
+        "attention_mask": attention_mask,
+        "past_key_values": past_key_values,
+        "position_ids": position_ids,
+    }
+    layer_types, causal_mask_mapping = config.layer_types, {}
+    if "full_attention" in layer_types:
+        layer_idx = layer_types.index("full_attention")
+        causal_mask_mapping["full_attention"] = create_causal_mask(**mask_kwargs, layer_idx=layer_idx)
+    if "sliding_attention" in layer_types:
+        layer_idx = layer_types.index("sliding_attention")
+        causal_mask_mapping["sliding_attention"] = create_sliding_window_causal_mask(
+            **mask_kwargs, layer_idx=layer_idx
+        )
+    if "conv" in layer_types:
+        causal_mask_mapping["conv"] = create_recurrent_attention_mask(**mask_kwargs)
+    return causal_mask_mapping
+
+
+@auto_docstring
+class Lfm3PreTrainedModel(PreTrainedModel):
+    config: Lfm3Config
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["Lfm3DecoderLayer"]
+    _skip_keys_device_placement = ["past_key_values"]
+    _supports_flash_attn = True
+    _supports_sdpa = True
+    _supports_flex_attn = True
+
+    _can_compile_fullgraph = True
+    _supports_attention_backend = True
+    _can_record_outputs = {
+        "hidden_states": Lfm3DecoderLayer,
+        "attentions": Lfm3Attention,
+    }
+
+    @torch.no_grad()
+    def _init_weights(self, module):
+        super()._init_weights(module)
+        if isinstance(module, Lfm3RotaryEmbedding):
+            for layer_type in module.layer_types:
+                curr_inv_freq, _ = module.compute_rope_parameters(module.config, layer_type=layer_type)
+                init.copy_(getattr(module, f"{layer_type}_inv_freq"), curr_inv_freq)
+                init.copy_(getattr(module, f"{layer_type}_original_inv_freq"), curr_inv_freq)
+
+    @staticmethod
+    def create_masks_for_generate(
+        config: PreTrainedConfig,
+        inputs_embeds: torch.Tensor,
+        attention_mask: torch.Tensor | None,
+        past_key_values: Cache | None,
+        position_ids: torch.Tensor | None = None,
+        **kwargs,
+    ) -> dict:
+        return create_masks_by_layer_type(config, inputs_embeds, attention_mask, past_key_values, position_ids)
+
+
+def packed_under_padding(position_ids: torch.Tensor, padding_mask: torch.Tensor, filled) -> torch.Tensor:
+    """Padding repeats position 0, so packing is a restart between two kept tokens, or a kept position 0 on a filled
+    cache whose row holds a kept token; attention under a padding mask cannot isolate it."""
+    restarts = torch.diff(position_ids, prepend=position_ids[:, :1] - 1, dim=-1) != 1
+    valid = padding_mask[:, -restarts.shape[-1] :].bool()
+    packed = restarts[:, 1:] & valid[:, 1:] & valid[:, :-1]
+    if filled:
+        # a row whose cached tokens are all padding (chunked prefill of a left-padded batch) has no document to pack
+        # against; a mask covering only the chunk keeps the strict rule
+        cached = padding_mask[:, : padding_mask.shape[-1] - restarts.shape[-1]]
+        cached_valid = cached.bool().any(-1, keepdim=True) if cached.shape[-1] else torch.ones_like(valid[:, :1])
+        packed = packed | (valid[:, :1] & (position_ids[:, :1] == 0) & cached_valid)
+    return packed
+
+
+@auto_docstring
+class Lfm3Model(Lfm3PreTrainedModel):
+    """LFM2 model with sliding-attention layers as a third layer type: the mask and the rotary embedding are built per
+    layer type, and packed documents (`position_ids` restarting) reset the short-conv window through `seq_idx`."""
+
+    def __init__(self, config: Lfm3Config):
+        super().__init__(config)
+        self.padding_idx = config.pad_token_id
+        self.vocab_size = config.vocab_size
+
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
+        self.layers = nn.ModuleList(
+            [Lfm3DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
+        )
+        self.rotary_emb = Lfm3RotaryEmbedding(config=config)
+        self.gradient_checkpointing = False
+        self.embedding_norm = Lfm3RMSNorm(config.hidden_size, eps=config.norm_eps)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @merge_with_config_defaults
+    @capture_outputs
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        use_cache: bool | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> BaseModelOutputWithPast:
+        r"""
+        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)` or `dict`, *optional*):
+            Mask to avoid performing attention on padding token indices (1 keeps a token, 0 masks it), or the mask
+            dict of `create_masks_for_generate`. A mask dict is used as given: `generate` refuses packed
+            `position_ids` before its prefill.
+        """
+        if (input_ids is None) ^ (inputs_embeds is not None):
+            raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
+        # `seq_idx` is a shared kwarg (`DataCollatorWithFlattening(return_seq_idx=True)`, read by Lfm2); LFM3 derives
+        # the documents from the restarting `position_ids` that come with it
+        if kwargs.pop("seq_idx", None) is not None and position_ids is None:
+            raise ValueError("Pass packed documents as `position_ids` restarting at 0; LFM3 does not read `seq_idx`")
+
+        if inputs_embeds is None:
+            inputs_embeds = self.embed_tokens(input_ids)
+
+        if use_cache and past_key_values is None:
+            past_key_values = DynamicCache(config=self.config)
+        filled = past_key_values is not None and (
+            past_key_values.get_seq_length() > 0
+            if any(t != "conv" for t in self.config.layer_types)
+            else past_key_values.has_previous_state()
+        )
+        packed_cache = getattr(past_key_values, "lfm3_packed", None)
+        if isinstance(packed_cache, torch.Tensor) or packed_cache:
+            # Reset layers have no history to mix. Ordinary caches carry a Python `False` and skip this, so the decode
+            # steps do no tensor work here; only a compiled packed prefill leaves a tensor flag, and `filled` stays
+            # tensor-valued for static caches.
+            packed_message = (
+                "A cache filled by a packed-document forward cannot be continued: its attention spans the documents"
+            )
+            packed_cache = packed_cache & filled if isinstance(packed_cache, torch.Tensor) else filled
+            if isinstance(packed_cache, torch.Tensor):
+                torch_compilable_check(~packed_cache, packed_message)
+                if not is_tracing(packed_cache):
+                    past_key_values.lfm3_packed = bool(packed_cache)
+            elif packed_cache:
+                raise ValueError(packed_message)
+            else:
+                past_key_values.lfm3_packed = False
+
+        # The mask utilities detect packing only with `attention_mask=None`, so a multi-token forward with explicit
+        # positions drops the all-ones mask tokenizers emit (a data-dependent check, hence not when traced).
+        explicit_positions = position_ids is not None and inputs_embeds.shape[1] > 1
+        if (
+            explicit_positions
+            and isinstance(attention_mask, torch.Tensor)
+            and attention_mask.ndim == 2
+            and not is_tracing(position_ids)
+            and bool(attention_mask.all())
+        ):
+            attention_mask = None
+        seq_idx = None
+        if position_ids is None:
+            past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
+            position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens
+            position_ids = position_ids.unsqueeze(0)
+        elif explicit_positions and not isinstance(attention_mask, dict):
+            # a mask dict comes from `generate`, which refused packing before its prefill
+            if isinstance(attention_mask, torch.Tensor) and attention_mask.ndim == 2:
+                # data-dependent: a traced forward skips it (`generate` refuses packing eagerly before its prefill)
+                if not is_tracing(position_ids):
+                    torch_compilable_check(
+                        ~packed_under_padding(position_ids, attention_mask, filled),
+                        "Packed documents (`position_ids` restarting) cannot be combined with an attention mask: "
+                        "attention would not isolate the documents. Pass `attention_mask=None` without a static cache.",
+                    )
+            else:
+                restarts = torch.diff(position_ids, prepend=position_ids[:, :1] - 1, dim=-1) != 1
+                document_ids = restarts.cumsum(-1)
+                if is_tracing(position_ids) or bool((document_ids[..., -1] > 0).any()):
+                    seq_idx = document_ids.to(torch.int32)
+                # Packed documents are prefilled in one forward on an empty cache (cached tokens would join the first
+                # document), and their cache is never continued
+                if filled:
+                    torch_compilable_check(
+                        (~restarts[:, 1:]).all() & (position_ids[:, 0] != 0).all(),
+                        "Packed documents must be prefilled in one forward; the cache already holds tokens",
+                    )
+                    # Ordinary continuation must retain the convolution window from the cached document.
+                    seq_idx = None
+                if attention_mask is None:
+                    # without a mask only the cacheless packed-sequence mask isolates repeated zero positions (left
+                    # padding): a cache-sized mask and every later decode step would attend them, so they count too
+                    packed = (document_ids[..., -1] > 0).any()
+                else:
+                    # a prepared 4D mask is the caller's and carries the padding (repeated zero positions), but may
+                    # or may not isolate documents, so the cache records the packing and is never continued
+                    steps = torch.diff(position_ids, dim=-1)
+                    packed = ((steps != 1) & ~((steps == 0) & (position_ids[:, :-1] == 0))).any()
+                if past_key_values is not None:
+                    # resolved here, where the prefill already syncs, so the decode steps carry a Python flag: no
+                    # per-step host sync, data-dependent op or CUDA-graph output on the cache
+                    past_key_values.lfm3_packed = packed if is_tracing(position_ids) else bool(packed)
+
+        if not isinstance(causal_mask_mapping := attention_mask, dict):
+            mask_kwargs = {
+                "config": self.config,
+                "inputs_embeds": inputs_embeds,
+                "attention_mask": attention_mask,
+                "past_key_values": past_key_values,
+                "position_ids": position_ids,
+            }
+            # The mask utilities only isolate packed documents without a cache, and an empty cache adds nothing.
+            layer_types = self.config.layer_types
+            if seq_idx is not None and past_key_values is not None:
+                attention_layer = next((i for i, t in enumerate(layer_types) if t != "conv"), None)
+                if attention_layer is not None and past_key_values.get_seq_length(attention_layer) == 0:
+                    if past_key_values.is_compileable:
+                        # static layers always return their full length, so the masks are sized by the cache, which
+                        # isolates neither documents nor positional-only left padding
+                        torch_compilable_check(
+                            ~packed,
+                            "Packed documents, or left padding without an attention mask, cannot be prefilled into a "
+                            "static cache: its masks are sized by the cache. Pass the padding as `attention_mask`, or "
+                            "use a `DynamicCache` for packed documents",
+                        )
+                    else:
+                        mask_kwargs["past_key_values"] = None
+            causal_mask_mapping = create_masks_by_layer_type(**mask_kwargs)
+
+        hidden_states = inputs_embeds
+        position_embeddings = {
+            layer_type: self.rotary_emb(hidden_states, position_ids=position_ids, layer_type=layer_type)
+            for layer_type in self.rotary_emb.layer_types
+        }
+
+        for i, decoder_layer in enumerate(self.layers[: self.config.num_hidden_layers]):
+            layer_type = self.config.layer_types[i]
+            hidden_states = decoder_layer(
+                hidden_states,
+                attention_mask=causal_mask_mapping[layer_type],
+                position_embeddings=position_embeddings.get(layer_type),
+                position_ids=position_ids,
+                past_key_values=past_key_values,
+                seq_idx=seq_idx,
+                **kwargs,
+            )
+
+        hidden_states = self.embedding_norm(hidden_states)
+
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            past_key_values=past_key_values,
+        )
+
+
+def reject_packed_generation(config: PreTrainedConfig, input_ids, generation_config, model_kwargs, is_first_iteration):
+    """`generate` decodes from the cache its prefill fills, and no mask it builds isolates packed documents
+    (`position_ids` restarting): the prompt is refused before the prefill, one forward or every chunk, writes to it."""
+    position_ids, attention_mask = model_kwargs.get("position_ids"), model_kwargs.get("attention_mask")
+    cache = model_kwargs.get("past_key_values")
+    if cache is None or position_ids is None or position_ids.ndim != 2:
+        return
+    if attention_mask is not None and attention_mask.ndim != 2:
+        return
+    has_attention = any(t != "conv" for t in config.get_text_config().layer_types)
+    filled = bool(cache.get_seq_length() > 0 if has_attention else cache.has_previous_state())
+    inputs_embeds = model_kwargs.get("inputs_embeds") if is_first_iteration else None
+    length = input_ids.shape[1] if inputs_embeds is None else inputs_embeds.shape[1]
+    if generation_config.prefill_chunk_size is not None:
+        # the chunks run from the start of the prompt
+        position_ids = position_ids[:, :length]
+    else:
+        if filled and has_attention:
+            # the slice `_prefill` feeds: a prompt that spans the cache is cut to its uncached part
+            mask_length = getattr(generation_config, "_mask_length", -1)
+            if attention_mask is not None:
+                mask_length = attention_mask.shape[1]
+            if inputs_embeds is not None or mask_length == length:
+                length -= int(cache.get_seq_length())
+        position_ids = position_ids[:, position_ids.shape[1] - length :]
+    if attention_mask is None:
+        attention_mask = torch.ones_like(position_ids)
+    if bool(packed_under_padding(position_ids, attention_mask, filled).any()):
+        raise ValueError(
+            "`generate` cannot decode packed documents (`position_ids` restarting, or starting at 0 on a cache that "
+            "holds tokens): attention would span them. Generate each document in its own row of a left-padded batch."
+        )
+
+
+@auto_docstring
+class Lfm3ForCausalLM(Lfm3PreTrainedModel, GenerationMixin):
+    _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"}
+    _tp_plan = {"lm_head": "colwise_gather_output"}
+    _pp_plan = {"lm_head": (["hidden_states"], ["logits"])}
+    _fsdp_plan = {"lm_head": "keep_full_weight"}
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.model = Lfm3Model(config)
+        self.vocab_size = config.vocab_size
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        labels: torch.LongTensor | None = None,
+        use_cache: bool | None = None,
+        logits_to_keep: int | torch.Tensor = 0,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> CausalLMOutputWithPast:
+        r"""
+        Example:
+
+        ```python
+        >>> from transformers import AutoTokenizer, Lfm3ForCausalLM
+
+        >>> model = Lfm3ForCausalLM.from_pretrained("LiquidAI/LFM3-1.7B")
+        >>> tokenizer = AutoTokenizer.from_pretrained("LiquidAI/LFM3-1.7B")
+
+        >>> inputs = tokenizer.apply_chat_template(
+        ...     [{"role": "user", "content": "What is C. elegans?"}], add_generation_prompt=True, return_tensors="pt"
+        ... )
+        >>> generate_ids = model.generate(**inputs, max_new_tokens=64)
+        >>> tokenizer.decode(generate_ids[0, inputs["input_ids"].shape[1] :], skip_special_tokens=True)
+        ```"""
+        outputs: BaseModelOutputWithPast = self.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            **kwargs,
+        )
+
+        hidden_states = outputs.last_hidden_state
+        # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
+        slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
+        logits = self.lm_head(hidden_states[:, slice_indices, :])
+
+        loss = None
+        if labels is not None:
+            loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs)
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+    def _prefill(self, input_ids, generation_config, model_kwargs, is_first_iteration=True):
+        # every prefill passes here once before its first forward: a check per chunk would leave the earlier chunks
+        # in the cache, and a dynamic cache would only refuse packing at the first decode step
+        reject_packed_generation(self.config, input_ids, generation_config, model_kwargs, is_first_iteration)
+        return super()._prefill(input_ids, generation_config, model_kwargs, is_first_iteration=is_first_iteration)
+
+
+__all__ = ["Lfm3ForCausalLM", "Lfm3Model", "Lfm3PreTrainedModel"]

modeling_lfm3_vl.py

@@ -0,0 +1,965 @@
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from collections.abc import Callable
+from dataclasses import dataclass
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+from transformers.activations import ACT2FN
+from transformers.cache_utils import Cache
+from transformers.configuration_utils import PreTrainedConfig
+from transformers.generation import GenerationMixin
+from transformers.integrations import use_kernel_forward_from_hub
+from transformers.masking_utils import (
+    create_causal_mask,
+    create_recurrent_attention_mask,
+    create_sliding_window_causal_mask,
+)
+from transformers.modeling_layers import GradientCheckpointingLayer
+from transformers.modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling, CausalLMOutputWithPast
+from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
+from transformers.processing_utils import Unpack
+from transformers.utils import (
+    TransformersKwargs,
+    auto_docstring,
+    can_return_tuple,
+    torch_compilable_check,
+)
+from transformers.utils.generic import (
+    accepts_precomputed_kwargs,
+    is_flash_attention_requested,
+    maybe_autocast,
+    merge_with_config_defaults,
+)
+from transformers.utils.output_capturing import OutputRecorder, capture_outputs
+from transformers.vision_utils import get_max_seqlen, get_vision_attention_seqlens, get_vision_position_ids
+
+from .configuration_lfm3_vl import Lfm3VlConfig, Lfm3VlVisionConfig
+from .modeling_lfm3 import Lfm3Model
+
+
+@use_kernel_forward_from_hub("RMSNorm")
+class Lfm3VlRMSNorm(nn.Module):
+    def __init__(self, hidden_size, eps: float = 1e-6) -> None:
+        """
+        Lfm3VlRMSNorm is equivalent to T5LayerNorm
+        """
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+        return self.weight * hidden_states.to(input_dtype)
+
+    def extra_repr(self):
+        return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
+
+
+class Lfm3VlMLP(nn.Module):
+    def __init__(self, config: Lfm3VlConfig):
+        super().__init__()
+        self.w1 = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
+        self.w3 = nn.Linear(config.hidden_size, config.intermediate_size, bias=False)
+        self.w2 = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
+
+    def forward(self, x):
+        return self.w2(F.silu(self.w1(x)) * self.w3(x))
+
+
+class Lfm3VlVisionRotaryEmbedding(nn.Module):
+    """
+    Simple axial 2D rope with same freqs used for H and W grids. The freqs are
+    pre-computed using `head-dim//4` which is later used to concat H and W positions.
+    The final angles rotate over the whole head dim, no partial rotation involved.
+    """
+
+    def __init__(self, config: Lfm3VlVisionConfig):
+        super().__init__()
+        self.config = config
+
+        self.rope_type = self.config.rope_parameters["rope_type"]
+        rope_init_fn: Callable = self.compute_axial_rope_parameters
+        if self.rope_type != "axial":
+            raise ValueError(f"{self.__class__.__name__} supports only axial rope, but requested {self.rope_type}")
+        inv_freq, self.attention_scaling = rope_init_fn(self.config)
+
+        self.inv_freq = nn.Buffer(inv_freq, persistent=False)
+        self.original_inv_freq = nn.Buffer(inv_freq.clone(), persistent=False)
+
+    @staticmethod
+    def compute_axial_rope_parameters(config: Lfm3VlVisionConfig, **kwargs) -> tuple[torch.Tensor, float]:
+        """Inverse frequencies of the axial RoPE over half the head dimension per axis, with an attention scaling of 1."""
+        base = config.rope_parameters["rope_theta"]
+        spatial_dim = config.hidden_size // config.num_attention_heads // 2
+        inv_freq = 1.0 / (base ** (torch.arange(0, spatial_dim, 2, dtype=torch.float) / spatial_dim))
+        return inv_freq, 1.0
+
+    @torch.no_grad()
+    def forward(self, x, position_ids):
+        # position_ids: (2, N) — row 0 = h coords, row 1 = w coords
+        position_ids_expanded = position_ids[..., None].float()
+        device_type = x.device.type if isinstance(x.device.type, str) else "cpu"
+        with maybe_autocast(device_type=device_type, enabled=False):
+            freqs = position_ids_expanded * self.inv_freq.float()
+            cos = freqs.cos() * self.attention_scaling
+            sin = freqs.sin() * self.attention_scaling
+
+        cos = self.recomposition_frequencies(cos)
+        sin = self.recomposition_frequencies(sin)
+        return cos, sin
+
+    def recomposition_frequencies(self, freq):
+        """
+        Recompose the frequencies into the final spatial layout used per each grid.
+        """
+        freq_h, freq_w = freq[:, 0], freq[:, 1]
+        freq_hw = torch.cat([freq_h, freq_w], dim=-1)
+        return torch.cat([freq_hw, freq_hw], dim=-1)
+
+
+def rotate_half(x):
+    """Rotates half the hidden dims of the input."""
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2 :]
+    return torch.cat((-x2, x1), dim=-1)
+
+
+def apply_rotary_pos_emb_vision(
+    q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor
+) -> tuple[torch.Tensor, torch.Tensor]:
+    orig_q_dtype = q.dtype
+    orig_k_dtype = k.dtype
+    q, k = q.float(), k.float()
+    cos, sin = cos.unsqueeze(-2).float(), sin.unsqueeze(-2).float()
+    q_embed = (q * cos) + (rotate_half(q) * sin)
+    k_embed = (k * cos) + (rotate_half(k) * sin)
+    q_embed = q_embed.to(orig_q_dtype)
+    k_embed = k_embed.to(orig_k_dtype)
+    return q_embed, k_embed
+
+
+def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
+    """
+    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
+    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
+    """
+    batch, num_key_value_heads, slen, head_dim = hidden_states.shape
+    if n_rep == 1:
+        return hidden_states
+    hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
+    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
+
+
+def eager_attention_forward(
+    module: nn.Module,
+    query: torch.Tensor,
+    key: torch.Tensor,
+    value: torch.Tensor,
+    attention_mask: torch.Tensor | None,
+    scaling: float,
+    dropout: float = 0.0,
+    **kwargs: Unpack[TransformersKwargs],
+):
+    key_states = repeat_kv(key, module.num_key_value_groups)
+    value_states = repeat_kv(value, module.num_key_value_groups)
+
+    attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling
+    if attention_mask is not None:
+        attn_weights = attn_weights + attention_mask
+
+    attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)
+    attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
+    attn_output = torch.matmul(attn_weights, value_states)
+    attn_output = attn_output.transpose(1, 2).contiguous()
+
+    return attn_output, attn_weights
+
+
+class Lfm3VlVisionAttention(nn.Module):
+    """Bidirectional grouped-query attention over the patches of one frame, with RMSNorm on the query and key heads."""
+
+    def __init__(self, config: Lfm3VlVisionConfig):
+        super().__init__()
+        self.config = config
+        self.num_heads = config.num_attention_heads
+        self.num_key_value_heads = config.num_key_value_heads
+        self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads
+        self.head_dim = config.hidden_size // config.num_attention_heads
+        self.scaling = self.head_dim**-0.5
+        self.attention_dropout = 0.0
+        self.is_causal = False
+        self.q_proj = nn.Linear(config.hidden_size, self.num_heads * self.head_dim, bias=False)
+        self.k_proj = nn.Linear(config.hidden_size, self.num_key_value_heads * self.head_dim, bias=False)
+        self.v_proj = nn.Linear(config.hidden_size, self.num_key_value_heads * self.head_dim, bias=False)
+        self.out_proj = nn.Linear(self.num_heads * self.head_dim, config.hidden_size, bias=False)
+        self.q_layernorm = Lfm3VlRMSNorm(self.head_dim, eps=config.norm_eps)
+        self.k_layernorm = Lfm3VlRMSNorm(self.head_dim, eps=config.norm_eps)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        cu_seqlens: torch.Tensor,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor],
+        max_seqlen: int | None = None,
+        frame_shapes: list[list[int]] | None = None,
+        **kwargs,
+    ) -> torch.Tensor:
+        seq_length = hidden_states.shape[0]
+        query_states = self.q_layernorm(self.q_proj(hidden_states).view(seq_length, self.num_heads, self.head_dim))
+        key_states = self.k_layernorm(
+            self.k_proj(hidden_states).view(seq_length, self.num_key_value_heads, self.head_dim)
+        )
+        value_states = self.v_proj(hidden_states).view(seq_length, self.num_key_value_heads, self.head_dim)
+
+        cos, sin = position_embeddings
+        query_states, key_states = apply_rotary_pos_emb_vision(query_states, key_states, cos, sin)
+
+        query_states = query_states.transpose(0, 1).unsqueeze(0)
+        key_states = key_states.transpose(0, 1).unsqueeze(0)
+        value_states = value_states.transpose(0, 1).unsqueeze(0)
+
+        attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(
+            self.config._attn_implementation, eager_attention_forward
+        )
+
+        if is_flash_attention_requested(self.config):
+            max_seqlen = get_max_seqlen(cu_seqlens, self.config, kwargs={"max_seqlen": max_seqlen})
+            attn_output, attn_weights = attention_interface(
+                self,
+                query_states,
+                key_states,
+                value_states,
+                attention_mask=None,
+                scaling=self.scaling,
+                dropout=0.0,
+                cu_seq_lens_q=cu_seqlens,
+                cu_seq_lens_k=cu_seqlens,
+                max_length_q=max_seqlen,
+                max_length_k=max_seqlen,
+                is_causal=False,
+                **kwargs,
+            )
+        else:
+            # Other implementations: frames of one shape (every video, equal-size images) attend as one batch, mixed
+            # shapes frame by frame. `frame_shapes` gives the lengths without a device sync.
+            if frame_shapes is not None:
+                lengths = [height * width for height, width in frame_shapes]
+            else:
+                lengths = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
+            if len(set(lengths)) == 1:
+                query_states, key_states, value_states = (
+                    tensor.reshape(-1, len(lengths), lengths[0], self.head_dim).transpose(0, 1)
+                    for tensor in (query_states, key_states, value_states)
+                )
+                attn_output, attn_weights = attention_interface(
+                    self,
+                    query_states,
+                    key_states,
+                    value_states,
+                    attention_mask=None,
+                    scaling=self.scaling,
+                    dropout=0.0,
+                    is_causal=False,
+                    **kwargs,
+                )
+            else:
+                splits = [torch.split(tensor, lengths, dim=2) for tensor in (query_states, key_states, value_states)]
+                attn_outputs, attn_weights = [], []
+                for q, k, v in zip(*splits):
+                    output, weights = attention_interface(
+                        self,
+                        q,
+                        k,
+                        v,
+                        attention_mask=None,
+                        scaling=self.scaling,
+                        dropout=0.0,
+                        is_causal=False,
+                        **kwargs,
+                    )
+                    attn_outputs.append(output)
+                    attn_weights.append(weights)
+                attn_output = torch.cat(attn_outputs, dim=1)
+                # frames of different shapes have differently sized maps: one entry per frame, or None under sdpa
+                attn_weights = None if attn_weights[0] is None else tuple(attn_weights)
+
+        attn_output = attn_output.reshape(seq_length, -1).contiguous()
+        attn_output = self.out_proj(attn_output)
+        return attn_output, attn_weights
+
+
+class Lfm3VlVisionConv2d(nn.Module):
+    """Gated depthwise 2D convolution over the patch grid of each frame."""
+
+    def __init__(self, config: Lfm3VlVisionConfig):
+        super().__init__()
+        self.spatial_merge_size = config.spatial_merge_size
+        self.in_proj = nn.Linear(config.hidden_size, 3 * config.hidden_size, bias=False)
+        self.spatial_conv = nn.Conv2d(
+            config.hidden_size,
+            config.hidden_size,
+            kernel_size=config.conv_kernel_size,
+            padding=config.conv_kernel_size // 2,
+            groups=config.hidden_size,
+            bias=False,
+        )
+        self.out_proj = nn.Linear(config.hidden_size, config.hidden_size, bias=False)
+
+    def forward(self, hidden_states: torch.Tensor, frame_shapes: list[list[int]]):
+        B, C, x = self.in_proj(hidden_states).chunk(3, dim=-1)
+        gated = B * x
+        merge = self.spatial_merge_size
+        num_frames = len(frame_shapes)
+        if len(set(map(tuple, frame_shapes))) == 1:
+            # frames of one shape (every video, equal-size images) convolve as one batch: merge-block order -> raster
+            # grid, convolve, back
+            height, width = frame_shapes[0]
+            frames = gated.view(num_frames, height // merge, width // merge, merge, merge, -1)
+            frames = frames.permute(0, 5, 1, 3, 2, 4).reshape(num_frames, -1, height, width)
+            frames = self.spatial_conv(frames)
+            frames = frames.view(num_frames, -1, height // merge, merge, width // merge, merge).permute(
+                0, 2, 4, 3, 5, 1
+            )
+            conv_output = frames.reshape(gated.shape[0], -1)
+        else:
+            conv_outputs = []
+            lengths = [height * width for height, width in frame_shapes]
+            for frame, (height, width) in zip(torch.split(gated, lengths, dim=0), frame_shapes):
+                frame = frame.view(height // merge, width // merge, merge, merge, -1)
+                frame = frame.permute(4, 0, 2, 1, 3).reshape(1, -1, height, width)
+                frame = self.spatial_conv(frame)
+                frame = frame.view(-1, height // merge, merge, width // merge, merge).permute(1, 3, 2, 4, 0)
+                conv_outputs.append(frame.reshape(height * width, -1))
+            conv_output = torch.cat(conv_outputs, dim=0)
+        return self.out_proj(C * conv_output)
+
+
+class Lfm3VlVisionLayer(GradientCheckpointingLayer):
+    def __init__(self, config: Lfm3VlVisionConfig, layer_idx: int):
+        super().__init__()
+        self.is_attention_layer = config.layer_types[layer_idx] == "full_attention"
+        if self.is_attention_layer:
+            self.self_attn = Lfm3VlVisionAttention(config)
+        else:
+            self.conv = Lfm3VlVisionConv2d(config)
+        self.feed_forward = Lfm3VlMLP(config)
+        self.operator_norm = Lfm3VlRMSNorm(config.hidden_size, eps=config.norm_eps)
+        self.ffn_norm = Lfm3VlRMSNorm(config.hidden_size, eps=config.norm_eps)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        cu_seqlens: torch.Tensor,
+        frame_shapes: list[list[int]],
+        position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
+        **kwargs,
+    ) -> torch.Tensor:
+        if self.is_attention_layer:
+            operator_output, _ = self.self_attn(
+                self.operator_norm(hidden_states),
+                cu_seqlens=cu_seqlens,
+                position_embeddings=position_embeddings,
+                frame_shapes=frame_shapes,
+                **kwargs,
+            )
+        else:
+            operator_output = self.conv(self.operator_norm(hidden_states), frame_shapes)
+        hidden_states = hidden_states + operator_output
+        hidden_states = hidden_states + self.feed_forward(self.ffn_norm(hidden_states))
+        return hidden_states
+
+
+def create_masks_by_layer_type(
+    config: PreTrainedConfig,
+    inputs_embeds: torch.Tensor,
+    attention_mask: torch.Tensor | None,
+    past_key_values: Cache | None,
+    position_ids: torch.Tensor | None = None,
+) -> dict:
+    """The cache sizes each mask from the layer it is asked about, and the generic mask utilities ask about the first
+    non-sliding layer, which for a leading `conv` layer redirects to the first attention layer of any type: name the
+    first layer of each mask's own type instead."""
+    config = config.get_text_config()
+    mask_kwargs = {
+        "config": config,
+        "inputs_embeds": inputs_embeds,
+        "attention_mask": attention_mask,
+        "past_key_values": past_key_values,
+        "position_ids": position_ids,
+    }
+    layer_types, causal_mask_mapping = config.layer_types, {}
+    if "full_attention" in layer_types:
+        layer_idx = layer_types.index("full_attention")
+        causal_mask_mapping["full_attention"] = create_causal_mask(**mask_kwargs, layer_idx=layer_idx)
+    if "sliding_attention" in layer_types:
+        layer_idx = layer_types.index("sliding_attention")
+        causal_mask_mapping["sliding_attention"] = create_sliding_window_causal_mask(
+            **mask_kwargs, layer_idx=layer_idx
+        )
+    if "conv" in layer_types:
+        causal_mask_mapping["conv"] = create_recurrent_attention_mask(**mask_kwargs)
+    return causal_mask_mapping
+
+
+@auto_docstring
+class Lfm3VlPreTrainedModel(PreTrainedModel):
+    config: Lfm3VlConfig
+    base_model_prefix = "model"
+    input_modalities = ("image", "video", "text")
+    supports_gradient_checkpointing = True
+    _skip_keys_device_placement = ["past_key_values"]
+
+    _supports_flash_attn = True
+    _supports_sdpa = True
+    # the cached decode step traces without graph breaks, like the text model
+    _can_compile_fullgraph = True
+    _supports_flex_attn = True
+    _supports_attention_backend = True
+    _no_split_modules = ["Lfm3VlVisionLayer"]
+
+    @staticmethod
+    def create_masks_for_generate(
+        config: PreTrainedConfig,
+        inputs_embeds: torch.Tensor,
+        attention_mask: torch.Tensor | None,
+        past_key_values: Cache | None,
+        position_ids: torch.Tensor | None = None,
+        **kwargs,
+    ) -> dict:
+        # the static-cache masks of the text model are sized by layer type, not by the generic first attention layer
+        return create_masks_by_layer_type(config, inputs_embeds, attention_mask, past_key_values, position_ids)
+
+
+def get_vision_frame_shapes(grid_thw: torch.Tensor) -> list[list[int]]:
+    return grid_thw[:, 1:].repeat_interleave(grid_thw[:, 0], dim=0).tolist()
+
+
+@auto_docstring(
+    custom_intro="""
+    The LFM3-VL vision encoder: a stack of gated convolution and bidirectional attention layers over the patches of
+    each image or video frame, embedded by a linear patchifier.
+    """
+)
+class Lfm3VlVisionModel(Lfm3VlPreTrainedModel):
+    config: Lfm3VlVisionConfig
+    input_modalities = ("image", "video")
+    _input_embed_layer = "patchifier"
+    _can_record_outputs = {
+        "hidden_states": Lfm3VlVisionLayer,
+        "attentions": OutputRecorder(Lfm3VlVisionAttention, index=1),
+    }
+
+    def __init__(self, config: Lfm3VlVisionConfig):
+        super().__init__(config)
+        self.spatial_merge_size = config.spatial_merge_size
+        patch_dim = config.in_channels * config.temporal_patch_size * config.patch_size**2
+        self.patchifier = nn.Linear(patch_dim, config.hidden_size, bias=False)
+        self.embedding_norm = Lfm3VlRMSNorm(config.hidden_size, eps=config.norm_eps)
+        self.rotary_pos_emb = Lfm3VlVisionRotaryEmbedding(config)
+        self.layers = nn.ModuleList(
+            [Lfm3VlVisionLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
+        )
+        self.gradient_checkpointing = False
+        self.post_init()
+
+    @merge_with_config_defaults
+    @capture_outputs
+    @auto_docstring
+    def forward(
+        self, pixel_values: torch.Tensor, grid_thw: torch.Tensor, **kwargs: Unpack[TransformersKwargs]
+    ) -> BaseModelOutputWithPooling:
+        r"""
+        pixel_values (`torch.Tensor` of shape `(num_patches, in_channels * temporal_patch_size * patch_size**2)`):
+            The flattened patches of every image or video, grouped by merge block.
+        grid_thw (`torch.LongTensor` of shape `(num_images_or_videos, 3)`):
+            The temporal, height and width dimensions of the patch grid of each image or video.
+        """
+        position_ids = get_vision_position_ids(grid_thw, self.spatial_merge_size, kwargs=kwargs)
+        cu_seqlens, max_seqlen = get_vision_attention_seqlens(grid_thw, self.config, kwargs=kwargs)
+        frame_shapes = get_vision_frame_shapes(grid_thw)
+
+        hidden_states = self.patchifier(pixel_values.to(self.patchifier.weight.dtype))
+        hidden_states = self.embedding_norm(hidden_states)
+        position_embeddings = self.rotary_pos_emb(hidden_states, position_ids)
+
+        for layer in self.layers:
+            hidden_states = layer(
+                hidden_states,
+                cu_seqlens=cu_seqlens,
+                frame_shapes=frame_shapes,
+                max_seqlen=max_seqlen,
+                position_embeddings=position_embeddings,
+                **kwargs,
+            )
+
+        return BaseModelOutputWithPooling(last_hidden_state=hidden_states)
+
+
+class Lfm3VlMultiModalProjector(nn.Module):
+    """Merges each block of `downsample_factor**2` vision tokens into one language token."""
+
+    def __init__(self, config: Lfm3VlConfig):
+        super().__init__()
+        self.downsample_factor = config.downsample_factor
+        self.layer_norm = nn.LayerNorm(config.vision_config.hidden_size, eps=config.projector_layer_norm_eps)
+        self.linear_1 = nn.Linear(
+            config.vision_config.hidden_size * config.downsample_factor**2,
+            config.projector_hidden_size,
+            bias=config.projector_bias,
+        )
+        self.act = ACT2FN[config.projector_hidden_act]
+        self.linear_2 = nn.Linear(
+            config.projector_hidden_size, config.text_config.hidden_size, bias=config.projector_bias
+        )
+
+    def forward(self, image_features: torch.Tensor):
+        hidden_states = self.layer_norm(image_features).reshape(-1, self.linear_1.in_features)
+        hidden_states = self.linear_1(hidden_states)
+        hidden_states = self.act(hidden_states)
+        hidden_states = self.linear_2(hidden_states)
+        return hidden_states
+
+
+@auto_docstring(
+    custom_intro="""
+    Base class for Lfm3Vl outputs, with hidden states and attentions.
+    """
+)
+@dataclass
+class Lfm3VlModelOutputWithPast(BaseModelOutputWithPast):
+    r"""
+    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
+        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).
+
+        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
+        `past_key_values` input) to speed up sequential decoding.
+    image_hidden_states (`torch.FloatTensor`, *optional*):
+        A `torch.FloatTensor` of size `(num_image_tokens, hidden_size)`.
+        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
+    video_hidden_states (`torch.FloatTensor`, *optional*):
+        A `torch.FloatTensor` of size `(num_video_tokens, hidden_size)`.
+        video_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
+    """
+
+    image_hidden_states: torch.FloatTensor | None = None
+    video_hidden_states: torch.FloatTensor | None = None
+
+
+@auto_docstring(
+    custom_intro="""
+    Base class for Lfm3Vl causal language model (or autoregressive) outputs.
+    """
+)
+@dataclass
+class Lfm3VlCausalLMOutputWithPast(CausalLMOutputWithPast):
+    r"""
+    image_hidden_states (`torch.FloatTensor`, *optional*):
+        A `torch.FloatTensor` of size `(num_image_tokens, hidden_size)`.
+        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
+    video_hidden_states (`torch.FloatTensor`, *optional*):
+        A `torch.FloatTensor` of size `(num_video_tokens, hidden_size)`.
+        video_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
+    """
+
+    image_hidden_states: torch.FloatTensor | None = None
+    video_hidden_states: torch.FloatTensor | None = None
+
+
+@auto_docstring(
+    custom_intro="""
+    The Lfm3Vl model which consists of a vision backbone and a language model, without a language modeling head.
+    """
+)
+class Lfm3VlModel(Lfm3VlPreTrainedModel):
+    def __init__(self, config: Lfm3VlConfig):
+        super().__init__(config)
+        self.vision_tower = Lfm3VlVisionModel(config.vision_config)
+        self.multi_modal_projector = Lfm3VlMultiModalProjector(config)
+        self.language_model = Lfm3Model(config.text_config)
+        self.post_init()
+
+    @accepts_precomputed_kwargs(modality="image")
+    @can_return_tuple
+    @auto_docstring(
+        custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection."
+    )
+    def get_image_features(
+        self,
+        pixel_values: torch.FloatTensor,
+        image_grid_thw: torch.LongTensor,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | BaseModelOutputWithPooling:
+        r"""
+        pixel_values (`torch.FloatTensor` of shape `(num_patches, in_channels * temporal_patch_size * patch_size**2)`):
+            The flattened patches of every image, grouped by merge block.
+        image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`):
+            The temporal, height and width dimensions of the patch grid of each image.
+        """
+        image_outputs = self.vision_tower(pixel_values, grid_thw=image_grid_thw, return_dict=True, **kwargs)
+        image_features = self.multi_modal_projector(image_outputs.last_hidden_state)
+        split_sizes = (image_grid_thw.prod(-1) // self.vision_tower.spatial_merge_size**2).tolist()
+        image_outputs.pooler_output = torch.split(image_features, split_sizes)
+        return image_outputs
+
+    def get_placeholder_mask(
+        self,
+        input_ids: torch.LongTensor,
+        inputs_embeds: torch.FloatTensor,
+        image_features: torch.FloatTensor | None = None,
+        video_features: torch.FloatTensor | None = None,
+    ):
+        """Placeholder masks from `input_ids` or `inputs_embeds`, checked against the feature counts."""
+        if input_ids is None:
+            special_image_mask = inputs_embeds == self.get_input_embeddings()(
+                torch.full((), self.config.image_token_id, dtype=torch.long, device=inputs_embeds.device)
+            )
+            special_image_mask = special_image_mask.all(-1)
+            special_video_mask = inputs_embeds == self.get_input_embeddings()(
+                torch.full((), self.config.video_token_id, dtype=torch.long, device=inputs_embeds.device)
+            )
+            special_video_mask = special_video_mask.all(-1)
+        else:
+            special_image_mask = input_ids == self.config.image_token_id
+            special_video_mask = input_ids == self.config.video_token_id
+
+        n_image_tokens = special_image_mask.sum()
+        special_image_mask = special_image_mask.unsqueeze(-1).to(inputs_embeds.device)
+        if image_features is not None:
+            torch_compilable_check(
+                n_image_tokens * inputs_embeds.shape[-1] == image_features.numel(),
+                f"Image features and image tokens do not match, tokens: {n_image_tokens}, features: {image_features.shape[0]}",
+            )
+
+        n_video_tokens = special_video_mask.sum()
+        special_video_mask = special_video_mask.unsqueeze(-1).to(inputs_embeds.device)
+        if video_features is not None:
+            torch_compilable_check(
+                n_video_tokens * inputs_embeds.shape[-1] == video_features.numel(),
+                f"Video features and video tokens do not match, tokens: {n_video_tokens}, features: {video_features.shape[0]}",
+            )
+        return special_image_mask, special_video_mask
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        pixel_values: torch.FloatTensor | None = None,
+        image_grid_thw: torch.LongTensor | None = None,
+        pixel_values_videos: torch.FloatTensor | None = None,
+        video_grid_thw: torch.LongTensor | None = None,
+        use_cache: bool | None = None,
+        mm_encoder_outputs: dict[str, BaseModelOutputWithPooling] | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | Lfm3VlModelOutputWithPast:
+        r"""
+        pixel_values (`torch.FloatTensor` of shape `(num_patches, in_channels * temporal_patch_size * patch_size**2)`, *optional*):
+            The flattened patches of every image, grouped by merge block.
+        image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):
+            The temporal, height and width dimensions of the patch grid of each image.
+        pixel_values_videos (`torch.FloatTensor` of shape `(num_patches, in_channels * temporal_patch_size * patch_size**2)`, *optional*):
+            The flattened patches of every video, grouped by merge block.
+        video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*):
+            The temporal, height and width dimensions of the patch grid of each video.
+        """
+        if (input_ids is None) ^ (inputs_embeds is not None):
+            raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
+
+        if (pixel_values is not None or pixel_values_videos is not None) and mm_encoder_outputs is not None:
+            raise ValueError(
+                "You cannot specify both pixel_values/pixel_values_videos and mm_encoder_outputs at the same time"
+            )
+
+        if inputs_embeds is None:
+            inputs_embeds = self.get_input_embeddings()(input_ids)
+
+        mm_encoder_outputs = mm_encoder_outputs if mm_encoder_outputs is not None else {}
+        if (pixel_values is None) != (image_grid_thw is None):
+            raise ValueError("pixel_values and image_grid_thw must be supplied together")
+        if (pixel_values_videos is None) != (video_grid_thw is None):
+            raise ValueError("pixel_values_videos and video_grid_thw must be supplied together")
+        if mm_encoder_outputs.get("image") is None and pixel_values is not None:
+            mm_encoder_outputs["image"] = self.get_image_features(
+                pixel_values, image_grid_thw, return_dict=True, **kwargs
+            )
+        if mm_encoder_outputs.get("video") is None and pixel_values_videos is not None:
+            mm_encoder_outputs["video"] = self.get_video_features(
+                pixel_values_videos, video_grid_thw, return_dict=True, **kwargs
+            )
+
+        image_features = None
+        if mm_encoder_outputs.get("image") is not None:
+            image_features = torch.cat(mm_encoder_outputs["image"].pooler_output, dim=0).to(
+                inputs_embeds.device, inputs_embeds.dtype
+            )
+            image_mask, _ = self.get_placeholder_mask(
+                input_ids, inputs_embeds=inputs_embeds, image_features=image_features
+            )
+            inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_features)
+
+        video_features = None
+        if mm_encoder_outputs.get("video") is not None:
+            video_features = torch.cat(mm_encoder_outputs["video"].pooler_output, dim=0).to(
+                inputs_embeds.device, inputs_embeds.dtype
+            )
+            _, video_mask = self.get_placeholder_mask(
+                input_ids, inputs_embeds=inputs_embeds, video_features=video_features
+            )
+            inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_features)
+
+        outputs = self.language_model(
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            **kwargs,
+        )
+
+        return Lfm3VlModelOutputWithPast(
+            last_hidden_state=outputs.last_hidden_state,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+            image_hidden_states=image_features,
+            video_hidden_states=video_features,
+        )
+
+    @accepts_precomputed_kwargs(modality="video")
+    @can_return_tuple
+    @auto_docstring(
+        custom_intro="Obtains video last hidden states from the vision tower and apply multimodal projection."
+    )
+    def get_video_features(
+        self,
+        pixel_values_videos: torch.FloatTensor,
+        video_grid_thw: torch.LongTensor,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | BaseModelOutputWithPooling:
+        r"""
+        pixel_values_videos (`torch.FloatTensor` of shape `(num_patches, in_channels * temporal_patch_size * patch_size**2)`):
+            The flattened patches of every video, grouped by merge block.
+        video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`):
+            The temporal, height and width dimensions of the patch grid of each video.
+        """
+        return self.get_image_features(pixel_values_videos, video_grid_thw, **kwargs)
+
+
+def reject_packed_generation(config: PreTrainedConfig, input_ids, generation_config, model_kwargs, is_first_iteration):
+    """`generate` decodes from the cache its prefill fills, and no mask it builds isolates packed documents
+    (`position_ids` restarting): the prompt is refused before the prefill, one forward or every chunk, writes to it."""
+    position_ids, attention_mask = model_kwargs.get("position_ids"), model_kwargs.get("attention_mask")
+    cache = model_kwargs.get("past_key_values")
+    if cache is None or position_ids is None or position_ids.ndim != 2:
+        return
+    if attention_mask is not None and attention_mask.ndim != 2:
+        return
+    has_attention = any(t != "conv" for t in config.get_text_config().layer_types)
+    filled = bool(cache.get_seq_length() > 0 if has_attention else cache.has_previous_state())
+    inputs_embeds = model_kwargs.get("inputs_embeds") if is_first_iteration else None
+    length = input_ids.shape[1] if inputs_embeds is None else inputs_embeds.shape[1]
+    if generation_config.prefill_chunk_size is not None:
+        # the chunks run from the start of the prompt
+        position_ids = position_ids[:, :length]
+    else:
+        if filled and has_attention:
+            # the slice `_prefill` feeds: a prompt that spans the cache is cut to its uncached part
+            mask_length = getattr(generation_config, "_mask_length", -1)
+            if attention_mask is not None:
+                mask_length = attention_mask.shape[1]
+            if inputs_embeds is not None or mask_length == length:
+                length -= int(cache.get_seq_length())
+        position_ids = position_ids[:, position_ids.shape[1] - length :]
+    if attention_mask is None:
+        attention_mask = torch.ones_like(position_ids)
+    if bool(packed_under_padding(position_ids, attention_mask, filled).any()):
+        raise ValueError(
+            "`generate` cannot decode packed documents (`position_ids` restarting, or starting at 0 on a cache that "
+            "holds tokens): attention would span them. Generate each document in its own row of a left-padded batch."
+        )
+
+
+def packed_under_padding(position_ids: torch.Tensor, padding_mask: torch.Tensor, filled) -> torch.Tensor:
+    """Padding repeats position 0, so packing is a restart between two kept tokens, or a kept position 0 on a filled
+    cache whose row holds a kept token; attention under a padding mask cannot isolate it."""
+    restarts = torch.diff(position_ids, prepend=position_ids[:, :1] - 1, dim=-1) != 1
+    valid = padding_mask[:, -restarts.shape[-1] :].bool()
+    packed = restarts[:, 1:] & valid[:, 1:] & valid[:, :-1]
+    if filled:
+        # a row whose cached tokens are all padding (chunked prefill of a left-padded batch) has no document to pack
+        # against; a mask covering only the chunk keeps the strict rule
+        cached = padding_mask[:, : padding_mask.shape[-1] - restarts.shape[-1]]
+        cached_valid = cached.bool().any(-1, keepdim=True) if cached.shape[-1] else torch.ones_like(valid[:, :1])
+        packed = packed | (valid[:, :1] & (position_ids[:, :1] == 0) & cached_valid)
+    return packed
+
+
+@auto_docstring(
+    custom_intro="""
+    The LFM3_VL model which consists of a vision backbone and a language model.
+    """
+)
+class Lfm3VlForConditionalGeneration(Lfm3VlPreTrainedModel, GenerationMixin):
+    _tied_weights_keys = {"lm_head.weight": "model.language_model.embed_tokens.weight"}
+
+    def __init__(self, config: Lfm3VlConfig):
+        super().__init__(config)
+        self.model = Lfm3VlModel(config)
+        self.lm_head = nn.Linear(config.text_config.hidden_size, config.text_config.vocab_size, bias=False)
+        self.post_init()
+
+    def get_output_embeddings(self) -> nn.Module:
+        return self.lm_head
+
+    @auto_docstring
+    def get_image_features(
+        self,
+        pixel_values: torch.FloatTensor,
+        image_grid_thw: torch.LongTensor,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | BaseModelOutputWithPooling:
+        r"""
+        pixel_values (`torch.FloatTensor` of shape `(num_patches, in_channels * temporal_patch_size * patch_size**2)`):
+            The flattened patches of every image, grouped by merge block.
+        image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`):
+            The temporal, height and width dimensions of the patch grid of each image.
+        """
+        return self.model.get_image_features(pixel_values, image_grid_thw, **kwargs)
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        pixel_values: torch.FloatTensor | None = None,
+        image_grid_thw: torch.LongTensor | None = None,
+        pixel_values_videos: torch.FloatTensor | None = None,
+        video_grid_thw: torch.LongTensor | None = None,
+        labels: torch.LongTensor | None = None,
+        use_cache: bool | None = None,
+        logits_to_keep: int | torch.Tensor = 0,
+        mm_encoder_outputs: dict[str, BaseModelOutputWithPooling] | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | Lfm3VlCausalLMOutputWithPast:
+        r"""
+        pixel_values (`torch.FloatTensor` of shape `(num_patches, in_channels * temporal_patch_size * patch_size**2)`, *optional*):
+            The flattened patches of every image, grouped by merge block.
+        image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):
+            The temporal, height and width dimensions of the patch grid of each image.
+        pixel_values_videos (`torch.FloatTensor` of shape `(num_patches, in_channels * temporal_patch_size * patch_size**2)`, *optional*):
+            The flattened patches of every video, grouped by merge block.
+        video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*):
+            The temporal, height and width dimensions of the patch grid of each video.
+        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+
+        Example:
+
+        ```python
+        >>> from transformers import AutoProcessor, AutoModelForImageTextToText
+
+        >>> model = AutoModelForImageTextToText.from_pretrained("LiquidAI/LFM3-VL-2B")
+        >>> processor = AutoProcessor.from_pretrained("LiquidAI/LFM3-VL-2B")
+
+        >>> conversation = [
+        ...     {
+        ...         "role": "user",
+        ...         "content": [
+        ...             {"type": "image", "url": "https://www.ilankelman.org/stopsigns/australia.jpg"},
+        ...             {"type": "text", "text": "What is in this image?"},
+        ...         ],
+        ...     },
+        ... ]
+
+        >>> inputs = processor.apply_chat_template(
+        ...     conversation, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt"
+        ... )
+
+        >>> outputs = model.generate(**inputs, max_new_tokens=45)
+        >>> processor.batch_decode(outputs, skip_special_tokens=True)[0]
+        ```"""
+        outputs = self.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            pixel_values=pixel_values,
+            image_grid_thw=image_grid_thw,
+            pixel_values_videos=pixel_values_videos,
+            video_grid_thw=video_grid_thw,
+            use_cache=use_cache,
+            mm_encoder_outputs=mm_encoder_outputs,
+            **kwargs,
+        )
+
+        hidden_states = outputs[0]
+        # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
+        slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
+        logits = self.lm_head(hidden_states[:, slice_indices, :])
+
+        loss = None
+        if labels is not None:
+            loss = self.loss_function(
+                logits=logits, labels=labels, vocab_size=self.config.text_config.vocab_size, **kwargs
+            )
+
+        return Lfm3VlCausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+            image_hidden_states=outputs.image_hidden_states,
+            video_hidden_states=outputs.video_hidden_states,
+        )
+
+    def _prefill(self, input_ids, generation_config, model_kwargs, is_first_iteration=True):
+        # the text model's refusal, before the prefill writes to the cache
+        reject_packed_generation(self.config, input_ids, generation_config, model_kwargs, is_first_iteration)
+        return super()._prefill(input_ids, generation_config, model_kwargs, is_first_iteration=is_first_iteration)
+
+    @auto_docstring
+    def get_video_features(
+        self,
+        pixel_values_videos: torch.FloatTensor,
+        video_grid_thw: torch.LongTensor,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | BaseModelOutputWithPooling:
+        r"""
+        pixel_values_videos (`torch.FloatTensor` of shape `(num_patches, in_channels * temporal_patch_size * patch_size**2)`):
+            The flattened patches of every video, grouped by merge block.
+        video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`):
+            The temporal, height and width dimensions of the patch grid of each video.
+        """
+        return self.model.get_video_features(pixel_values_videos, video_grid_thw, **kwargs)
+
+
+__all__ = ["Lfm3VlForConditionalGeneration", "Lfm3VlModel", "Lfm3VlPreTrainedModel", "Lfm3VlVisionModel"]

processing_lfm3_vl.py

@@ -0,0 +1,248 @@
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import math
+
+import numpy as np
+
+from transformers.image_utils import is_pil_image, is_valid_image, make_flat_list_of_images
+from transformers.processing_utils import MultiModalData, ProcessingKwargs, ProcessorMixin
+from transformers.utils import auto_docstring, is_numpy_array, is_torch_tensor, logging
+from transformers.video_utils import VideoMetadata, is_valid_video, make_batched_videos
+
+
+logger = logging.get_logger(__name__)
+
+
+class Lfm3VlProcessorKwargs(ProcessingKwargs, total=False):
+    _defaults = {
+        "text_kwargs": {
+            "padding": False,
+            "return_token_type_ids": False,
+            "return_mm_token_type_ids": False,
+        },
+        "videos_kwargs": {"return_metadata": True},
+    }
+
+
+def _check_video_fps(metadata: VideoMetadata) -> None:
+    if metadata.fps is not None and not (math.isfinite(metadata.fps) and metadata.fps > 0):
+        raise ValueError(f"`video_metadata.fps` must be a positive number, got {metadata.fps}")
+
+
+def _media_per_prompt(media, is_item) -> list[int] | None:
+    # rows name the prompt that owns them while the expansion consumes media in flat order; a row is a list of whole
+    # items or one stacked batch, and a list of single frames is one video, not a row
+    if not isinstance(media, (list, tuple)):
+        return None
+    counts = []
+    for row in media:
+        if isinstance(row, (list, tuple)) and all(is_item(item) for item in row):
+            counts.append(len(row))
+        elif (is_numpy_array(row) or is_torch_tensor(row)) and row.ndim == 4 and (len(row) == 0 or is_item(row[0])):
+            counts.append(len(row))
+        else:
+            return None
+    return counts
+
+
+def _check_vision_placeholder_counts(processor, prompts, images, videos) -> None:
+    for name, token, media, flatten, is_item in (
+        (
+            "images",
+            processor.image_token,
+            images,
+            make_flat_list_of_images,
+            lambda item: is_valid_image(item) and (is_pil_image(item) or item.ndim == 3),
+        ),
+        (
+            "videos",
+            processor.video_token,
+            videos,
+            make_batched_videos,
+            lambda item: isinstance(item, str) or (not is_pil_image(item) and is_valid_video(item)),
+        ),
+    ):
+        num_placeholders = sum(prompt.count(token) for prompt in prompts)
+        num_media = 0 if media is None else len(flatten(media))
+        if num_placeholders != num_media:
+            raise ValueError(f"Found {num_placeholders} {token} tokens in the text but received {num_media} {name}")
+        # one prompt cannot take another's media
+        per_prompt = _media_per_prompt(media, is_item) if len(prompts) > 1 else None
+        in_text = [prompt.count(token) for prompt in prompts]
+        if per_prompt is not None and per_prompt != in_text:
+            raise ValueError(f"Found {in_text} {token} tokens per prompt but received {per_prompt} {name} per prompt")
+
+
+@auto_docstring
+class Lfm3VlProcessor(ProcessorMixin):
+    valid_processor_kwargs = Lfm3VlProcessorKwargs
+
+    def __init__(self, image_processor=None, tokenizer=None, video_processor=None, chat_template=None, **kwargs):
+        self.image_token = getattr(tokenizer, "image_token", "<|image|>")
+        self.video_token = getattr(tokenizer, "video_token", "<|video|>")
+        self.vision_start_token = getattr(tokenizer, "vision_start_token", "<|vision_start|>")
+        self.vision_end_token = getattr(tokenizer, "vision_end_token", "<|vision_end|>")
+        self.image_token_id = tokenizer.convert_tokens_to_ids(self.image_token)
+        self.video_token_id = tokenizer.convert_tokens_to_ids(self.video_token)
+        self.vision_start_token_id = tokenizer.convert_tokens_to_ids(self.vision_start_token)
+        self.vision_end_token_id = tokenizer.convert_tokens_to_ids(self.vision_end_token)
+        super().__init__(image_processor, tokenizer, video_processor, chat_template=chat_template)
+
+    def replace_image_token(self, image_inputs: dict, image_idx: int, **kwargs) -> str:
+        merge_length = self.image_processor.merge_size**2
+        num_image_tokens = image_inputs["image_grid_thw"][image_idx].prod() // merge_length
+        return self.image_token * num_image_tokens
+
+    def replace_video_token(self, video_inputs: dict, video_idx: int, **kwargs) -> str:
+        metadata = video_inputs["video_metadata"][video_idx]
+        if metadata.frames_indices is None:
+            raise ValueError("Frame timestamps need the sampled frame indices in `video_metadata`")
+        _check_video_fps(metadata)
+        merge_length = self.video_processor.merge_size**2
+        num_frames = video_inputs["video_grid_thw"][video_idx][0]
+        frame_seqlen = video_inputs["video_grid_thw"][video_idx][1:].prod() // merge_length
+        metadata = video_inputs["video_metadata"][video_idx]
+        video_placeholder = ""
+
+        if metadata.fps is None:
+            logger.warning_once(
+                "Lfm3Vl requires frame timestamps to construct prompts, but the `fps` of the input video could not be inferred. "
+                "Probably `video_metadata` was missing from inputs and you passed pre-sampled frames. "
+                "Defaulting to `fps=24`. Please provide `video_metadata` for more accurate results."
+            )
+        # a released transformers reads torchcodec's scanned average fps, which carries float noise
+        # (30.000000000000004) where the fork reads the container header
+        metadata.fps = 24 if metadata.fps is None else round(metadata.fps, 6)
+
+        # if timestamps are not provided, calculate them
+        curr_timestamp = self._calculate_timestamps(
+            metadata.frames_indices,
+            metadata.fps,
+            self.video_processor.temporal_patch_size,
+        )
+
+        for frame_idx in range(num_frames):
+            curr_time = curr_timestamp[frame_idx]
+            video_placeholder += f"<{curr_time:.1f} seconds>"
+            video_placeholder += self.vision_start_token + self.video_token * frame_seqlen + self.vision_end_token
+        return video_placeholder
+
+    def _get_num_multimodal_tokens(self, image_sizes=None, video_sizes=None, **kwargs):
+        """
+        Computes the number of placeholder tokens needed for multimodal inputs with the given sizes.
+
+        Args:
+            image_sizes (`list[list[int]]`, *optional*):
+                The input sizes formatted as (height, width) per each image.
+            video_sizes (`list[list[int]]`, *optional*):
+                The input sizes formatted as (num_frames, height, width) per each video.
+
+        Returns:
+            `MultiModalData`: the number of tokens per provided modality.
+        """
+        vision_data = {}
+        if image_sizes is not None:
+            # copy: `update` on the class-level defaults would leak kwargs into every later call
+            images_kwargs = dict(Lfm3VlProcessorKwargs._defaults.get("images_kwargs", {}))
+            images_kwargs.update(kwargs)
+            merge_size = images_kwargs.get("merge_size") or self.image_processor.merge_size
+
+            num_image_patches = [
+                self.image_processor.get_number_of_image_patches(*image_size, images_kwargs)
+                for image_size in image_sizes
+            ]
+            num_image_tokens = [(num_patches // merge_size**2) for num_patches in num_image_patches]
+            vision_data.update({"num_image_tokens": num_image_tokens, "num_image_patches": num_image_patches})
+
+        if video_sizes is not None:
+            videos_kwargs = dict(Lfm3VlProcessorKwargs._defaults.get("videos_kwargs", {}))
+            videos_kwargs.update(kwargs)
+            merge_size = videos_kwargs.get("merge_size") or self.video_processor.merge_size
+            num_video_patches = [
+                self.video_processor.get_num_of_video_patches(*video_size, videos_kwargs) for video_size in video_sizes
+            ]
+            num_video_tokens = [(num_patches // merge_size**2) for num_patches in num_video_patches]
+            vision_data["num_video_tokens"] = num_video_tokens
+
+        return MultiModalData(**vision_data)
+
+    def post_process_image_text_to_text(
+        self, generated_outputs, skip_special_tokens=True, clean_up_tokenization_spaces=False, **kwargs
+    ):
+        """
+        Post-process the output of the model to decode the text.
+
+        Args:
+            generated_outputs (`torch.Tensor` or `np.ndarray`):
+                The output of the model `generate` function. The output is expected to be a tensor of shape `(batch_size, sequence_length)`
+                or `(sequence_length,)`.
+            skip_special_tokens (`bool`, *optional*, defaults to `True`):
+                Whether or not to remove special tokens in the output. Argument passed to the tokenizer's `batch_decode` method.
+            clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
+                Whether or not to clean up the tokenization spaces. Argument passed to the tokenizer's `batch_decode` method.
+            **kwargs:
+                Additional arguments to be passed to the tokenizer's `batch_decode method`.
+
+        Returns:
+            `list[str]`: The decoded text.
+        """
+        return self.tokenizer.batch_decode(
+            generated_outputs,
+            skip_special_tokens=skip_special_tokens,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            **kwargs,
+        )
+
+    @property
+    def model_input_names(self):
+        return list(
+            dict.fromkeys(
+                self.tokenizer.model_input_names
+                + self.image_processor.model_input_names
+                + self.video_processor.model_input_names
+            )
+        )
+
+    def _calculate_timestamps(self, indices: list[int] | np.ndarray, video_fps: float, merge_size: int = 2):
+        if not isinstance(indices, list):
+            indices = indices.tolist()
+        if len(indices) % merge_size != 0:
+            indices.extend(indices[-1] for _ in range(merge_size - len(indices) % merge_size))
+        timestamps = [idx / video_fps for idx in indices]
+        # @JJJYmmm frames are merged by self.merge_size, \
+        # so we need to average the timestamps between the first/last frame within the temporal patch
+        timestamps = [
+            (timestamps[i] + timestamps[i + merge_size - 1]) / 2 for i in range(0, len(timestamps), merge_size)
+        ]
+        return timestamps
+
+    def prepare_inputs_layout(self, images=None, text=None, videos=None, audio=None, **kwargs):
+        for name, media in (("images", images), ("videos", videos)):
+            if isinstance(media, (list, tuple)) and len(media) == 0:
+                raise ValueError(f"`{name}` must hold at least one item; pass `{name}=None` for prompts without one")
+        return super().prepare_inputs_layout(images=images, text=text, videos=videos, audio=audio, **kwargs)
+
+    def validate_inputs(self, images=None, text=None, videos=None, audio=None, **kwargs):
+        super().validate_inputs(images=images, text=text, videos=videos, audio=audio, **kwargs)
+        if text is not None:
+            _check_vision_placeholder_counts(self, [text] if isinstance(text, str) else text, images, videos)
+
+    def _process_videos(self, videos, **kwargs):
+        # the frame timestamps need the metadata whatever the caller's `return_metadata`; `__call__` drops it from
+        # the output unless it was asked for
+        kwargs["return_metadata"] = True
+        return super()._process_videos(videos, **kwargs)
+
+
+__all__ = ["Lfm3VlProcessor"]

processor_config.json

@@ -0,0 +1,71 @@
+{
+  "image_processor": {
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_processor_type": "Lfm3VlImageProcessor",
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "merge_size": 2,
+    "patch_size": 16,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "size": {
+      "longest_edge": 16777216,
+      "shortest_edge": 65536
+    },
+    "temporal_patch_size": 2,
+    "auto_map": {
+      "AutoImageProcessor": "image_processing_lfm3_vl.Lfm3VlImageProcessor"
+    }
+  },
+  "processor_class": "Lfm3VlProcessor",
+  "video_processor": {
+    "cap_pixels_per_frame": false,
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "do_sample_frames": true,
+    "fps": 2.0,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "max_frames": 768,
+    "max_video_tokens": 768,
+    "merge_size": 2,
+    "min_frames": 4,
+    "patch_size": 16,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "return_metadata": false,
+    "size": {
+      "longest_edge": 25165824,
+      "shortest_edge": 4096
+    },
+    "temporal_patch_size": 2,
+    "video_processor_type": "Lfm3VlVideoProcessor",
+    "auto_map": {
+      "AutoVideoProcessor": "video_processing_lfm3_vl.Lfm3VlVideoProcessor"
+    }
+  },
+  "auto_map": {
+    "AutoProcessor": "processing_lfm3_vl.Lfm3VlProcessor"
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a39eed58008cbcbc09e5b629b81338a654269c64cd13348aaa6090f4db75c7e7
+size 17908258

tokenizer_config.json

@@ -0,0 +1,61 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|startoftext|>",
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": [],
+  "image_token": "<|image|>",
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "model_specific_special_tokens": {
+    "image_token": "<|image|>",
+    "video_token": "<|video|>",
+    "vision_end_token": "<|vision_end|>",
+    "vision_start_token": "<|vision_start|>"
+  },
+  "pad_token": "<|pad|>",
+  "processor_class": "Lfm3VlProcessor",
+  "tokenizer_class": "TokenizersBackend",
+  "video_token": "<|video|>",
+  "vision_end_token": "<|vision_end|>",
+  "vision_start_token": "<|vision_start|>",
+  "response_template": {
+    "defaults": {
+      "role": "assistant"
+    },
+    "start_anchor_pattern": "<\\|im_start\\|>assistant\\n",
+    "fields": {
+      "reasoning_content": {
+        "open_pattern": "<think>\\n?",
+        "close_pattern": "</think>\\s*",
+        "content": "text"
+      },
+      "tool_calls": {
+        "open_pattern": "<tool_call>\\s*<function=(?P<name>[^\\n>]+)>",
+        "close_pattern": "</tool_call>\\s*",
+        "repeats": true,
+        "content": "xml-inline",
+        "content_args": {
+          "tag_pattern": "<parameter=(?P<key>[^>\\n]+)>\\n?(?P<value>.*?)\\n?</parameter>(?=\\s*(?:<parameter=|</function>\\s*\\Z|\\Z))",
+          "value_parser": {
+            "name": "text",
+            "args": {
+              "strip": false
+            }
+          }
+        },
+        "transform": {
+          "type": "function",
+          "function": {
+            "name": "{name}",
+            "arguments": "{content}"
+          }
+        }
+      },
+      "content": {
+        "close_pattern": "<\\|im_end\\|>\\s*",
+        "content": "text"
+      }
+    }
+  }
+}

video_processing_lfm3_vl.py

@@ -0,0 +1,396 @@
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import math
+
+import numpy as np
+import torch
+from torchvision.transforms.v2 import functional as tvF
+
+from transformers.image_processing_utils import BatchFeature
+from transformers.image_utils import PILImageResampling, SizeDict
+from transformers.processing_utils import Unpack, VideosKwargs
+from transformers.utils import TensorType, auto_docstring, logging
+from transformers.video_processing_utils import BaseVideoProcessor
+from transformers.video_utils import (
+    VideoMetadata,
+    group_videos_by_shape,
+    make_batched_metadata,
+    make_batched_videos,
+    reorder_videos,
+)
+
+
+logger = logging.get_logger(__name__)
+
+
+class Lfm3VlVideoProcessorInitKwargs(VideosKwargs, total=False):
+    r"""
+    patch_size (`int`, *optional*, defaults to 16):
+        The spatial patch size of the vision encoder.
+    temporal_patch_size (`int`, *optional*, defaults to 2):
+        The temporal patch size of the vision encoder.
+    merge_size (`int`, *optional*, defaults to 2):
+        The merge size of the vision encoder to llm encoder.
+    min_frames (`int`, *optional*, defaults to 4):
+        The minimum number of frames to sample from video.
+    max_frames (`int`, *optional*, defaults to 768):
+        The maximum number of frames to sample from video.
+    cap_pixels_per_frame (`bool`, *optional*):
+        Whether to cap each frame's pixel cost the way the reference implementation (qwen-vl-utils) does:
+        per-frame pixels are limited to `min(max_video_tokens * factor**2, size["longest_edge"] / num_frames)`
+        and floored at `1.05 * size["shortest_edge"]`, so token cost scales with clip duration. Without the
+        cap, videos that sample few frames spend the whole `size["longest_edge"]` budget on those frames and
+        keep near-native per-frame resolution, so a short clip can cost almost as many tokens as a long
+        video. If unset, the current uncapped behavior is kept and a warning is emitted: the default will
+        change to `True` in v5.22, after which the argument will be removed.
+    max_video_tokens (`int`, *optional*, defaults to 768):
+        The per-frame token ceiling applied by `cap_pixels_per_frame`, in vision tokens per frame
+        (qwen-vl-utils' `VIDEO_MAX_TOKEN_NUM`).
+    """
+
+    patch_size: int
+    temporal_patch_size: int
+    merge_size: int
+    min_frames: int
+    max_frames: int
+    cap_pixels_per_frame: bool
+    max_video_tokens: int
+
+
+def smart_resize(
+    num_frames: int,
+    height: int,
+    width: int,
+    temporal_factor: int = 2,
+    factor: int = 28,
+    min_pixels: int = 112 * 112,
+    max_pixels: int = 14 * 14 * 2 * 2 * 2 * 6144,
+):
+    if num_frames < temporal_factor:
+        raise ValueError(f"t:{num_frames} must be larger than temporal_factor:{temporal_factor}")
+    if height < factor or width < factor:
+        scale = max(factor / height, factor / width)
+        height = int(height * scale)
+        width = int(width * scale)
+
+    if max(height, width) / min(height, width) > 200:
+        raise ValueError(
+            f"absolute aspect ratio must be smaller than 200, got {max(height, width) / min(height, width)}"
+        )
+    h_bar = round(height / factor) * factor
+    w_bar = round(width / factor) * factor
+    t_bar = round(num_frames / temporal_factor) * temporal_factor
+
+    if t_bar * h_bar * w_bar > max_pixels:
+        beta = math.sqrt((num_frames * height * width) / max_pixels)
+        h_bar = max(factor, math.floor(height / beta / factor) * factor)
+        w_bar = max(factor, math.floor(width / beta / factor) * factor)
+    elif t_bar * h_bar * w_bar < min_pixels:
+        beta = math.sqrt(min_pixels / (num_frames * height * width))
+        h_bar = math.ceil(height * beta / factor) * factor
+        w_bar = math.ceil(width * beta / factor) * factor
+
+    return h_bar, w_bar
+
+
+def _check_video_fps(metadata: VideoMetadata) -> None:
+    if metadata.fps is not None and not (math.isfinite(metadata.fps) and metadata.fps > 0):
+        raise ValueError(f"`video_metadata.fps` must be a positive number, got {metadata.fps}")
+
+
+def _check_video_metadata_count(videos, video_metadata) -> None:
+    # the shared sampler pairs clips with metadata by `zip`, which drops the clips past the last entry
+    if video_metadata is None:
+        return
+    videos = make_batched_videos(videos)
+    entries = make_batched_metadata(videos, video_metadata) if video_metadata != [] else []
+    if len(entries) != len(videos):
+        raise ValueError(
+            f"Received {len(videos)} videos but {len(entries)} `video_metadata` entries; pass one entry per video"
+        )
+
+
+@auto_docstring
+class Lfm3VlVideoProcessor(BaseVideoProcessor):
+    resample = PILImageResampling.BICUBIC
+    size = {"shortest_edge": 4_096, "longest_edge": 25_165_824}
+    image_mean = [0.5, 0.5, 0.5]
+    image_std = [0.5, 0.5, 0.5]
+    do_resize = True
+    do_rescale = True
+    do_normalize = True
+    do_convert_rgb = True
+    patch_size = 16
+    temporal_patch_size = 2
+    merge_size = 2
+    min_frames = 4
+    max_frames = 768
+    do_sample_frames = True
+    cap_pixels_per_frame = False
+    max_video_tokens = 768
+    valid_kwargs = Lfm3VlVideoProcessorInitKwargs
+    model_input_names = ["pixel_values_videos", "video_grid_thw"]
+    max_image_size = {"longest_edge": 28 * 28 * 2 * 30000}
+    max_duration = None
+    num_frames = None
+    fps = 2.0
+
+    def __init__(self, **kwargs: Unpack[Lfm3VlVideoProcessorInitKwargs]):
+        super().__init__(**kwargs)
+
+    def sample_frames(
+        self, metadata: VideoMetadata, num_frames: int | None = None, fps: int | float | None = None, **kwargs
+    ):
+        """Decoded frames whose metadata carries no fps are sampled and timestamped at the 24 fps the shared sampler
+        assumes (with a warning); pass `video_metadata` with the clip's fps for accurate timestamps."""
+        _check_video_fps(metadata)
+        if fps is not None and not (math.isfinite(fps) and fps > 0):
+            raise ValueError(f"`fps` must be a positive number, got {fps}")
+        if fps is not None and num_frames is not None:
+            raise ValueError("`num_frames` and `fps` are mutually exclusive arguments, please use only one!")
+
+        total_num_frames = metadata.total_num_frames
+        fps = fps if fps is not None else self.fps
+
+        # If num_frames is not given but fps is, calculate num_frames from fps
+        if num_frames is None and fps is not None:
+            if metadata.fps is None:
+                metadata.fps = 24
+                logger.warning_once(
+                    "Asked to sample `fps` frames per second but no video metadata was provided which is required when sampling with `fps`. "
+                    "Defaulting to `fps=24`. Please provide `video_metadata` for more accurate results."
+                )
+            num_frames = int(total_num_frames / metadata.fps * fps)
+            num_frames = min(max(num_frames, self.min_frames), self.max_frames, total_num_frames)
+
+        if num_frames is None:
+            num_frames = min(max(total_num_frames, self.min_frames), self.max_frames)
+
+        indices = np.linspace(0, total_num_frames - 1, num_frames).round().astype(int)
+
+        return indices
+
+    def resize(
+        self,
+        videos: "torch.Tensor",
+        size: SizeDict,
+        resample: "PILImageResampling | tvF.InterpolationMode | int | None",
+        factor: int,
+        temporal_factor: int,
+        cap_pixels_per_frame: bool | None = None,
+        **kwargs,
+    ) -> "torch.Tensor":
+        """Resize dynamically based on input video aspect ratio."""
+        if not size.shortest_edge or not size.longest_edge:
+            raise ValueError(f"`size` dict must contain 'shortest_edge' and 'longest_edge' keys but got {size}.")
+
+        num_frames = videos.shape[1]
+        max_pixels = size.longest_edge
+        if cap_pixels_per_frame:
+            # per-frame pixels are capped at `max_video_tokens` patches or the budget's even share per frame
+            frame_cap = self.max_video_tokens * factor * factor
+            pixels_per_frame = max(min(frame_cap, size.longest_edge // num_frames), int(size.shortest_edge * 1.05))
+            max_pixels = pixels_per_frame * num_frames
+
+        height, width = videos.shape[-2:]
+        resized_height, resized_width = smart_resize(
+            height=height,
+            width=width,
+            num_frames=num_frames,
+            factor=factor,
+            temporal_factor=temporal_factor,
+            min_pixels=size.shortest_edge,
+            max_pixels=max_pixels,
+        )
+        return super().resize(
+            image=videos,
+            size=SizeDict(height=resized_height, width=resized_width),
+            resample=resample,
+        )
+
+    def patchify(
+        self,
+        videos: "torch.Tensor",
+        patch_size: int,
+        merge_size: int,
+        temporal_patch_size: int,
+    ) -> tuple["torch.Tensor", int, int]:
+        "Patchifies each video into flat layout of shape (`seq_len`, `patch_dim`) so we can concat dynamically shaped pixels."
+        batch_size, num_frames, channel, resized_height, resized_width = videos.shape
+
+        # Check that videos have `num_frames` divisible by `temporal_patch_size`
+        if pad := -num_frames % temporal_patch_size:
+            repeats = videos[:, -1:].expand(-1, pad, -1, -1, -1)
+            videos = torch.cat((videos, repeats), dim=1)
+            num_frames += pad
+
+        grid_t = num_frames // temporal_patch_size
+        grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
+
+        patches = videos.view(
+            batch_size,
+            grid_t,
+            temporal_patch_size,
+            channel,
+            grid_h // merge_size,
+            merge_size,
+            patch_size,
+            grid_w // merge_size,
+            merge_size,
+            patch_size,
+        )
+        patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
+        flatten_patches = patches.reshape(
+            batch_size,
+            grid_t * grid_h * grid_w,
+            channel * temporal_patch_size * patch_size * patch_size,
+        )
+
+        return flatten_patches, grid_t, grid_h, grid_w
+
+    def _preprocess(
+        self,
+        videos: list["torch.Tensor"],
+        do_resize: bool,
+        size: SizeDict,
+        resample: "PILImageResampling | tvF.InterpolationMode | int | None",
+        do_rescale: bool,
+        rescale_factor: float,
+        do_normalize: bool,
+        image_mean: float | list[float] | None,
+        image_std: float | list[float] | None,
+        patch_size: int | None = None,
+        temporal_patch_size: int | None = None,
+        merge_size: int | None = None,
+        cap_pixels_per_frame: bool | None = None,
+        return_tensors: str | TensorType | None = None,
+        **kwargs,
+    ):
+        if cap_pixels_per_frame is None:
+            logger.warning_once(
+                "Lfm3Vl video processing does not apply the per-frame pixel cap the reference "
+                "implementation (qwen-vl-utils) applies, so some videos cost far more tokens than they "
+                "would there. In v5.22 the capped behavior will become the default and "
+                "`cap_pixels_per_frame` will be removed. Pass `cap_pixels_per_frame=True` to adopt the "
+                "reference behavior now, or `False` to keep the current behavior and silence this "
+                "warning."
+            )
+            cap_pixels_per_frame = False
+        # Group videos by size for batched resizing
+        grouped_videos, grouped_videos_index = group_videos_by_shape(videos)
+        resized_videos_grouped = {}
+        for shape, stacked_videos in grouped_videos.items():
+            if do_resize:
+                stacked_videos = self.resize(
+                    videos=stacked_videos,
+                    size=size,
+                    resample=resample,
+                    factor=patch_size * merge_size,
+                    temporal_factor=temporal_patch_size,
+                    cap_pixels_per_frame=cap_pixels_per_frame,
+                )
+            resized_videos_grouped[shape] = stacked_videos
+        resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index)
+
+        # Group videos by size for further processing
+        # Needed in case do_resize is False, or resize returns videos with different sizes
+        grouped_videos, grouped_videos_index = group_videos_by_shape(resized_videos)
+        processed_videos_grouped = {}
+        processed_grids = {}
+        for shape, stacked_videos in grouped_videos.items():
+            # Fused rescale and normalize
+            stacked_videos = self.rescale_and_normalize(
+                stacked_videos, do_rescale, rescale_factor, do_normalize, image_mean, image_std
+            )
+            patches, grid_t, grid_h, grid_w = self.patchify(
+                stacked_videos,
+                patch_size=patch_size,
+                merge_size=merge_size,
+                temporal_patch_size=temporal_patch_size,
+            )
+
+            processed_videos_grouped[shape] = patches
+            processed_grids[shape] = [[grid_t, grid_h, grid_w]] * len(stacked_videos)
+
+        processed_videos = reorder_videos(processed_videos_grouped, grouped_videos_index)
+        processed_grids = reorder_videos(processed_grids, grouped_videos_index)
+        pixel_values_videos = torch.cat(processed_videos, dim=0)
+        video_grid_thw = torch.tensor(processed_grids)
+
+        return BatchFeature(
+            data={"pixel_values_videos": pixel_values_videos, "video_grid_thw": video_grid_thw},
+            tensor_type=return_tensors,
+        )
+
+    def get_num_of_video_patches(self, num_frames: int, height: int, width: int, videos_kwargs=None):
+        """
+        A utility that returns number of video patches a given video size.
+
+        Args:
+            num_frames (`int`):
+                Number of frames in the input video.
+            height (`int`):
+                Height of the input video.
+            width (`int`):
+                Width of the input video.
+            videos_kwargs (`dict`, *optional*)
+                Any kwargs to override defaults of the video processor.
+        Returns:
+            `int`: Number of video patches per video.
+        """
+        videos_kwargs = videos_kwargs if videos_kwargs is not None else {}
+        size = videos_kwargs.get("size", None) or self.size
+        min_pixels = size["shortest_edge"]
+        max_pixels = size["longest_edge"]
+        patch_size = videos_kwargs.get("patch_size", None) or self.patch_size
+        merge_size = videos_kwargs.get("merge_size", None) or self.merge_size
+        temporal_patch_size = videos_kwargs.get("temporal_patch_size", None) or self.temporal_patch_size
+        cap_pixels_per_frame = videos_kwargs.get("cap_pixels_per_frame", None) or self.cap_pixels_per_frame
+
+        factor = patch_size * merge_size
+        if cap_pixels_per_frame:
+            # Keep the count in sync with `resize` when the per-frame cap is active.
+            frame_cap = self.max_video_tokens * factor * factor
+            pixels_per_frame = max(min(frame_cap, max_pixels // num_frames), int(min_pixels * 1.05))
+            max_pixels = pixels_per_frame * num_frames
+
+        resized_height, resized_width = smart_resize(
+            num_frames,
+            height,
+            width,
+            temporal_factor=temporal_patch_size,
+            factor=factor,
+            min_pixels=min_pixels,
+            max_pixels=max_pixels,
+        )
+        grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
+        grid_t = (num_frames + -num_frames % temporal_patch_size) // temporal_patch_size
+        return grid_t * grid_h * grid_w
+
+    def preprocess(self, videos, **kwargs):
+        _check_video_metadata_count(videos, kwargs.get("video_metadata"))
+        # the shared `preprocess` fills every unset option from the instance before `sample_frames` runs, so the
+        # caller's options are told apart from the instance ones here: a caller option beats an instance one, and a
+        # configured `num_frames` beats the default fps
+        num_frames, fps = kwargs.get("num_frames"), kwargs.get("fps")
+        if num_frames is not None and fps is not None:
+            raise ValueError("`num_frames` and `fps` are mutually exclusive arguments, please use only one!")
+        if num_frames is not None or (fps is None and self.num_frames is not None):
+            kwargs["fps"] = None
+        elif fps is not None:
+            kwargs["num_frames"] = None
+        return super().preprocess(videos, **kwargs)
+
+
+__all__ = ["Lfm3VlVideoProcessor"]