Upload Cosmos-Embed1-448p anomaly-detection fine-tune (LoRA, Vad-Reasoning)

config.json

@@ -0,0 +1,20 @@
+{
+  "architectures": [
+    "CosmosEmbed1"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_embed1.CosmosEmbed1Config",
+    "AutoProcessor": "preprocessing_embed1.CosmosEmbed1Processor",
+    "AutoModel": "modeling_embed1.CosmosEmbed1"
+  },
+  "model_type": "cosmos-embed1",
+  "embed_dim": 768,
+  "num_query_tokens": 32,
+  "max_txt_len": 128,
+  "num_video_frames": 8,
+  "resolution": 448,
+  "temporal_encoding_type": "neighboring_token_propagation",
+  "vocab_size": 30523,
+  "transformer_engine": false,
+  "use_fp8": false
+}

configuration_embed1.py

@@ -0,0 +1,69 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Configuration class for Cosmos-Embed1."""
+
+from typing import Any, Literal, Tuple, Union
+
+from transformers import AutoConfig, PretrainedConfig
+
+
+class CosmosEmbed1Config(PretrainedConfig):
+    model_type = "cosmos-embed1"
+
+    def __init__(
+        self,
+        embed_dim: int = 768,
+        num_query_tokens: int = 32,
+        max_txt_len: int = 128,
+        num_video_frames: int = 8,
+        temporal_encoding_type: Literal[
+            "neighboring_token_propagation", "temporal_parameter"
+        ] = "neighboring_token_propagation",
+        resolution: Union[int, Tuple[int, int]] = 224,
+        vocab_size: int = 30523,
+        transformer_engine: bool = False,
+        use_fp8: bool = False,
+        **kwargs: Any,
+    ) -> None:
+        """Configuration for `CosmosEmbed1Config`.
+
+        Args:
+            embed_dim (int): the dimension of extracted text-visual embeddings.
+            num_query_tokens (int): number of learnable query tokens.
+            max_txt_len (int): max length of text token sequences before truncation.
+            num_video_frames (int): number of input video frames.
+            temporal_encoding_type (str): temporal encoding module type.
+            resolution (Union[int, Tuple[int, int]]): input video frame resolution.
+                Can be an integer for square images (height=width) or a tuple of (height, width) for non-square.
+            vocab_size (int): vocab size for text tokenizer.
+                The default is from `bert-base-uncased` with an extra [DEC] token.
+            transformer_engine (bool): whether to use TransformerEngine for acceleration.
+            use_fp8 (bool): whether to use FP8 precision (requires transformer_engine=True).
+        """
+        super().__init__(**kwargs)
+
+        self.embed_dim = embed_dim
+        self.num_query_tokens = num_query_tokens
+        self.max_txt_len = max_txt_len
+        self.num_video_frames = num_video_frames
+        self.temporal_encoding_type = temporal_encoding_type
+        self.resolution = resolution
+        self.vocab_size = vocab_size
+        self.transformer_engine = transformer_engine
+        self.use_fp8 = use_fp8
+
+
+AutoConfig.register("cosmos-embed1", CosmosEmbed1Config)

export_config.yaml

@@ -0,0 +1,238 @@
+wandb:
+  enable: false
+  project: cosmos_embed1
+  group: ''
+  name: ''
+  tags: []
+  save_code: false
+  api_key: ''
+model:
+  network:
+    visual_encoder:
+      type: eva_vit_g
+      img_size: 224
+      pretrained: false
+      use_fp8: false
+      transformer_engine: false
+      checkpoint_activations: false
+      checkpoint_attention: false
+    embed_dim: 768
+    num_query_tokens: 32
+    max_txt_len: 128
+    num_video_frames: 8
+    spatial_resolution:
+    - 448
+    - 448
+    temporal_encoding_type: neighboring_token_propagation
+    contrastive_type: clip
+    qformer_pretrain_ckpt: null
+    query_pooling_type: avg
+    pretrained_text_encoder: false
+    pretrained_visual_encoder: false
+    num_heldout_frames: 0
+  pretrained_model_path: null
+  pretrained_model_strict: true
+  precision: fp32
+  input_hw:
+  - 512
+  - 512
+  fsdp:
+    enabled: false
+    shard_size: null
+    replica_size: null
+  fsdp_shard_size: 8
+dataset:
+  train_dataset:
+    dataset_type: mock
+    metadata: null
+    data_root: null
+    num_video_frames: 8
+    resolution:
+    - 224
+    - 224
+    batch_size: 4
+    workers: 4
+    drop_last: true
+    prefetch_factor: 2
+    pin_memory: true
+    split: null
+    random_caption: false
+    path_prefix_mapping: {}
+    skip_missing_files: true
+    caption_field: anomaly_type
+    mp4_urls: null
+    caption_to_label: {}
+    chunk_size_sec: 5.0
+  val_dataset:
+    dataset_type: mock
+    metadata: null
+    data_root: null
+    num_video_frames: 8
+    resolution:
+    - 224
+    - 224
+    batch_size: 4
+    workers: 4
+    drop_last: true
+    prefetch_factor: 2
+    pin_memory: true
+    split: null
+    random_caption: false
+    path_prefix_mapping: {}
+    skip_missing_files: true
+    caption_field: anomaly_type
+    mp4_urls: null
+    caption_to_label: {}
+    chunk_size_sec: 5.0
+  test_dataset:
+    dataset_type: mock
+    metadata: null
+    data_root: null
+    num_video_frames: 8
+    resolution:
+    - 224
+    - 224
+    batch_size: 4
+    workers: 4
+    drop_last: true
+    prefetch_factor: 2
+    pin_memory: true
+    split: null
+    random_caption: false
+    path_prefix_mapping: {}
+    skip_missing_files: true
+    caption_field: anomaly_type
+    mp4_urls: null
+    caption_to_label: {}
+    chunk_size_sec: 5.0
+  inference_dataset:
+    dataset_type: mock
+    metadata: null
+    data_root: null
+    num_video_frames: 8
+    resolution:
+    - 224
+    - 224
+    batch_size: 4
+    workers: 4
+    drop_last: true
+    prefetch_factor: 2
+    pin_memory: true
+    split: null
+    random_caption: false
+    path_prefix_mapping: {}
+    skip_missing_files: true
+    caption_field: anomaly_type
+    mp4_urls: null
+    caption_to_label: {}
+    chunk_size_sec: 5.0
+train:
+  optim:
+    optim: adamw
+    lr: 1.0e-05
+    weight_decay: 1.0e-05
+    betas:
+    - 0.9
+    - 0.98
+    warmup_steps: 1000
+    policy: cosine
+    lr_decay_iters: 50000
+  loss_weights:
+    contrastive_loss: 1.0
+    captioning_loss: 1.0
+    matching_loss: 1.0
+  lora:
+    enabled: false
+    lora_rank: 8
+    lora_alpha: 16
+    lora_dropout: 0.1
+    bias: none
+    use_rslora: false
+    use_dora: false
+    target_modules: []
+    modules_to_save: []
+  seed: 1234
+  max_iter: 50000
+  num_nodes: 1
+  num_gpus: 1
+  gpu_ids:
+  - 0
+  validation_iter: 1000
+  checkpoint_iter: 1000
+  clip_grad_norm: 0.0
+  precision: bf16
+  resume_training_checkpoint_path: null
+  callbacks:
+    wandb: {}
+    clamp_logit_scale: {}
+    logit_parameters_monitor: {}
+    iter_speed:
+      every_n: 50
+      save_s3: false
+    gradient_clip:
+      clip_norm: 3.0
+    grad_norm_monitor:
+      every_n: 500
+      verbose: false
+    spectral_norm_monitor:
+      every_n: 1000
+      verbose: true
+    ema: {}
+    log_losses:
+      every_n: 50
+      verbose: true
+    text_frames_visualizer:
+      every_n: 500
+    pca_feature_map_visualizer:
+      every_n: 500
+  max_val_iter: null
+  freeze_visual_encoder: true
+  use_captioning_loss: true
+  use_text_matching_loss: false
+  ema:
+    enabled: false
+    beta: 0.9999
+  spectral_reparam: false
+  damp:
+    enabled: false
+    beta: 0.1
+    mode: const
+  load_training_state: false
+  strict_resume: false
+evaluate:
+  checkpoint: null
+  max_val_batches: -1
+  num_gpus: 1
+  callbacks:
+    topk_classification: true
+    embedding_visualization: false
+    top_k_values:
+    - 1
+    - 3
+    - 5
+    - 10
+    max_eval_samples: 2000
+  load_dataset_pkl: null
+  save_dataset_pkl: null
+inference:
+  checkpoint: null
+  query:
+    input_videos: []
+    input_texts: []
+  num_gpus: 1
+  k: 5
+  load_dataset_pkl: null
+  save_dataset_pkl: null
+export:
+  checkpoint: /workspace/alicli/experiments/cosmos_embed1_finetune/finetune/finetune_448p_hf/train/checkpoints/iter_000006000.pt
+  onnx_file: null
+  mode: huggingface
+  opset_version: 17
+  batch_size: 1
+  on_cpu: true
+  verbose: false
+  simplify: false
+  hf_output_dir: /workspace/alicli/experiments/cosmos_embed1_finetune/finetune/finetune_448p_hf/cosmos_embed1_448p_6000iter_hf
+results_dir: /workspace/alicli/experiments/cosmos_embed1_finetune/finetune/finetune_448p_hf
+encryption_key: null
+model_name: cosmos_embed1

model-00001-of-00010.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe3ea01cd802f56b560fc34b1a52f417821eb534d694f2e36c8c4f4385cc12cd
+size 517488448

model-00002-of-00010.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8f7f924243c33217a764f817c5d27ba8ffff5f9c15a2e42abd788468b3eb2cf
+size 493842736

model-00003-of-00010.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62c1e0b262e862efac6c2aa083c25a897363a2bd116aefa44dc259b810513878
+size 505010128

model-00004-of-00010.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6bb2c6eb056cd177ca4c4e4b7b5fe2b9c4e47b61659c13da33f03abfe522851
+size 505010136

model-00005-of-00010.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:277dd3fbe9b5f8197c117244edff18bc220a6762294d6d5e9bfe5dd2380e0dc2
+size 505010120

model-00006-of-00010.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98e45d9d21e0fc0af325908c5654825f9d3aa5384e1754a5bdee1c3ff4b17c0b
+size 505010136

model-00007-of-00010.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7d65fc3291c03329047b0b6214353fb107a079dcef53c3c7d05ef39ba19b663
+size 505010120

model-00008-of-00010.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4dcf217ab3159b2ec92465e0d266d3bd18fff55c5dd2a6264edd6f13ab165fd1
+size 505010136

model-00009-of-00010.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e028831d5bef332882391436d7ebb104fcb95dcdf75f9d56b47e96623c10b144
+size 505010088

model-00010-of-00010.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15ef5ea4966b056b4a5070307ff37ff438abad5f869c77a688f0660401a942c1
+size 245726200

model.safetensors.index.json

@@ -0,0 +1,827 @@
+{
+  "metadata": {
+    "total_size": 4792032744
+  },
+  "weight_map": {
+    "itm_proj.bias": "model-00001-of-00010.safetensors",
+    "itm_proj.weight": "model-00001-of-00010.safetensors",
+    "ln_vision.bias": "model-00001-of-00010.safetensors",
+    "ln_vision.weight": "model-00001-of-00010.safetensors",
+    "logit_bias": "model-00001-of-00010.safetensors",
+    "logit_scale": "model-00001-of-00010.safetensors",
+    "qformer.bert.embeddings.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.embeddings.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.embeddings.position_embeddings.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.embeddings.position_ids": "model-00001-of-00010.safetensors",
+    "qformer.bert.embeddings.word_embeddings.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.attention.output.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.attention.output.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.attention.output.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.attention.output.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.attention.self.key.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.attention.self.key.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.attention.self.query.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.attention.self.query.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.attention.self.value.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.attention.self.value.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.crossattention.output.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.crossattention.output.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.crossattention.output.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.crossattention.output.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.crossattention.self.key.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.crossattention.self.key.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.crossattention.self.query.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.crossattention.self.query.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.crossattention.self.value.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.crossattention.self.value.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.intermediate.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.intermediate.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.intermediate_query.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.intermediate_query.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.output.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.output.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.output.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.output.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.output_query.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.output_query.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.output_query.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.0.output_query.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.1.attention.output.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.1.attention.output.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.1.attention.output.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.1.attention.output.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.1.attention.self.key.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.1.attention.self.key.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.1.attention.self.query.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.1.attention.self.query.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.1.attention.self.value.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.1.attention.self.value.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.1.intermediate.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.1.intermediate.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.1.intermediate_query.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.1.intermediate_query.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.1.output.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.1.output.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.1.output.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.1.output.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.1.output_query.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.1.output_query.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.1.output_query.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.1.output_query.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.attention.output.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.attention.output.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.attention.output.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.attention.output.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.attention.self.key.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.attention.self.key.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.attention.self.query.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.attention.self.query.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.attention.self.value.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.attention.self.value.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.crossattention.output.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.crossattention.output.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.crossattention.output.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.crossattention.output.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.crossattention.self.key.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.crossattention.self.key.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.crossattention.self.query.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.crossattention.self.query.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.crossattention.self.value.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.crossattention.self.value.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.intermediate.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.intermediate.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.intermediate_query.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.intermediate_query.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.output.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.output.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.output.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.output.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.output_query.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.output_query.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.output_query.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.10.output_query.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.11.attention.output.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.11.attention.output.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.11.attention.output.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.11.attention.output.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.11.attention.self.key.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.11.attention.self.key.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.11.attention.self.query.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.11.attention.self.query.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.11.attention.self.value.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.11.attention.self.value.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.11.intermediate.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.11.intermediate.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.11.intermediate_query.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.11.intermediate_query.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.11.output.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.11.output.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.11.output.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.11.output.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.11.output_query.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.11.output_query.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.11.output_query.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.11.output_query.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.attention.output.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.attention.output.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.attention.output.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.attention.output.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.attention.self.key.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.attention.self.key.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.attention.self.query.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.attention.self.query.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.attention.self.value.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.attention.self.value.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.crossattention.output.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.crossattention.output.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.crossattention.output.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.crossattention.output.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.crossattention.self.key.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.crossattention.self.key.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.crossattention.self.query.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.crossattention.self.query.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.crossattention.self.value.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.crossattention.self.value.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.intermediate.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.intermediate.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.intermediate_query.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.intermediate_query.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.output.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.output.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.output.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.output.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.output_query.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.output_query.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.output_query.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.2.output_query.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.3.attention.output.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.3.attention.output.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.3.attention.output.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.3.attention.output.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.3.attention.self.key.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.3.attention.self.key.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.3.attention.self.query.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.3.attention.self.query.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.3.attention.self.value.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.3.attention.self.value.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.3.intermediate.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.3.intermediate.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.3.intermediate_query.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.3.intermediate_query.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.3.output.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.3.output.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.3.output.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.3.output.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.3.output_query.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.3.output_query.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.3.output_query.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.3.output_query.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.attention.output.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.attention.output.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.attention.output.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.attention.output.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.attention.self.key.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.attention.self.key.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.attention.self.query.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.attention.self.query.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.attention.self.value.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.attention.self.value.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.crossattention.output.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.crossattention.output.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.crossattention.output.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.crossattention.output.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.crossattention.self.key.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.crossattention.self.key.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.crossattention.self.query.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.crossattention.self.query.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.crossattention.self.value.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.crossattention.self.value.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.intermediate.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.intermediate.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.intermediate_query.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.intermediate_query.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.output.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.output.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.output.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.output.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.output_query.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.output_query.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.output_query.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.4.output_query.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.5.attention.output.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.5.attention.output.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.5.attention.output.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.5.attention.output.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.5.attention.self.key.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.5.attention.self.key.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.5.attention.self.query.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.5.attention.self.query.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.5.attention.self.value.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.5.attention.self.value.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.5.intermediate.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.5.intermediate.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.5.intermediate_query.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.5.intermediate_query.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.5.output.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.5.output.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.5.output.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.5.output.dense.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.5.output_query.LayerNorm.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.5.output_query.LayerNorm.weight": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.5.output_query.dense.bias": "model-00001-of-00010.safetensors",
+    "qformer.bert.encoder.layer.5.output_query.dense.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.attention.output.LayerNorm.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.attention.output.LayerNorm.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.attention.output.dense.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.attention.output.dense.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.attention.self.key.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.attention.self.key.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.attention.self.query.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.attention.self.query.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.attention.self.value.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.attention.self.value.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.crossattention.output.LayerNorm.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.crossattention.output.LayerNorm.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.crossattention.output.dense.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.crossattention.output.dense.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.crossattention.self.key.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.crossattention.self.key.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.crossattention.self.query.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.crossattention.self.query.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.crossattention.self.value.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.crossattention.self.value.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.intermediate.dense.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.intermediate.dense.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.intermediate_query.dense.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.intermediate_query.dense.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.output.LayerNorm.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.output.LayerNorm.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.output.dense.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.output.dense.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.output_query.LayerNorm.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.output_query.LayerNorm.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.output_query.dense.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.6.output_query.dense.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.7.attention.output.LayerNorm.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.7.attention.output.LayerNorm.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.7.attention.output.dense.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.7.attention.output.dense.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.7.attention.self.key.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.7.attention.self.key.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.7.attention.self.query.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.7.attention.self.query.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.7.attention.self.value.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.7.attention.self.value.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.7.intermediate.dense.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.7.intermediate.dense.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.7.intermediate_query.dense.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.7.intermediate_query.dense.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.7.output.LayerNorm.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.7.output.LayerNorm.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.7.output.dense.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.7.output.dense.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.7.output_query.LayerNorm.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.7.output_query.LayerNorm.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.7.output_query.dense.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.7.output_query.dense.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.attention.output.LayerNorm.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.attention.output.LayerNorm.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.attention.output.dense.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.attention.output.dense.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.attention.self.key.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.attention.self.key.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.attention.self.query.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.attention.self.query.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.attention.self.value.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.attention.self.value.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.crossattention.output.LayerNorm.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.crossattention.output.LayerNorm.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.crossattention.output.dense.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.crossattention.output.dense.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.crossattention.self.key.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.crossattention.self.key.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.crossattention.self.query.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.crossattention.self.query.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.crossattention.self.value.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.crossattention.self.value.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.intermediate.dense.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.intermediate.dense.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.intermediate_query.dense.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.intermediate_query.dense.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.output.LayerNorm.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.output.LayerNorm.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.output.dense.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.output.dense.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.output_query.LayerNorm.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.output_query.LayerNorm.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.output_query.dense.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.8.output_query.dense.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.9.attention.output.LayerNorm.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.9.attention.output.LayerNorm.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.9.attention.output.dense.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.9.attention.output.dense.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.9.attention.self.key.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.9.attention.self.key.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.9.attention.self.query.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.9.attention.self.query.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.9.attention.self.value.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.9.attention.self.value.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.9.intermediate.dense.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.9.intermediate.dense.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.9.intermediate_query.dense.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.9.intermediate_query.dense.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.9.output.LayerNorm.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.9.output.LayerNorm.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.9.output.dense.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.9.output.dense.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.9.output_query.LayerNorm.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.9.output_query.LayerNorm.weight": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.9.output_query.dense.bias": "model-00002-of-00010.safetensors",
+    "qformer.bert.encoder.layer.9.output_query.dense.weight": "model-00002-of-00010.safetensors",
+    "qformer.cls.predictions.bias": "model-00002-of-00010.safetensors",
+    "qformer.cls.predictions.decoder.bias": "model-00002-of-00010.safetensors",
+    "qformer.cls.predictions.decoder.weight": "model-00002-of-00010.safetensors",
+    "qformer.cls.predictions.transform.LayerNorm.bias": "model-00002-of-00010.safetensors",
+    "qformer.cls.predictions.transform.LayerNorm.weight": "model-00002-of-00010.safetensors",
+    "qformer.cls.predictions.transform.dense.bias": "model-00002-of-00010.safetensors",
+    "qformer.cls.predictions.transform.dense.weight": "model-00002-of-00010.safetensors",
+    "query_tokens": "model-00002-of-00010.safetensors",
+    "temporal_encoding.encoding.temp_embed": "model-00002-of-00010.safetensors",
+    "text_proj.bias": "model-00002-of-00010.safetensors",
+    "text_proj.weight": "model-00002-of-00010.safetensors",
+    "vision_proj.bias": "model-00002-of-00010.safetensors",
+    "vision_proj.weight": "model-00002-of-00010.safetensors",
+    "visual_encoder.blocks.0.attn.proj.bias": "model-00002-of-00010.safetensors",
+    "visual_encoder.blocks.0.attn.proj.weight": "model-00002-of-00010.safetensors",
+    "visual_encoder.blocks.0.attn.qkv.bias": "model-00002-of-00010.safetensors",
+    "visual_encoder.blocks.0.attn.qkv.weight": "model-00002-of-00010.safetensors",
+    "visual_encoder.blocks.0.mlp.fc1.bias": "model-00002-of-00010.safetensors",
+    "visual_encoder.blocks.0.mlp.fc1.weight": "model-00002-of-00010.safetensors",
+    "visual_encoder.blocks.0.mlp.fc2.bias": "model-00002-of-00010.safetensors",
+    "visual_encoder.blocks.0.mlp.fc2.weight": "model-00002-of-00010.safetensors",
+    "visual_encoder.blocks.0.norm1.bias": "model-00002-of-00010.safetensors",
+    "visual_encoder.blocks.0.norm1.weight": "model-00002-of-00010.safetensors",
+    "visual_encoder.blocks.0.norm2.bias": "model-00002-of-00010.safetensors",
+    "visual_encoder.blocks.0.norm2.weight": "model-00002-of-00010.safetensors",
+    "visual_encoder.blocks.1.attn.proj.bias": "model-00002-of-00010.safetensors",
+    "visual_encoder.blocks.1.attn.proj.weight": "model-00002-of-00010.safetensors",
+    "visual_encoder.blocks.1.attn.qkv.bias": "model-00002-of-00010.safetensors",
+    "visual_encoder.blocks.1.attn.qkv.weight": "model-00002-of-00010.safetensors",
+    "visual_encoder.blocks.1.mlp.fc1.bias": "model-00002-of-00010.safetensors",
+    "visual_encoder.blocks.1.mlp.fc1.weight": "model-00002-of-00010.safetensors",
+    "visual_encoder.blocks.1.mlp.fc2.bias": "model-00002-of-00010.safetensors",
+    "visual_encoder.blocks.1.mlp.fc2.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.1.norm1.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.1.norm1.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.1.norm2.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.1.norm2.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.10.attn.proj.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.10.attn.proj.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.10.attn.qkv.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.10.attn.qkv.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.10.mlp.fc1.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.10.mlp.fc1.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.10.mlp.fc2.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.10.mlp.fc2.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.10.norm1.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.10.norm1.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.10.norm2.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.10.norm2.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.11.attn.proj.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.11.attn.proj.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.11.attn.qkv.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.11.attn.qkv.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.11.mlp.fc1.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.11.mlp.fc1.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.11.mlp.fc2.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.11.mlp.fc2.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.11.norm1.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.11.norm1.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.11.norm2.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.11.norm2.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.12.attn.proj.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.12.attn.proj.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.12.attn.qkv.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.12.attn.qkv.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.12.mlp.fc1.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.12.mlp.fc1.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.12.mlp.fc2.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.12.mlp.fc2.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.12.norm1.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.12.norm1.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.12.norm2.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.12.norm2.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.13.attn.proj.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.13.attn.proj.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.13.attn.qkv.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.13.attn.qkv.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.13.mlp.fc1.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.13.mlp.fc1.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.13.mlp.fc2.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.13.mlp.fc2.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.13.norm1.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.13.norm1.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.13.norm2.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.13.norm2.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.14.attn.proj.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.14.attn.proj.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.14.attn.qkv.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.14.attn.qkv.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.14.mlp.fc1.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.14.mlp.fc1.weight": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.14.mlp.fc2.bias": "model-00003-of-00010.safetensors",
+    "visual_encoder.blocks.14.mlp.fc2.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.14.norm1.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.14.norm1.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.14.norm2.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.14.norm2.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.15.attn.proj.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.15.attn.proj.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.15.attn.qkv.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.15.attn.qkv.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.15.mlp.fc1.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.15.mlp.fc1.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.15.mlp.fc2.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.15.mlp.fc2.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.15.norm1.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.15.norm1.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.15.norm2.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.15.norm2.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.16.attn.proj.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.16.attn.proj.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.16.attn.qkv.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.16.attn.qkv.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.16.mlp.fc1.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.16.mlp.fc1.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.16.mlp.fc2.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.16.mlp.fc2.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.16.norm1.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.16.norm1.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.16.norm2.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.16.norm2.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.17.attn.proj.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.17.attn.proj.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.17.attn.qkv.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.17.attn.qkv.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.17.mlp.fc1.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.17.mlp.fc1.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.17.mlp.fc2.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.17.mlp.fc2.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.17.norm1.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.17.norm1.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.17.norm2.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.17.norm2.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.18.attn.proj.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.18.attn.proj.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.18.attn.qkv.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.18.attn.qkv.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.18.mlp.fc1.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.18.mlp.fc1.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.18.mlp.fc2.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.18.mlp.fc2.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.18.norm1.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.18.norm1.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.18.norm2.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.18.norm2.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.19.attn.proj.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.19.attn.proj.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.19.attn.qkv.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.19.attn.qkv.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.19.mlp.fc1.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.19.mlp.fc1.weight": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.19.mlp.fc2.bias": "model-00004-of-00010.safetensors",
+    "visual_encoder.blocks.19.mlp.fc2.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.19.norm1.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.19.norm1.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.19.norm2.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.19.norm2.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.2.attn.proj.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.2.attn.proj.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.2.attn.qkv.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.2.attn.qkv.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.2.mlp.fc1.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.2.mlp.fc1.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.2.mlp.fc2.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.2.mlp.fc2.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.2.norm1.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.2.norm1.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.2.norm2.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.2.norm2.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.20.attn.proj.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.20.attn.proj.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.20.attn.qkv.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.20.attn.qkv.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.20.mlp.fc1.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.20.mlp.fc1.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.20.mlp.fc2.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.20.mlp.fc2.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.20.norm1.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.20.norm1.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.20.norm2.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.20.norm2.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.21.attn.proj.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.21.attn.proj.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.21.attn.qkv.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.21.attn.qkv.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.21.mlp.fc1.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.21.mlp.fc1.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.21.mlp.fc2.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.21.mlp.fc2.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.21.norm1.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.21.norm1.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.21.norm2.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.21.norm2.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.22.attn.proj.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.22.attn.proj.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.22.attn.qkv.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.22.attn.qkv.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.22.mlp.fc1.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.22.mlp.fc1.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.22.mlp.fc2.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.22.mlp.fc2.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.22.norm1.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.22.norm1.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.22.norm2.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.22.norm2.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.23.attn.proj.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.23.attn.proj.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.23.attn.qkv.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.23.attn.qkv.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.23.mlp.fc1.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.23.mlp.fc1.weight": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.23.mlp.fc2.bias": "model-00005-of-00010.safetensors",
+    "visual_encoder.blocks.23.mlp.fc2.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.23.norm1.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.23.norm1.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.23.norm2.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.23.norm2.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.24.attn.proj.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.24.attn.proj.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.24.attn.qkv.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.24.attn.qkv.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.24.mlp.fc1.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.24.mlp.fc1.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.24.mlp.fc2.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.24.mlp.fc2.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.24.norm1.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.24.norm1.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.24.norm2.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.24.norm2.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.25.attn.proj.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.25.attn.proj.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.25.attn.qkv.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.25.attn.qkv.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.25.mlp.fc1.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.25.mlp.fc1.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.25.mlp.fc2.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.25.mlp.fc2.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.25.norm1.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.25.norm1.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.25.norm2.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.25.norm2.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.26.attn.proj.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.26.attn.proj.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.26.attn.qkv.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.26.attn.qkv.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.26.mlp.fc1.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.26.mlp.fc1.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.26.mlp.fc2.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.26.mlp.fc2.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.26.norm1.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.26.norm1.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.26.norm2.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.26.norm2.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.27.attn.proj.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.27.attn.proj.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.27.attn.qkv.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.27.attn.qkv.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.27.mlp.fc1.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.27.mlp.fc1.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.27.mlp.fc2.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.27.mlp.fc2.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.27.norm1.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.27.norm1.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.27.norm2.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.27.norm2.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.28.attn.proj.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.28.attn.proj.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.28.attn.qkv.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.28.attn.qkv.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.28.mlp.fc1.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.28.mlp.fc1.weight": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.28.mlp.fc2.bias": "model-00006-of-00010.safetensors",
+    "visual_encoder.blocks.28.mlp.fc2.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.28.norm1.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.28.norm1.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.28.norm2.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.28.norm2.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.29.attn.proj.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.29.attn.proj.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.29.attn.qkv.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.29.attn.qkv.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.29.mlp.fc1.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.29.mlp.fc1.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.29.mlp.fc2.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.29.mlp.fc2.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.29.norm1.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.29.norm1.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.29.norm2.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.29.norm2.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.3.attn.proj.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.3.attn.proj.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.3.attn.qkv.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.3.attn.qkv.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.3.mlp.fc1.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.3.mlp.fc1.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.3.mlp.fc2.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.3.mlp.fc2.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.3.norm1.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.3.norm1.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.3.norm2.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.3.norm2.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.30.attn.proj.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.30.attn.proj.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.30.attn.qkv.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.30.attn.qkv.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.30.mlp.fc1.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.30.mlp.fc1.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.30.mlp.fc2.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.30.mlp.fc2.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.30.norm1.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.30.norm1.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.30.norm2.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.30.norm2.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.31.attn.proj.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.31.attn.proj.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.31.attn.qkv.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.31.attn.qkv.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.31.mlp.fc1.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.31.mlp.fc1.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.31.mlp.fc2.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.31.mlp.fc2.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.31.norm1.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.31.norm1.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.31.norm2.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.31.norm2.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.32.attn.proj.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.32.attn.proj.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.32.attn.qkv.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.32.attn.qkv.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.32.mlp.fc1.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.32.mlp.fc1.weight": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.32.mlp.fc2.bias": "model-00007-of-00010.safetensors",
+    "visual_encoder.blocks.32.mlp.fc2.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.32.norm1.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.32.norm1.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.32.norm2.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.32.norm2.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.33.attn.proj.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.33.attn.proj.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.33.attn.qkv.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.33.attn.qkv.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.33.mlp.fc1.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.33.mlp.fc1.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.33.mlp.fc2.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.33.mlp.fc2.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.33.norm1.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.33.norm1.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.33.norm2.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.33.norm2.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.34.attn.proj.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.34.attn.proj.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.34.attn.qkv.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.34.attn.qkv.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.34.mlp.fc1.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.34.mlp.fc1.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.34.mlp.fc2.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.34.mlp.fc2.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.34.norm1.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.34.norm1.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.34.norm2.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.34.norm2.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.35.attn.proj.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.35.attn.proj.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.35.attn.qkv.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.35.attn.qkv.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.35.mlp.fc1.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.35.mlp.fc1.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.35.mlp.fc2.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.35.mlp.fc2.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.35.norm1.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.35.norm1.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.35.norm2.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.35.norm2.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.36.attn.proj.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.36.attn.proj.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.36.attn.qkv.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.36.attn.qkv.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.36.mlp.fc1.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.36.mlp.fc1.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.36.mlp.fc2.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.36.mlp.fc2.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.36.norm1.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.36.norm1.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.36.norm2.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.36.norm2.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.37.attn.proj.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.37.attn.proj.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.37.attn.qkv.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.37.attn.qkv.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.37.mlp.fc1.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.37.mlp.fc1.weight": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.37.mlp.fc2.bias": "model-00008-of-00010.safetensors",
+    "visual_encoder.blocks.37.mlp.fc2.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.37.norm1.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.37.norm1.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.37.norm2.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.37.norm2.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.38.attn.proj.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.38.attn.proj.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.38.attn.qkv.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.38.attn.qkv.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.38.mlp.fc1.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.38.mlp.fc1.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.38.mlp.fc2.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.38.mlp.fc2.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.38.norm1.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.38.norm1.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.38.norm2.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.38.norm2.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.4.attn.proj.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.4.attn.proj.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.4.attn.qkv.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.4.attn.qkv.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.4.mlp.fc1.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.4.mlp.fc1.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.4.mlp.fc2.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.4.mlp.fc2.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.4.norm1.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.4.norm1.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.4.norm2.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.4.norm2.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.5.attn.proj.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.5.attn.proj.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.5.attn.qkv.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.5.attn.qkv.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.5.mlp.fc1.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.5.mlp.fc1.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.5.mlp.fc2.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.5.mlp.fc2.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.5.norm1.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.5.norm1.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.5.norm2.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.5.norm2.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.6.attn.proj.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.6.attn.proj.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.6.attn.qkv.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.6.attn.qkv.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.6.mlp.fc1.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.6.mlp.fc1.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.6.mlp.fc2.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.6.mlp.fc2.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.6.norm1.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.6.norm1.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.6.norm2.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.6.norm2.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.7.attn.proj.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.7.attn.proj.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.7.attn.qkv.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.7.attn.qkv.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.7.mlp.fc1.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.7.mlp.fc1.weight": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.7.mlp.fc2.bias": "model-00009-of-00010.safetensors",
+    "visual_encoder.blocks.7.mlp.fc2.weight": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.7.norm1.bias": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.7.norm1.weight": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.7.norm2.bias": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.7.norm2.weight": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.8.attn.proj.bias": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.8.attn.proj.weight": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.8.attn.qkv.bias": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.8.attn.qkv.weight": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.8.mlp.fc1.bias": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.8.mlp.fc1.weight": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.8.mlp.fc2.bias": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.8.mlp.fc2.weight": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.8.norm1.bias": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.8.norm1.weight": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.8.norm2.bias": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.8.norm2.weight": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.9.attn.proj.bias": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.9.attn.proj.weight": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.9.attn.qkv.bias": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.9.attn.qkv.weight": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.9.mlp.fc1.bias": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.9.mlp.fc1.weight": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.9.mlp.fc2.bias": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.9.mlp.fc2.weight": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.9.norm1.bias": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.9.norm1.weight": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.9.norm2.bias": "model-00010-of-00010.safetensors",
+    "visual_encoder.blocks.9.norm2.weight": "model-00010-of-00010.safetensors",
+    "visual_encoder.cls_token": "model-00010-of-00010.safetensors",
+    "visual_encoder.patch_embed.proj.bias": "model-00010-of-00010.safetensors",
+    "visual_encoder.patch_embed.proj.weight": "model-00010-of-00010.safetensors",
+    "visual_encoder.pos_embed": "model-00010-of-00010.safetensors"
+  }
+}

modeling_embed1.py

@@ -0,0 +1,261 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Cosmos-Embed1 text+video embedder."""
+
+import math
+from copy import deepcopy
+
+import torch
+from einops import rearrange
+from torch import nn
+from torch.nn import functional as F
+from transformers import AutoModel, PreTrainedModel
+
+from .configuration_embed1 import CosmosEmbed1Config
+from .modeling_outputs import TextEmbedderOutput, TextVideoEmbedderOutput, VideoEmbedderOutput
+from .modeling_qformer import BertLMHeadModel, load_qformer
+from .modeling_utils import EncodingFactory, rank0_first
+from .modeling_vit import EvaViTG
+
+
+class CosmosEmbed1(PreTrainedModel):
+    config_class = CosmosEmbed1Config
+
+    def __init__(self, config: CosmosEmbed1Config) -> None:
+        """Cosmos-Embed1 video embedder constructor.
+
+        Args:
+            config (CosmosEmbed1Config): Model configuration.
+        """
+        super().__init__(config)
+
+        self.embed_dim = config.embed_dim
+        self.num_query_tokens = config.num_query_tokens
+        self.num_video_frames = config.num_video_frames
+        self.temporal_encoding_type = config.temporal_encoding_type
+        self.resolution = config.resolution
+        self.vocab_size = config.vocab_size
+        self.transformer_engine = config.transformer_engine
+        self.use_fp8 = config.use_fp8
+
+        # visual encoder initialization
+        self.register_buffer(
+            "normalization_mean",
+            torch.tensor([0.485, 0.456, 0.406]).view(1, 1, 3, 1, 1),
+            persistent=False,
+        )
+        self.register_buffer(
+            "normalization_std",
+            torch.tensor([0.229, 0.224, 0.225]).view(1, 1, 3, 1, 1),
+            persistent=False,
+        )
+        self.visual_encoder = EvaViTG(
+            img_size=self.resolution,
+            transformer_engine=self.transformer_engine,
+            use_fp8=self.use_fp8,
+        )
+        self.ln_vision = nn.LayerNorm(self.visual_encoder.embed_dim)
+
+        # qformer initialization
+        self.qformer, self.query_tokens = self._init_qformer(
+            num_query_tokens=self.num_query_tokens,
+            encoder_width=self.visual_encoder.embed_dim,
+            vocab_size=self.vocab_size,
+        )
+        # self.qformer.
+        state_dict = self.qformer.state_dict()
+        for name, param in self.qformer.named_parameters():
+            if "_query" in name:
+                key_orig = name.replace("_query", "")
+                param.data.copy_(state_dict[key_orig])
+
+        # temporal encoding
+        self.temporal_encoding = EncodingFactory(
+            self.temporal_encoding_type,
+            embed_dim=self.visual_encoder.embed_dim,
+            max_len=self.num_video_frames,
+        )
+
+        # output projections
+        self.vision_proj = nn.Linear(self.qformer.config.hidden_size, self.embed_dim)
+        self.text_proj = nn.Linear(self.qformer.config.hidden_size, self.embed_dim)
+        self.itm_proj = nn.Linear(self.qformer.config.hidden_size, 2)
+        # initialize logit scale/bias like SigLIP (as per Table 4 in https://arxiv.org/pdf/2303.15343)
+        self.logit_scale = nn.Parameter(torch.tensor(math.log(10.0)))
+        self.logit_bias = nn.Parameter(torch.tensor(-10.0))
+
+    @property
+    def hidden_dim(self) -> int:
+        return self.visual_encoder.embed_dim
+
+    @torch.jit.ignore
+    def no_weight_decay(self) -> set:
+        ret = {"logit_scale", "logit_bias"}
+        return ret
+
+    def forward(
+        self,
+        videos: torch.FloatTensor,
+        input_ids: torch.LongTensor,
+        attention_mask: torch.FloatTensor,
+    ) -> TextVideoEmbedderOutput:
+        """Forward function for `ComosEmbed1`.
+
+        Args:
+            videos (`torch.Tensor` of shape `(batch_size, num_frames, RGB, height, width)`):
+                batched videos with fixed number of RGB frames.
+            input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+                Indices of input sequence tokens in the vocabulary.
+                Indices can be obtained by using [`AutoTokenizer`, `CosmosEmbed1Tokenizer`].
+            attention_mask: (`torch.Tensor` of shape `(batch_size, sequence_length)`):
+                Mask to avoid performing attention on padding token indices.
+                Mask values select in `[0, 1]`.
+                - 1 for tokens that are **not masked**.
+                - 0 for tokens that are **masked**.
+        """
+        video_output = self.get_video_embeddings(videos)
+        text_output = self.get_text_embeddings(input_ids, attention_mask)
+        return TextVideoEmbedderOutput(**video_output, **text_output)
+
+    def get_video_embeddings(self, videos: torch.Tensor) -> VideoEmbedderOutput:
+        videos = (videos - self.normalization_mean) / self.normalization_std
+        batch_size, num_frames, _, H, W = videos.shape
+        frame_batch = rearrange(videos, "b t c h w -> (b t) c h w")
+
+        # process video frames through ViT
+        visual_embs = self.visual_encoder(frame_batch)
+        visual_embs = self.ln_vision(visual_embs)
+        visual_embs = rearrange(
+            visual_embs,
+            "(b t) k d -> b t k d",
+            b=batch_size,
+            t=num_frames,
+            k=visual_embs.size(1),
+            d=visual_embs.size(2),
+        )
+
+        # add temporal encoding
+        visual_embs = self.temporal_encoding(visual_embs)
+
+        # Q-Former cross-attention
+        encoder_hidden_states = rearrange(visual_embs, "b t k d -> b (t k) d")
+        encoder_attention_mask = torch.ones(encoder_hidden_states.size()[:-1], dtype=torch.long).to(videos.device)
+        query_tokens = self.query_tokens.expand(encoder_hidden_states.size(0), -1, -1)
+        visual_query_output = self.qformer.bert(
+            query_embeds=query_tokens,
+            encoder_hidden_states=encoder_hidden_states,
+            encoder_attention_mask=encoder_attention_mask,
+            use_cache=True,
+            return_dict=True,
+        )
+
+        visual_cls_tokens = visual_query_output.last_hidden_state.mean(dim=1, keepdim=False)
+        visual_proj = self.vision_proj(visual_cls_tokens)
+        visual_proj = F.normalize(visual_proj, dim=-1)
+
+        # reshape visual embs to (B,T,H,W,D), to confirm with expected output.
+        # separate out the frame-level cls tokens if necessary.
+        frame_cls_tokens, visual_embs = visual_embs[:, :, 0:1], visual_embs[:, :, 1:]
+        h = H // self.visual_encoder.patch_size
+        w = W // self.visual_encoder.patch_size
+        visual_embs = rearrange(visual_embs, "b t (h w) d -> b t h w d", h=h, w=w)
+
+        return VideoEmbedderOutput(
+            visual_proj=visual_proj,
+            visual_embs=visual_embs,
+            visual_query_output=visual_query_output,
+            visual_cls_tokens=visual_cls_tokens,
+            frame_cls_tokens=frame_cls_tokens,
+        )
+
+    def get_text_embeddings(
+        self,
+        input_ids: torch.LongTensor,
+        attention_mask: torch.FloatTensor,
+    ) -> TextEmbedderOutput:
+        text_query_output = self.qformer.bert(
+            input_ids=input_ids,
+            attention_mask=attention_mask.to(dtype=self.query_tokens.dtype),
+            return_dict=True,
+        )
+        text_proj = text_query_output.last_hidden_state[:, 0, :]
+        text_proj = self.text_proj(text_proj)
+        text_proj = F.normalize(text_proj, dim=-1)
+
+        return TextEmbedderOutput(
+            text_proj=text_proj,
+            text_embs=text_query_output.last_hidden_state,
+            text_query_output=text_query_output,
+        )
+
+    @classmethod
+    @rank0_first
+    def _init_qformer(
+        cls: "CosmosEmbed1",
+        num_query_tokens: int,
+        encoder_width: int,
+        vocab_size: int,
+        hidden_size: int = 768,
+    ) -> tuple[BertLMHeadModel, nn.Parameter]:
+        """Convenience function for initializing QFormer module."""
+        qformer = load_qformer(
+            num_query_tokens=num_query_tokens,
+            encoder_width=encoder_width,
+            hidden_size=hidden_size,
+            vocab_size=vocab_size,
+        )
+        query_tokens = nn.Parameter(torch.zeros(1, num_query_tokens, hidden_size))
+        query_tokens.data.normal_(mean=0.0, std=0.02)
+        return qformer, query_tokens
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path: str, **kwargs):
+        # Get config from kwargs or load from pretrained path
+        config = kwargs.get("config", None)
+        if config is None:
+            config = CosmosEmbed1Config.from_pretrained(pretrained_model_name_or_path)
+
+        if config.transformer_engine:
+            config_no_te = deepcopy(config)
+            config_no_te.transformer_engine = False
+            config_no_te.use_fp8 = False  # Also disable FP8 for the base model
+
+            # Remove 'config' from kwargs to avoid conflict, we'll pass config_no_te
+            kwargs_no_te = deepcopy(kwargs)
+            kwargs_no_te["config"] = config_no_te
+
+            # Load standard (non-TE) model & weights
+            base_model = super().from_pretrained(pretrained_model_name_or_path, **kwargs_no_te)
+            base_state_dict = base_model.state_dict()
+
+            # Now build the TE version of the model
+            model_with_te = cls(config=config)
+
+            # Load weights from non-TE model
+            missing, unexpected = model_with_te.load_state_dict(base_state_dict, strict=False)
+
+            # Optional debug log
+            if missing:
+                print(f"[TransformerEngine] Missing keys: {missing}")
+            if unexpected:
+                print(f"[TransformerEngine] Unexpected keys: {unexpected}")
+
+            return model_with_te
+        else:
+            return super().from_pretrained(pretrained_model_name_or_path, **kwargs)
+
+
+AutoModel.register(CosmosEmbed1Config, CosmosEmbed1)

modeling_outputs.py

@@ -0,0 +1,69 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Output definitions for Cosmos-Embed1."""
+
+from dataclasses import dataclass
+from typing import Optional
+
+import torch
+from transformers.modeling_outputs import CausalLMOutputWithCrossAttentions, ModelOutput
+
+
+@dataclass
+class TextEmbedderOutput(ModelOutput):
+    """Output of a video embedder branch `get_text_embeddings` function.
+
+    Attrs:
+        text_proj (`torch.FloatTensor` of shape `(batch_size, num_visual_embs, embed_dim)` or `(batch_size, embed_dim)`:
+            text (video-aligned) projected embeddings from text branch.
+        text_embs (`torch.FloatTensor` of shape `(batch_size, ...)`:
+            text tokens from text branch.
+        text_query_output (`transformer.modeling_outputs.CausalLMOutputWithCrossAttentions`):
+            Useful text branch intermediate outputs like hidden states, past key values, attentions etc.
+    """
+
+    text_proj: Optional[torch.FloatTensor] = None
+    text_embs: Optional[torch.FloatTensor] = None
+    text_query_output: Optional[CausalLMOutputWithCrossAttentions] = None
+
+
+@dataclass
+class VideoEmbedderOutput(ModelOutput):
+    """Output of a video embedder branch `get_video_embeddings` function.
+
+    Attrs:
+        visual_proj (`torch.FloatTensor` of shape `(batch_size, embed_dim)`):
+            visual (text-aligned) projected embeddings from visual branch.
+        visual_embs (`torch.FloatTensor` of shape `(batch_size, num_frames, height, width, encoder_dim)`):
+            per-frame dense visual embeddings from visual encoder.
+        visual_cls_tokens (`torch.FloatTensor` of shape `(batch_size, qformer_dim)`):
+            visual pooled tokens from visual branch prior to projection and normalization.
+        frame_cls_tokens (`torch.FloatTensor` of shape `(batch_size, num_frames, encoder_dim)`):
+            per-frame cls tokens from visual encoder.
+        visual_query_output (`transformer.modeling_outputs.CausalLMOutputWithCrossAttentions`):
+            Useful visual branch intermediate outputs like hidden states, past key values, attentions etc.
+    """
+
+    visual_proj: Optional[torch.FloatTensor] = None
+    visual_embs: Optional[torch.FloatTensor] = None
+    visual_cls_tokens: Optional[torch.FloatTensor] = None
+    frame_cls_tokens: Optional[torch.FloatTensor] = None
+    visual_query_output: Optional[CausalLMOutputWithCrossAttentions] = None
+
+
+@dataclass
+class TextVideoEmbedderOutput(VideoEmbedderOutput, TextEmbedderOutput):
+    """Merged class of `VideoEmbedderOutput` and `TextEmbedderOutput`."""

modeling_qformer.py

@@ -0,0 +1,1060 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Copyright (c) 2023, salesforce.com, inc.
+# All rights reserved.
+# SPDX-License-Identifier: BSD-3-Clause
+# For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+"""Q-Former module.
+
+Code adapted from:
+https://github.com/salesforce/LAVIS/blob/main/lavis/models/blip2_models/Qformer.py
+"""
+
+import math
+from logging import getLogger
+from typing import Literal, Tuple
+
+import torch
+import torch.utils.checkpoint
+from torch import Tensor, device, nn
+from torch.nn import CrossEntropyLoss
+from transformers import GenerationMixin
+from transformers.activations import ACT2FN
+from transformers.modeling_outputs import (
+    BaseModelOutputWithPastAndCrossAttentions,
+    BaseModelOutputWithPoolingAndCrossAttentions,
+    CausalLMOutputWithCrossAttentions,
+)
+from transformers.modeling_utils import (
+    PreTrainedModel,
+    apply_chunking_to_forward,
+    find_pruneable_heads_and_indices,
+    prune_linear_layer,
+)
+from transformers.models.bert.configuration_bert import BertConfig
+
+logger = getLogger(__file__)
+
+
+class BertEmbeddings(nn.Module):
+    """Construct the embeddings from word and position embeddings."""
+
+    def __init__(self, config):
+        super().__init__()
+        self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id)
+        self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size)
+
+        # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load
+        # any TensorFlow checkpoint file
+        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+
+        # position_ids (1, len position emb) is contiguous in memory and exported when serialized
+        self.register_buffer("position_ids", torch.arange(config.max_position_embeddings).expand((1, -1)))
+        self.position_embedding_type = getattr(config, "position_embedding_type", "absolute")
+
+        self.config = config
+
+    def forward(
+        self,
+        input_ids=None,
+        position_ids=None,
+        query_embeds=None,
+        past_key_values_length=0,
+    ):
+        if input_ids is not None:
+            seq_length = input_ids.size()[1]
+        else:
+            seq_length = 0
+
+        if position_ids is None:
+            position_ids = self.position_ids[:, past_key_values_length : seq_length + past_key_values_length].clone()
+
+        if input_ids is not None:
+            embeddings = self.word_embeddings(input_ids)
+            if self.position_embedding_type == "absolute":
+                position_embeddings = self.position_embeddings(position_ids)
+                embeddings = embeddings + position_embeddings
+
+            if query_embeds is not None:
+                embeddings = torch.cat((query_embeds, embeddings), dim=1)
+        else:
+            embeddings = query_embeds
+
+        embeddings = self.LayerNorm(embeddings)
+        embeddings = self.dropout(embeddings)
+        return embeddings
+
+
+# TODO: add more efficient attention kernels like FlashAttention V2/V3.
+class BertSelfAttention(nn.Module):
+    def __init__(self, config, is_cross_attention):
+        super().__init__()
+        self.config = config
+        if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"):
+            raise ValueError(
+                "The hidden size (%d) is not a multiple of the number of attention "
+                "heads (%d)" % (config.hidden_size, config.num_attention_heads)
+            )
+
+        self.num_attention_heads = config.num_attention_heads
+        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
+        self.all_head_size = self.num_attention_heads * self.attention_head_size
+
+        self.query = nn.Linear(config.hidden_size, self.all_head_size)
+        if is_cross_attention:
+            self.key = nn.Linear(config.encoder_width, self.all_head_size)
+            self.value = nn.Linear(config.encoder_width, self.all_head_size)
+        else:
+            self.key = nn.Linear(config.hidden_size, self.all_head_size)
+            self.value = nn.Linear(config.hidden_size, self.all_head_size)
+
+        self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
+        self.position_embedding_type = getattr(config, "position_embedding_type", "absolute")
+        if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query":
+            self.max_position_embeddings = config.max_position_embeddings
+            self.distance_embedding = nn.Embedding(2 * config.max_position_embeddings - 1, self.attention_head_size)
+        self.save_attention = False
+
+    def save_attn_gradients(self, attn_gradients):
+        self.attn_gradients = attn_gradients
+
+    def get_attn_gradients(self):
+        return self.attn_gradients
+
+    def save_attention_map(self, attention_map):
+        self.attention_map = attention_map
+
+    def get_attention_map(self):
+        return self.attention_map
+
+    def transpose_for_scores(self, x):
+        new_x_shape = x.size()[:-1] + (
+            self.num_attention_heads,
+            self.attention_head_size,
+        )
+        x = x.view(*new_x_shape)
+        return x.permute(0, 2, 1, 3)
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask=None,
+        head_mask=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        past_key_value=None,
+        output_attentions=False,
+    ):
+        # If this is instantiated as a cross-attention module, the keys
+        # and values come from an encoder; the attention mask needs to be
+        # such that the encoder's padding tokens are not attended to.
+        is_cross_attention = encoder_hidden_states is not None
+
+        if is_cross_attention:
+            key_layer = self.transpose_for_scores(self.key(encoder_hidden_states))
+            value_layer = self.transpose_for_scores(self.value(encoder_hidden_states))
+            attention_mask = encoder_attention_mask
+        elif past_key_value is not None:
+            key_layer = self.transpose_for_scores(self.key(hidden_states))
+            value_layer = self.transpose_for_scores(self.value(hidden_states))
+            key_layer = torch.cat([past_key_value[0], key_layer], dim=2)
+            value_layer = torch.cat([past_key_value[1], value_layer], dim=2)
+        else:
+            key_layer = self.transpose_for_scores(self.key(hidden_states))
+            value_layer = self.transpose_for_scores(self.value(hidden_states))
+
+        mixed_query_layer = self.query(hidden_states)
+
+        query_layer = self.transpose_for_scores(mixed_query_layer)
+
+        past_key_value = (key_layer, value_layer)
+
+        # Take the dot product between "query" and "key" to get the raw attention scores.
+        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
+
+        if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query":
+            seq_length = hidden_states.size()[1]
+            position_ids_l = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(-1, 1)
+            position_ids_r = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(1, -1)
+            distance = position_ids_l - position_ids_r
+            positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1)
+            positional_embedding = positional_embedding.to(dtype=query_layer.dtype)  # fp16 compatibility
+
+            if self.position_embedding_type == "relative_key":
+                relative_position_scores = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
+                attention_scores = attention_scores + relative_position_scores
+            elif self.position_embedding_type == "relative_key_query":
+                relative_position_scores_query = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
+                relative_position_scores_key = torch.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding)
+                attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key
+
+        attention_scores = attention_scores / math.sqrt(self.attention_head_size)
+        if attention_mask is not None:
+            # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
+            attention_scores = attention_scores + attention_mask
+
+        # Normalize the attention scores to probabilities.
+        attention_probs = nn.Softmax(dim=-1)(attention_scores)
+
+        if is_cross_attention and self.save_attention:
+            self.save_attention_map(attention_probs)
+            attention_probs.register_hook(self.save_attn_gradients)
+
+        # This is actually dropping out entire tokens to attend to, which might
+        # seem a bit unusual, but is taken from the original Transformer paper.
+        attention_probs_dropped = self.dropout(attention_probs)
+
+        # Mask heads if we want to
+        if head_mask is not None:
+            attention_probs_dropped = attention_probs_dropped * head_mask
+
+        context_layer = torch.matmul(attention_probs_dropped, value_layer)
+
+        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
+        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
+        context_layer = context_layer.view(*new_context_layer_shape)
+
+        outputs = (context_layer, attention_probs) if output_attentions else (context_layer,)
+
+        outputs = outputs + (past_key_value,)
+        return outputs
+
+
+class BertSelfOutput(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+
+    def forward(self, hidden_states, input_tensor):
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.dropout(hidden_states)
+        hidden_states = self.LayerNorm(hidden_states + input_tensor)
+        return hidden_states
+
+
+class BertAttention(nn.Module):
+    def __init__(self, config, is_cross_attention=False):
+        super().__init__()
+        self.self = BertSelfAttention(config, is_cross_attention)
+        self.output = BertSelfOutput(config)
+        self.pruned_heads = set()
+
+    def prune_heads(self, heads):
+        if len(heads) == 0:
+            return
+        heads, index = find_pruneable_heads_and_indices(
+            heads,
+            self.self.num_attention_heads,
+            self.self.attention_head_size,
+            self.pruned_heads,
+        )
+
+        # Prune linear layers
+        self.self.query = prune_linear_layer(self.self.query, index)
+        self.self.key = prune_linear_layer(self.self.key, index)
+        self.self.value = prune_linear_layer(self.self.value, index)
+        self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)
+
+        # Update hyper params and store pruned heads
+        self.self.num_attention_heads = self.self.num_attention_heads - len(heads)
+        self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads
+        self.pruned_heads = self.pruned_heads.union(heads)
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask=None,
+        head_mask=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        past_key_value=None,
+        output_attentions=False,
+    ):
+        self_outputs = self.self(
+            hidden_states,
+            attention_mask,
+            head_mask,
+            encoder_hidden_states,
+            encoder_attention_mask,
+            past_key_value,
+            output_attentions,
+        )
+        attention_output = self.output(self_outputs[0], hidden_states)
+
+        outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
+        return outputs
+
+
+class BertIntermediate(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
+        if isinstance(config.hidden_act, str):
+            self.intermediate_act_fn = ACT2FN[config.hidden_act]
+        else:
+            self.intermediate_act_fn = config.hidden_act
+
+    def forward(self, hidden_states):
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.intermediate_act_fn(hidden_states)
+        return hidden_states
+
+
+class BertOutput(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
+        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+
+    def forward(self, hidden_states, input_tensor):
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.dropout(hidden_states)
+        hidden_states = self.LayerNorm(hidden_states + input_tensor)
+        return hidden_states
+
+
+class BertLayer(nn.Module):
+    def __init__(self, config, layer_num):
+        super().__init__()
+        self.config = config
+        self.chunk_size_feed_forward = config.chunk_size_feed_forward
+        self.seq_len_dim = 1
+        self.attention = BertAttention(config)
+        self.layer_num = layer_num
+        if self.config.add_cross_attention and layer_num % self.config.cross_attention_freq == 0:
+            self.crossattention = BertAttention(config, is_cross_attention=self.config.add_cross_attention)
+            self.has_cross_attention = True
+        else:
+            self.has_cross_attention = False
+        self.intermediate = BertIntermediate(config)
+        self.output = BertOutput(config)
+
+        self.intermediate_query = BertIntermediate(config)
+        self.output_query = BertOutput(config)
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask=None,
+        head_mask=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        past_key_value=None,
+        output_attentions=False,
+        query_length=0,
+    ):
+        # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
+        self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
+        self_attention_outputs = self.attention(
+            hidden_states,
+            attention_mask,
+            head_mask,
+            output_attentions=output_attentions,
+            past_key_value=self_attn_past_key_value,
+        )
+        attention_output = self_attention_outputs[0]
+        outputs = self_attention_outputs[1:-1]
+
+        present_key_value = self_attention_outputs[-1]
+
+        if query_length > 0:
+            query_attention_output = attention_output[:, :query_length, :]
+
+            if self.has_cross_attention:
+                assert (
+                    encoder_hidden_states is not None
+                ), "encoder_hidden_states must be given for cross-attention layers"
+                cross_attention_outputs = self.crossattention(
+                    query_attention_output,
+                    attention_mask,
+                    head_mask,
+                    encoder_hidden_states,
+                    encoder_attention_mask,
+                    output_attentions=output_attentions,
+                )
+                query_attention_output = cross_attention_outputs[0]
+                outputs = outputs + cross_attention_outputs[1:-1]  # add cross attentions if we output attention weights
+
+            layer_output = apply_chunking_to_forward(
+                self.feed_forward_chunk_query,
+                self.chunk_size_feed_forward,
+                self.seq_len_dim,
+                query_attention_output,
+            )
+            if attention_output.shape[1] > query_length:
+                layer_output_text = apply_chunking_to_forward(
+                    self.feed_forward_chunk,
+                    self.chunk_size_feed_forward,
+                    self.seq_len_dim,
+                    attention_output[:, query_length:, :],
+                )
+                layer_output = torch.cat([layer_output, layer_output_text], dim=1)
+        else:
+            layer_output = apply_chunking_to_forward(
+                self.feed_forward_chunk,
+                self.chunk_size_feed_forward,
+                self.seq_len_dim,
+                attention_output,
+            )
+        outputs = (layer_output,) + outputs
+
+        outputs = outputs + (present_key_value,)
+
+        return outputs
+
+    def feed_forward_chunk(self, attention_output):
+        intermediate_output = self.intermediate(attention_output)
+        layer_output = self.output(intermediate_output, attention_output)
+        return layer_output
+
+    def feed_forward_chunk_query(self, attention_output):
+        intermediate_output = self.intermediate_query(attention_output)
+        layer_output = self.output_query(intermediate_output, attention_output)
+        return layer_output
+
+
+class BertEncoder(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.layer = nn.ModuleList([BertLayer(config, i) for i in range(config.num_hidden_layers)])
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask=None,
+        head_mask=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        past_key_values=None,
+        use_cache=None,
+        output_attentions=False,
+        output_hidden_states=False,
+        return_dict=True,
+        query_length=0,
+    ):
+        all_hidden_states = () if output_hidden_states else None
+        all_self_attentions = () if output_attentions else None
+        all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None
+
+        next_decoder_cache = () if use_cache else None
+
+        for i in range(self.config.num_hidden_layers):
+            layer_module = self.layer[i]
+            if output_hidden_states:
+                all_hidden_states = all_hidden_states + (hidden_states,)
+
+            layer_head_mask = head_mask[i] if head_mask is not None else None
+            past_key_value = past_key_values[i] if past_key_values is not None else None
+
+            if getattr(self.config, "gradient_checkpointing", False) and self.training:
+                if use_cache:
+                    logger.warning(
+                        "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
+                    )
+                    use_cache = False
+
+                def create_custom_forward(module):
+                    def custom_forward(*inputs):
+                        return module(*inputs, past_key_value, output_attentions, query_length)
+
+                    return custom_forward
+
+                layer_outputs = torch.utils.checkpoint.checkpoint(
+                    create_custom_forward(layer_module),
+                    hidden_states,
+                    attention_mask,
+                    layer_head_mask,
+                    encoder_hidden_states,
+                    encoder_attention_mask,
+                )
+            else:
+                layer_outputs = layer_module(
+                    hidden_states,
+                    attention_mask,
+                    layer_head_mask,
+                    encoder_hidden_states,
+                    encoder_attention_mask,
+                    past_key_value,
+                    output_attentions,
+                    query_length,
+                )
+
+            hidden_states = layer_outputs[0]
+            if use_cache:
+                next_decoder_cache += (layer_outputs[-1],)
+            if output_attentions:
+                all_self_attentions = all_self_attentions + (layer_outputs[1],)
+                all_cross_attentions = all_cross_attentions + (layer_outputs[2],)
+
+        if output_hidden_states:
+            all_hidden_states = all_hidden_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(
+                v
+                for v in [
+                    hidden_states,
+                    next_decoder_cache,
+                    all_hidden_states,
+                    all_self_attentions,
+                    all_cross_attentions,
+                ]
+                if v is not None
+            )
+        return BaseModelOutputWithPastAndCrossAttentions(
+            last_hidden_state=hidden_states,
+            past_key_values=next_decoder_cache,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attentions,
+            cross_attentions=all_cross_attentions,
+        )
+
+
+class BertPooler(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.activation = nn.Tanh()
+
+    def forward(self, hidden_states):
+        # We "pool" the model by simply taking the hidden state corresponding
+        # to the first token.
+        first_token_tensor = hidden_states[:, 0]
+        pooled_output = self.dense(first_token_tensor)
+        pooled_output = self.activation(pooled_output)
+        return pooled_output
+
+
+class BertPredictionHeadTransform(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        if isinstance(config.hidden_act, str):
+            self.transform_act_fn = ACT2FN[config.hidden_act]
+        else:
+            self.transform_act_fn = config.hidden_act
+        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+
+    def forward(self, hidden_states):
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.transform_act_fn(hidden_states)
+        hidden_states = self.LayerNorm(hidden_states)
+        return hidden_states
+
+
+class BertLMPredictionHead(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.transform = BertPredictionHeadTransform(config)
+
+        # The output weights are the same as the input embeddings, but there is
+        # an output-only bias for each token.
+        self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        self.bias = nn.Parameter(torch.zeros(config.vocab_size))
+
+        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`
+        self.decoder.bias = self.bias
+
+    def forward(self, hidden_states):
+        hidden_states = self.transform(hidden_states)
+        hidden_states = self.decoder(hidden_states)
+        return hidden_states
+
+
+class BertOnlyMLMHead(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.predictions = BertLMPredictionHead(config)
+
+    def forward(self, sequence_output):
+        prediction_scores = self.predictions(sequence_output)
+        return prediction_scores
+
+
+class BertPreTrainedModel(PreTrainedModel):
+    """
+    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
+    models.
+    """
+
+    config_class = BertConfig
+    base_model_prefix = "bert"
+    _keys_to_ignore_on_load_missing = [r"position_ids"]
+
+    def _init_weights(self, module):
+        """Initialize the weights"""
+        if isinstance(module, (nn.Linear, nn.Embedding)):
+            # Modified from original LAVIS implementation to add truncated normal.
+            # This matches the original Tensorflow implementation from Google.
+            nn.init.trunc_normal_(module.weight, std=self.config.initializer_range)
+        elif isinstance(module, nn.LayerNorm):
+            module.bias.data.zero_()
+            module.weight.data.fill_(1.0)
+        if isinstance(module, nn.Linear) and module.bias is not None:
+            module.bias.data.zero_()
+
+
+class BertModel(BertPreTrainedModel):
+    """
+    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
+    cross-attention is added between the self-attention layers, following the architecture described in `Attention is
+    all you need <https://arxiv.org/abs/1706.03762>`__ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
+    Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
+    argument and :obj:`add_cross_attention` set to :obj:`True`; an :obj:`encoder_hidden_states` is then expected as an
+    input to the forward pass.
+    """
+
+    def __init__(self, config, add_pooling_layer=False):
+        super().__init__(config)
+        self.config = config
+
+        self.embeddings = BertEmbeddings(config)
+
+        self.encoder = BertEncoder(config)
+
+        self.pooler = BertPooler(config) if add_pooling_layer else None
+
+        self.init_weights()
+
+    def get_input_embeddings(self):
+        return self.embeddings.word_embeddings
+
+    def set_input_embeddings(self, value):
+        self.embeddings.word_embeddings = value
+
+    def _prune_heads(self, heads_to_prune):
+        """
+        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
+        class PreTrainedModel
+        """
+        for layer, heads in heads_to_prune.items():
+            self.encoder.layer[layer].attention.prune_heads(heads)
+
+    def get_extended_attention_mask(
+        self,
+        attention_mask: Tensor,
+        input_shape: Tuple[int],
+        device: device,
+        is_decoder: bool,
+        has_query: bool = False,
+    ) -> Tensor:
+        """
+        Makes broadcastable attention and causal masks so that future and masked tokens are ignored.
+
+        Arguments:
+            attention_mask (:obj:`torch.Tensor`):
+                Mask with ones indicating tokens to attend to, zeros for tokens to ignore.
+            input_shape (:obj:`Tuple[int]`):
+                The shape of the input to the model.
+            device: (:obj:`torch.device`):
+                The device of the input to the model.
+
+        Returns:
+            :obj:`torch.Tensor` The extended attention mask, with a the same dtype as :obj:`attention_mask.dtype`.
+        """
+        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
+        # ourselves in which case we just need to make it broadcastable to all heads.
+        if attention_mask.dim() == 3:
+            extended_attention_mask = attention_mask[:, None, :, :]
+        elif attention_mask.dim() == 2:
+            # Provided a padding mask of dimensions [batch_size, seq_length]
+            # - if the model is a decoder, apply a causal mask in addition to the padding mask
+            # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length]
+            if is_decoder:
+                batch_size, seq_length = input_shape
+
+                seq_ids = torch.arange(seq_length, device=device)
+                causal_mask = seq_ids[None, None, :].repeat(batch_size, seq_length, 1) <= seq_ids[None, :, None]
+
+                # add a prefix ones mask to the causal mask
+                # causal and attention masks must have same type with pytorch version < 1.3
+                causal_mask = causal_mask.to(attention_mask.dtype)
+
+                if causal_mask.shape[1] < attention_mask.shape[1]:
+                    prefix_seq_len = attention_mask.shape[1] - causal_mask.shape[1]
+                    if has_query:  # UniLM style attention mask
+                        causal_mask = torch.cat(
+                            [
+                                torch.zeros(
+                                    (batch_size, prefix_seq_len, seq_length),
+                                    device=device,
+                                    dtype=causal_mask.dtype,
+                                ),
+                                causal_mask,
+                            ],
+                            axis=1,
+                        )
+                    causal_mask = torch.cat(
+                        [
+                            torch.ones(
+                                (batch_size, causal_mask.shape[1], prefix_seq_len),
+                                device=device,
+                                dtype=causal_mask.dtype,
+                            ),
+                            causal_mask,
+                        ],
+                        axis=-1,
+                    )
+                extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :]
+            else:
+                extended_attention_mask = attention_mask[:, None, None, :]
+        else:
+            raise ValueError(
+                "Wrong shape for input_ids (shape {}) or attention_mask (shape {})".format(
+                    input_shape, attention_mask.shape
+                )
+            )
+
+        # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
+        # masked positions, this operation will create a tensor which is 0.0 for
+        # positions we want to attend and -10000.0 for masked positions.
+        # Since we are adding it to the raw scores before the softmax, this is
+        # effectively the same as removing these entirely.
+        extended_attention_mask = extended_attention_mask.to(dtype=attention_mask.dtype)  # fp16 compatibility
+        extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0
+        return extended_attention_mask
+
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        position_ids=None,
+        head_mask=None,
+        query_embeds=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        past_key_values=None,
+        use_cache=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+        is_decoder=False,
+    ):
+        r"""
+        encoder_hidden_states  (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`):
+            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
+            the model is configured as a decoder.
+        encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
+            Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
+            the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``:
+            - 1 for tokens that are **not masked**,
+            - 0 for tokens that are **masked**.
+        past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
+            Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
+            If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids`
+            (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)`
+            instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`.
+        use_cache (:obj:`bool`, `optional`):
+            If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up
+            decoding (see :obj:`past_key_values`).
+        """
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        # use_cache = use_cache if use_cache is not None else self.config.use_cache
+
+        if input_ids is None:
+            assert query_embeds is not None, "You have to specify query_embeds when input_ids is None"
+
+        # past_key_values_length
+        past_key_values_length = (
+            past_key_values[0][0].shape[2] - self.config.query_length if past_key_values is not None else 0
+        )
+
+        query_length = query_embeds.shape[1] if query_embeds is not None else 0
+
+        embedding_output = self.embeddings(
+            input_ids=input_ids,
+            position_ids=position_ids,
+            query_embeds=query_embeds,
+            past_key_values_length=past_key_values_length,
+        )
+
+        input_shape = embedding_output.size()[:-1]
+        batch_size, seq_length = input_shape
+        device = embedding_output.device
+
+        if attention_mask is None:
+            attention_mask = torch.ones(
+                ((batch_size, seq_length + past_key_values_length)), device=device, dtype=embedding_output.dtype
+            )
+
+        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
+        # ourselves in which case we just need to make it broadcastable to all heads.
+        if is_decoder:
+            extended_attention_mask = self.get_extended_attention_mask(
+                attention_mask,
+                input_ids.shape,
+                device,
+                is_decoder,
+                has_query=(query_embeds is not None),
+            )
+        else:
+            extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape, device, is_decoder)
+
+        # If a 2D or 3D attention mask is provided for the cross-attention
+        # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
+        if encoder_hidden_states is not None:
+            if type(encoder_hidden_states) == list:
+                encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states[0].size()
+            else:
+                (
+                    encoder_batch_size,
+                    encoder_sequence_length,
+                    _,
+                ) = encoder_hidden_states.size()
+            encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
+
+            if type(encoder_attention_mask) == list:
+                encoder_extended_attention_mask = [self.invert_attention_mask(mask) for mask in encoder_attention_mask]
+            elif encoder_attention_mask is None:
+                encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device, dtype=attention_mask.dtype)
+                encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
+            else:
+                encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
+        else:
+            encoder_extended_attention_mask = None
+
+        # Prepare head mask if needed
+        # 1.0 in head_mask indicate we keep the head
+        # attention_probs has shape bsz x n_heads x N x N
+        # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
+        # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
+        head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)
+
+        encoder_outputs = self.encoder(
+            embedding_output,
+            attention_mask=extended_attention_mask,
+            head_mask=head_mask,
+            encoder_hidden_states=encoder_hidden_states,
+            encoder_attention_mask=encoder_extended_attention_mask,
+            past_key_values=past_key_values,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            query_length=query_length,
+        )
+        sequence_output = encoder_outputs[0]
+        pooled_output = self.pooler(sequence_output) if self.pooler is not None else None
+
+        if not return_dict:
+            return (sequence_output, pooled_output) + encoder_outputs[1:]
+
+        return BaseModelOutputWithPoolingAndCrossAttentions(
+            last_hidden_state=sequence_output,
+            pooler_output=pooled_output,
+            past_key_values=encoder_outputs.past_key_values,
+            hidden_states=encoder_outputs.hidden_states,
+            attentions=encoder_outputs.attentions,
+            cross_attentions=encoder_outputs.cross_attentions,
+        )
+
+
+class BertLMHeadModel(BertPreTrainedModel, GenerationMixin):
+    _keys_to_ignore_on_load_unexpected = [r"pooler"]
+    _keys_to_ignore_on_load_missing = [r"position_ids", r"predictions.decoder.bias"]
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        self.bert = BertModel(config, add_pooling_layer=False)
+        self.cls = BertOnlyMLMHead(config)
+
+        self.init_weights()
+
+    def get_output_embeddings(self):
+        return self.cls.predictions.decoder
+
+    def set_output_embeddings(self, new_embeddings):
+        self.cls.predictions.decoder = new_embeddings
+
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        position_ids=None,
+        head_mask=None,
+        query_embeds=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        labels=None,
+        past_key_values=None,
+        use_cache=True,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+        return_logits=False,
+        is_decoder=True,
+        reduction="mean",
+    ):
+        r"""
+        encoder_hidden_states  (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`):
+            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
+            the model is configured as a decoder.
+        encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
+            Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
+            the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``:
+            - 1 for tokens that are **not masked**,
+            - 0 for tokens that are **masked**.
+        labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
+            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
+            ``[-100, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) Tokens with indices set to ``-100`` are
+            ignored (masked), the loss is only computed for the tokens with labels n ``[0, ..., config.vocab_size]``
+        past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
+            Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
+            If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids`
+            (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)`
+            instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`.
+        use_cache (:obj:`bool`, `optional`):
+            If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up
+            decoding (see :obj:`past_key_values`).
+        Returns:
+        Example::
+            >>> from transformers import BertTokenizer, BertLMHeadModel, BertConfig
+            >>> import torch
+            >>> tokenizer = BertTokenizer.from_pretrained('bert-base-cased')
+            >>> config = BertConfig.from_pretrained("bert-base-cased")
+            >>> model = BertLMHeadModel.from_pretrained('bert-base-cased', config=config)
+            >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
+            >>> outputs = model(**inputs)
+            >>> prediction_logits = outputs.logits
+        """
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        if labels is not None:
+            use_cache = False
+        if past_key_values is not None:
+            query_embeds = None
+
+        outputs = self.bert(
+            input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            query_embeds=query_embeds,
+            encoder_hidden_states=encoder_hidden_states,
+            encoder_attention_mask=encoder_attention_mask,
+            past_key_values=past_key_values,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            is_decoder=is_decoder,
+        )
+
+        sequence_output = outputs[0]
+        if query_embeds is not None:
+            sequence_output = outputs[0][:, query_embeds.shape[1] :, :]
+
+        prediction_scores = self.cls(sequence_output)
+
+        if return_logits:
+            return prediction_scores[:, :-1, :].contiguous()
+
+        lm_loss = None
+        if labels is not None:
+            # we are doing next-token prediction; shift prediction scores and input ids by one
+            shifted_prediction_scores = prediction_scores[:, :-1, :].contiguous()
+            labels = labels[:, 1:].contiguous()
+            loss_fct = CrossEntropyLoss(reduction=reduction, label_smoothing=0.1)
+            lm_loss = loss_fct(
+                shifted_prediction_scores.view(-1, self.config.vocab_size),
+                labels.view(-1),
+            )
+            if reduction == "none":
+                lm_loss = lm_loss.view(prediction_scores.size(0), -1).sum(1)
+
+        if not return_dict:
+            output = (prediction_scores,) + outputs[2:]
+            return ((lm_loss,) + output) if lm_loss is not None else output
+
+        return CausalLMOutputWithCrossAttentions(
+            loss=lm_loss,
+            logits=prediction_scores,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+            cross_attentions=outputs.cross_attentions,
+        )
+
+    def prepare_inputs_for_generation(self, input_ids, query_embeds, past=None, attention_mask=None, **model_kwargs):
+        # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly
+        if attention_mask is None:
+            attention_mask = input_ids.new_ones(input_ids.shape)
+        query_mask = input_ids.new_ones(query_embeds.shape[:-1])
+        attention_mask = torch.cat([query_mask, attention_mask], dim=-1)
+
+        # cut decoder_input_ids if past is used
+        if past is not None:
+            input_ids = input_ids[:, -1:]
+
+        return {
+            "input_ids": input_ids,
+            "query_embeds": query_embeds,
+            "attention_mask": attention_mask,
+            "past_key_values": past,
+            "encoder_hidden_states": model_kwargs.get("encoder_hidden_states", None),
+            "encoder_attention_mask": model_kwargs.get("encoder_attention_mask", None),
+            "is_decoder": True,
+        }
+
+    def _reorder_cache(self, past, beam_idx):
+        reordered_past = ()
+        for layer_past in past:
+            reordered_past += (tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),)
+        return reordered_past
+
+
+def load_qformer(
+    num_query_tokens: int,
+    encoder_width: int,
+    vocab_size: int = 30523,
+    hidden_size: int = 768,
+    cross_attention_freq: int = 2,
+    base_model: Literal["bert-base-uncased", "bert-large-uncased"] = "bert-base-uncased",
+) -> BertLMHeadModel:
+    """Utility to load QFormer module.
+
+    Args:
+        num_query_tokens (int): number of query tokens.
+        encoder_width (int): vector length of visual encoder embeddings.
+        hidden_size (int): vector length of BERT's attention blocks.
+        cross_attention_freq (int): block frequency of visual cross-attention.
+        base_model (str): Base text model for QFormer. Default `bert-base-uncased`.
+
+    Returns:
+        `BertLMHeadModel` module.
+    """
+
+    encoder_config = BertConfig.from_pretrained(base_model)
+    encoder_config.encoder_width = encoder_width
+    encoder_config.hidden_size = hidden_size
+    encoder_config.add_cross_attention = True
+    encoder_config.cross_attention_freq = cross_attention_freq
+    encoder_config.query_length = num_query_tokens
+    encoder_config.vocab_size = vocab_size
+    qformer = BertLMHeadModel(encoder_config)
+    return qformer

modeling_utils.py

@@ -0,0 +1,180 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Misc functions and modules for Cosmos-Embed1."""
+
+import functools
+from logging import getLogger
+from typing import Callable, Optional, Protocol
+
+import torch
+import torch.distributed as dist
+import torch.nn as nn
+
+logger = getLogger(__file__)
+
+
+def get_rank(group: Optional[dist.ProcessGroup] = None) -> int:
+    """Get the rank (GPU device) of the worker.
+
+    Returns:
+        rank (int): The rank of the worker.
+    """
+    rank = 0
+    if dist.is_available() and dist.is_initialized():
+        rank = dist.get_rank(group)
+    return rank
+
+
+def barrier() -> None:
+    """Barrier for all GPUs."""
+    if dist.is_available() and dist.is_initialized():
+        dist.barrier()
+
+
+def rank0_first(func: Callable) -> Callable:
+    """Run the function on rank 0 first, then on other ranks."""
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):  # noqa: ANN202
+        if get_rank() == 0:
+            result = func(*args, **kwargs)
+        barrier()
+        if get_rank() != 0:
+            result = func(*args, **kwargs)
+        return result
+
+    return wrapper
+
+
+def add_docstring(docstring: str):
+    def decorator(func):
+        func.__doc__ = docstring
+        return func
+
+    return decorator
+
+
+INIT_DOCSTRING = """
+Constructor for encoding module.
+
+Args:
+    embed_dim: size of embedding vectors, e.g. x.shape[3].
+    max_len: maximum length of temporal sequence, e.g. x.shape[1].
+"""
+
+FORWARD_DOCSTRING = """
+Forward function.
+
+Args:
+    x (`torch.Tensor`): rank 4 tensor to add spatio-temporal encodings to.
+
+Returns:
+    `torch.Tensor` of rank 4.
+"""
+
+
+class EncodingProtocol(Protocol):
+    def __init__(self, embed_dim: int, max_len: int) -> None:
+        pass
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        pass
+
+
+def interpolate_temp_pos_embed(temp_embed: torch.Tensor, num_frames: int) -> torch.Tensor:
+    """Linearly interpolates temporal encoding from `temp_embed.shape[0] to num_frames."""
+
+    temp_embed_resized = temp_embed.permute(1, 0).unsqueeze(0)
+    temp_embed_resized = nn.functional.interpolate(
+        temp_embed_resized,
+        size=(num_frames),
+        mode="linear",
+        align_corners=False,
+    )
+    return temp_embed_resized.squeeze(0).permute(1, 0)
+
+
+class TemporalParameterEncoding(nn.Module, EncodingProtocol):
+    @add_docstring(INIT_DOCSTRING)
+    def __init__(self, embed_dim: int, max_len: int) -> None:
+        super().__init__()
+        self.embed_dim = embed_dim
+        self.max_len = max_len
+        self.temp_embed = nn.Parameter(torch.zeros(self.max_len, self.embed_dim))
+        nn.init.trunc_normal_(self.temp_embed, std=0.02)
+
+    @add_docstring(FORWARD_DOCSTRING)
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        _, t, _, _ = x.shape
+        if t != self.temp_embed.shape[0]:
+            logger.debug(f"Interpolating temporal encodings from {self.temp_embed.shape[0]} to {t}.")
+            temp_embed = interpolate_temp_pos_embed(self.temp_embed, t)
+        else:
+            temp_embed = self.temp_embed
+        temp_embed = temp_embed.unsqueeze(0).unsqueeze(2)
+        return x + temp_embed
+
+
+def create_neighbor_weight_matrix(num_tokens: int, device: torch.device, dtype: torch.dtype) -> torch.Tensor:
+    indices = torch.arange(num_tokens, dtype=dtype, device=device)
+    abs_diff = torch.abs(indices.unsqueeze(0) - indices.unsqueeze(1))
+    weights = 1.0 / (2.0**abs_diff)
+    return weights
+
+
+def compute_t_adj(x: torch.Tensor, weights: torch.Tensor) -> torch.Tensor:
+    return torch.einsum("bfnd,nk->bfkd", x, weights)
+
+
+def token_propagation(x: torch.Tensor, num_tokens: int) -> torch.Tensor:
+    """Apply neighboring token propagation update."""
+    weights = create_neighbor_weight_matrix(num_tokens, x.device, x.dtype)
+    t_adj = compute_t_adj(x, weights)
+    return x + t_adj - t_adj.detach()
+
+
+class NeighboringTokenPropagationEncoding(TemporalParameterEncoding):
+    """
+    Neighboring Token Propagation method inspired by Momentor (https://arxiv.org/abs/2402.11435)
+    """
+
+    @add_docstring(FORWARD_DOCSTRING)
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        _, t, q, _ = x.shape
+        if t != self.temp_embed.shape[0]:
+            logger.debug(f"Interpolating temporal encodings from {self.temp_embed.shape[0]} to {t}.")
+            temp_embed = interpolate_temp_pos_embed(self.temp_embed, t)
+        else:
+            temp_embed = self.temp_embed
+        temp_embed = temp_embed.unsqueeze(0).unsqueeze(2)
+
+        if self.training:
+            temp_embed = token_propagation(temp_embed, q)
+        return x + temp_embed
+
+
+class EncodingFactory(nn.Module):
+    def __init__(self, encoding_type: str, embed_dim: int, max_len: int) -> None:
+        super().__init__()
+        fn = {
+            "temporal_parameter": TemporalParameterEncoding,
+            "neighboring_token_propagation": NeighboringTokenPropagationEncoding,
+        }[encoding_type]
+        self.encoding = fn(embed_dim=embed_dim, max_len=max_len)
+
+    @add_docstring(FORWARD_DOCSTRING)
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.encoding(x)

modeling_vit.py

@@ -0,0 +1,696 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Copyright (c) 2023, salesforce.com, inc.
+# All rights reserved.
+# SPDX-License-Identifier: BSD-3-Clause
+# For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+"""
+EVA-CLIP backbone used in BLIP2.
+
+Code adapted from:
+https://github.com/salesforce/LAVIS/blob/main/lavis/models/eva_vit.py
+"""
+
+
+import math
+from functools import partial
+from logging import getLogger
+from typing import Any, Optional, Tuple, Union
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.utils.checkpoint as checkpoint
+
+logger = getLogger(__file__)
+
+TRANSFORMER_ENGINE_AVAILABLE = False
+try:
+    import transformer_engine.pytorch as te
+    from transformer_engine.common.recipe import DelayedScaling, Format
+
+    TRANSFORMER_ENGINE_AVAILABLE = True
+    logger.info("Transformer Engine is available, can set `transformer_engine=True` in config " "for faster inference.")
+except ImportError:
+    pass
+
+
+def drop_path(x, drop_prob: float = 0.0, training: bool = False, scale_by_keep: bool = True):
+    """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).
+
+    From https://github.com/huggingface/pytorch-image-models/blob/main/timm/layers/drop.py
+    """
+    if drop_prob == 0.0 or not training:
+        return x
+    keep_prob = 1 - drop_prob
+    shape = (x.shape[0],) + (1,) * (x.ndim - 1)  # work with diff dim tensors, not just 2D ConvNets
+    random_tensor = x.new_empty(shape).bernoulli_(keep_prob)
+    if keep_prob > 0.0 and scale_by_keep:
+        random_tensor.div_(keep_prob)
+    return x * random_tensor
+
+
+class DropPath(nn.Module):
+    """Drop paths (Stochastic Depth) per sample  (when applied in main path of residual blocks)."""
+
+    def __init__(self, drop_prob: float) -> None:
+        super().__init__()
+        self.drop_prob = drop_prob
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return drop_path(x, self.drop_prob, self.training)
+
+    def extra_repr(self) -> str:
+        return "p={}".format(self.drop_prob)
+
+
+class Mlp(nn.Module):
+    def __init__(
+        self,
+        in_features: int,
+        hidden_features: Optional[int] = None,
+        out_features: Optional[int] = None,
+        act_layer=nn.GELU,
+        drop: float = 0.0,
+        transformer_engine: bool = False,
+    ) -> None:
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        fn = te.Linear if transformer_engine else nn.Linear
+        self.fc1 = fn(in_features, hidden_features)
+        self.act = act_layer()
+        self.fc2 = fn(hidden_features, out_features)
+        self.drop = nn.Dropout(drop)
+
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.fc2(x)
+        x = self.drop(x)
+        return x
+
+
+class Attention(nn.Module):
+    def __init__(
+        self,
+        dim,
+        num_heads=8,
+        qkv_bias=False,
+        qk_scale=None,
+        attn_drop=0.0,
+        proj_drop=0.0,
+        window_size=None,
+        attn_head_dim=None,
+        **kwargs,
+    ):
+        super().__init__()
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        if attn_head_dim is not None:
+            head_dim = attn_head_dim
+        all_head_dim = head_dim * self.num_heads
+        self.scale = qk_scale or head_dim**-0.5
+
+        self.qkv = nn.Linear(dim, all_head_dim * 3, bias=qkv_bias)
+
+        if window_size:
+            self.window_size = window_size
+            self.num_relative_distance = (2 * window_size[0] - 1) * (2 * window_size[1] - 1) + 3
+            self.relative_position_bias_table = nn.Parameter(
+                torch.zeros(self.num_relative_distance, num_heads)
+            )  # 2*Wh-1 * 2*Ww-1, nH
+            # cls to token & token 2 cls & cls to cls
+
+            # get pair-wise relative position index for each token inside the window
+            coords_h = torch.arange(window_size[0])
+            coords_w = torch.arange(window_size[1])
+            coords = torch.stack(torch.meshgrid([coords_h, coords_w]))  # 2, Wh, Ww
+            coords_flatten = torch.flatten(coords, 1)  # 2, Wh*Ww
+            relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :]  # 2, Wh*Ww, Wh*Ww
+            relative_coords = relative_coords.permute(1, 2, 0).contiguous()  # Wh*Ww, Wh*Ww, 2
+            relative_coords[:, :, 0] += window_size[0] - 1  # shift to start from 0
+            relative_coords[:, :, 1] += window_size[1] - 1
+            relative_coords[:, :, 0] *= 2 * window_size[1] - 1
+            relative_position_index = torch.zeros(
+                size=(window_size[0] * window_size[1] + 1,) * 2, dtype=relative_coords.dtype
+            )
+            relative_position_index[1:, 1:] = relative_coords.sum(-1)  # Wh*Ww, Wh*Ww
+            relative_position_index[0, 0:] = self.num_relative_distance - 3
+            relative_position_index[0:, 0] = self.num_relative_distance - 2
+            relative_position_index[0, 0] = self.num_relative_distance - 1
+
+            self.register_buffer("relative_position_index", relative_position_index)
+        else:
+            self.window_size = None
+            self.relative_position_bias_table = None
+            self.relative_position_index = None
+
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(all_head_dim, dim)
+        self.proj_drop = nn.Dropout(proj_drop)
+
+    def forward(self, x, rel_pos_bias=None):
+        B, N, C = x.shape
+        qkv = self.qkv(x)
+        qkv = qkv.reshape(B, N, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4)
+        q, k, v = qkv[0], qkv[1], qkv[2]
+
+        q = q * self.scale
+        attn = q @ k.transpose(-2, -1)
+
+        if self.relative_position_bias_table is not None:
+            relative_position_bias = self.relative_position_bias_table[self.relative_position_index.view(-1)].view(
+                self.window_size[0] * self.window_size[1] + 1, self.window_size[0] * self.window_size[1] + 1, -1
+            )  # Wh*Ww,Wh*Ww,nH
+            relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous()  # nH, Wh*Ww, Wh*Ww
+            attn = attn + relative_position_bias.unsqueeze(0)
+
+        if rel_pos_bias is not None:
+            attn = attn + rel_pos_bias
+
+        attn = attn.softmax(dim=-1)
+        attn = self.attn_drop(attn)
+
+        x = (attn @ v).transpose(1, 2).reshape(B, N, -1)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+
+
+class TransformerEngineAttention(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        num_heads: int = 8,
+        qkv_bias: bool = False,
+        qk_scale: Optional[float] = None,
+        attn_drop: float = 0.0,
+        proj_drop: float = 0.0,
+        window_size: Optional[int] = None,
+        attn_head_dim: Optional[int] = None,
+        checkpoint_attention: bool = False,
+    ):
+        super().__init__()
+        self.num_heads = num_heads
+        self.checkpoint_attention = checkpoint_attention
+        head_dim = dim // num_heads
+        if attn_head_dim is not None:
+            head_dim = attn_head_dim
+        all_head_dim = head_dim * self.num_heads
+        self.scale = qk_scale or head_dim**-0.5
+
+        # QKV projection
+        self.qkv = te.Linear(dim, all_head_dim * 3, bias=qkv_bias)
+
+        if window_size:
+            raise NotImplementedError("`window_size` not implemented for TE!")
+
+        self.te_attn = te.DotProductAttention(
+            num_attention_heads=num_heads,
+            kv_channels=head_dim,
+            attention_dropout=attn_drop,
+            qkv_format="bshd",
+            softmax_scale=self.scale,
+            attn_mask_type="no_mask",
+        )
+
+        # output projection + dropout
+        self.proj = te.Linear(all_head_dim, dim)
+        self.proj_drop = nn.Dropout(proj_drop)
+
+    def forward(self, x: torch.Tensor, rel_pos_bias: Optional[torch.Tensor] = None) -> torch.Tensor:
+        """
+        x: [B, N, C]
+        rel_pos_bias (optional): tensor of shape [num_heads, N, N]
+        """
+        B, N, _ = x.shape
+        qkv = self.qkv(x)
+        qkv = qkv.reshape(B, N, 3, self.num_heads, -1).permute(2, 0, 1, 3, 4)
+        q, k, v = qkv[0], qkv[1], qkv[2]  # BNHC format
+
+        if rel_pos_bias is not None:
+            raise NotImplementedError("`rel_pos_bias` not implemented for TE!")
+
+        # run TE's fused attention
+        y = self.te_attn(q, k, v, checkpoint_core_attention=self.checkpoint_attention)
+
+        # final proj + dropout
+        return self.proj_drop(self.proj(y))
+
+
+class Block(nn.Module):
+    def __init__(
+        self,
+        dim,
+        num_heads,
+        mlp_ratio=4.0,
+        qkv_bias=False,
+        qk_scale=None,
+        drop=0.0,
+        attn_drop=0.0,
+        drop_path=0.0,
+        init_values=None,
+        act_layer=nn.GELU,
+        norm_layer=nn.LayerNorm,
+        window_size=None,
+        attn_head_dim=None,
+        transformer_engine: bool = False,
+        checkpoint_attention: bool = False,
+    ):
+        super().__init__()
+        self.transformer_engine = transformer_engine
+        self.window_size = window_size
+        self.checkpoint_attention = checkpoint_attention
+
+        if checkpoint_attention and not transformer_engine:
+            raise ValueError("`checkpoint_attention` needs `transformer_engine`!")
+
+        self.norm1 = norm_layer(dim)
+        attn_fn = TransformerEngineAttention if transformer_engine else Attention
+        self.attn = attn_fn(
+            dim,
+            num_heads=num_heads,
+            qkv_bias=qkv_bias,
+            qk_scale=qk_scale,
+            attn_drop=attn_drop,
+            proj_drop=drop,
+            window_size=window_size,
+            attn_head_dim=attn_head_dim,
+            checkpoint_attention=checkpoint_attention,
+        )
+
+        # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here
+        self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
+        self.norm2 = norm_layer(dim)
+        mlp_hidden_dim = int(dim * mlp_ratio)
+        self.mlp = Mlp(
+            in_features=dim,
+            hidden_features=mlp_hidden_dim,
+            act_layer=act_layer,
+            drop=drop,
+            transformer_engine=transformer_engine,
+        )
+
+        if init_values is not None and init_values > 0:
+            self.gamma_1 = nn.Parameter(init_values * torch.ones((dim)), requires_grad=True)
+            self.gamma_2 = nn.Parameter(init_values * torch.ones((dim)), requires_grad=True)
+        else:
+            self.gamma_1, self.gamma_2 = None, None
+
+    def forward(self, x, rel_pos_bias=None):
+        if self.gamma_1 is None:
+            x = x + self.drop_path(self.attn(self.norm1(x), rel_pos_bias=rel_pos_bias))
+            x = x + self.drop_path(self.mlp(self.norm2(x)))
+        else:
+            x = x + self.drop_path(self.gamma_1 * self.attn(self.norm1(x), rel_pos_bias=rel_pos_bias))
+            x = x + self.drop_path(self.gamma_2 * self.mlp(self.norm2(x)))
+        return x
+
+
+class PatchEmbed(nn.Module):
+    """Image to Patch Embedding"""
+
+    def __init__(
+        self,
+        img_size: Union[int, Tuple[int, int]] = 224,
+        patch_size: Union[int, Tuple[int, int]] = 16,
+        in_chans: int = 3,
+        embed_dim: int = 768,
+    ):
+        super().__init__()
+        img_size = (img_size, img_size) if isinstance(img_size, int) else img_size
+        patch_size = (patch_size, patch_size) if isinstance(patch_size, int) else patch_size
+        num_patches = (img_size[1] // patch_size[1]) * (img_size[0] // patch_size[0])
+        self.patch_shape = (img_size[0] // patch_size[0], img_size[1] // patch_size[1])
+        self.img_size = img_size
+        self.patch_size = patch_size
+        self.num_patches = num_patches
+
+        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)
+
+    def forward(self, x, **kwargs):
+        B, C, H, W = x.shape
+        assert (
+            H == self.img_size[0] and W == self.img_size[1]
+        ), f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})."
+        x = self.proj(x).flatten(2).transpose(1, 2)
+        return x
+
+
+class RelativePositionBias(nn.Module):
+    def __init__(self, window_size, num_heads):
+        super().__init__()
+        self.window_size = window_size
+        self.num_relative_distance = (2 * window_size[0] - 1) * (2 * window_size[1] - 1) + 3
+        self.relative_position_bias_table = nn.Parameter(
+            torch.zeros(self.num_relative_distance, num_heads)
+        )  # 2*Wh-1 * 2*Ww-1, nH
+        # cls to token & token 2 cls & cls to cls
+
+        # get pair-wise relative position index for each token inside the window
+        coords_h = torch.arange(window_size[0])
+        coords_w = torch.arange(window_size[1])
+        coords = torch.stack(torch.meshgrid([coords_h, coords_w]))  # 2, Wh, Ww
+        coords_flatten = torch.flatten(coords, 1)  # 2, Wh*Ww
+        relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :]  # 2, Wh*Ww, Wh*Ww
+        relative_coords = relative_coords.permute(1, 2, 0).contiguous()  # Wh*Ww, Wh*Ww, 2
+        relative_coords[:, :, 0] += window_size[0] - 1  # shift to start from 0
+        relative_coords[:, :, 1] += window_size[1] - 1
+        relative_coords[:, :, 0] *= 2 * window_size[1] - 1
+        relative_position_index = torch.zeros(
+            size=(window_size[0] * window_size[1] + 1,) * 2, dtype=relative_coords.dtype
+        )
+        relative_position_index[1:, 1:] = relative_coords.sum(-1)  # Wh*Ww, Wh*Ww
+        relative_position_index[0, 0:] = self.num_relative_distance - 3
+        relative_position_index[0:, 0] = self.num_relative_distance - 2
+        relative_position_index[0, 0] = self.num_relative_distance - 1
+
+        self.register_buffer("relative_position_index", relative_position_index)
+
+    def forward(self):
+        relative_position_bias = self.relative_position_bias_table[self.relative_position_index.view(-1)].view(
+            self.window_size[0] * self.window_size[1] + 1, self.window_size[0] * self.window_size[1] + 1, -1
+        )  # Wh*Ww,Wh*Ww,nH
+        return relative_position_bias.permute(2, 0, 1).contiguous()  # nH, Wh*Ww, Wh*Ww
+
+
+class VisionTransformer(nn.Module):
+    """Vision Transformer with support for patch or hybrid CNN input stage"""
+
+    def __init__(
+        self,
+        img_size=224,
+        patch_size=16,
+        in_chans=3,
+        num_classes=1000,
+        embed_dim=768,
+        depth=12,
+        num_heads=12,
+        mlp_ratio=4.0,
+        qkv_bias=False,
+        qk_scale=None,
+        drop_rate=0.0,
+        attn_drop_rate=0.0,
+        drop_path_rate=0.0,
+        norm_layer=nn.LayerNorm,
+        init_values=None,
+        use_abs_pos_emb=True,
+        use_rel_pos_bias=False,
+        use_shared_rel_pos_bias=False,
+        use_mean_pooling=True,
+        init_scale=0.001,
+        checkpoint_activations: bool = False,
+        checkpoint_attention: bool = False,
+        transformer_engine: bool = False,
+        use_fp8: bool = False,
+    ):
+        super().__init__()
+        self.image_size = img_size
+        self.patch_size = patch_size
+        self.num_classes = num_classes
+        self.num_features = self.embed_dim = embed_dim  # num_features for consistency with other models
+        self.transformer_engine = transformer_engine
+        self.use_fp8 = use_fp8
+        self.fp8_recipe = None
+
+        if use_fp8 and not transformer_engine:
+            raise ValueError("`transformer_engine` must be enabled for `use_fp8`.")
+        if use_fp8:
+            # FP8 Recipe: Hybrid E4M3 forward, E5M2 backward
+            self.fp8_recipe = DelayedScaling(fp8_format=Format.HYBRID, amax_history_len=16, amax_compute_algo="max")
+
+        self.patch_embed = PatchEmbed(img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)
+        num_patches = self.patch_embed.num_patches
+
+        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
+        if use_abs_pos_emb:
+            self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + 1, embed_dim))
+        else:
+            self.pos_embed = None
+        self.pos_drop = nn.Dropout(p=drop_rate)
+
+        if use_shared_rel_pos_bias:
+            self.rel_pos_bias = RelativePositionBias(window_size=self.patch_embed.patch_shape, num_heads=num_heads)
+        else:
+            self.rel_pos_bias = None
+        self.checkpoint_activations = checkpoint_activations
+        self.checkpoint_attention = checkpoint_attention
+
+        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule
+        self.use_rel_pos_bias = use_rel_pos_bias
+        self.blocks = nn.ModuleList(
+            [
+                Block(
+                    dim=embed_dim,
+                    num_heads=num_heads,
+                    mlp_ratio=mlp_ratio,
+                    qkv_bias=qkv_bias,
+                    qk_scale=qk_scale,
+                    drop=drop_rate,
+                    attn_drop=attn_drop_rate,
+                    drop_path=dpr[i],
+                    norm_layer=norm_layer,
+                    init_values=init_values,
+                    window_size=self.patch_embed.patch_shape if use_rel_pos_bias else None,
+                    transformer_engine=transformer_engine,
+                    checkpoint_attention=self.checkpoint_attention,
+                )
+                for i in range(depth)
+            ]
+        )
+
+        if self.pos_embed is not None:
+            nn.init.trunc_normal_(self.pos_embed, std=0.02)
+        nn.init.trunc_normal_(self.cls_token, std=0.02)
+
+        self.apply(self._init_weights)
+        self.fix_init_weight()
+
+    def fix_init_weight(self):
+        def rescale(param, layer_id):
+            param.div_(math.sqrt(2.0 * layer_id))
+
+        for layer_id, layer in enumerate(self.blocks):
+            rescale(layer.attn.proj.weight.data, layer_id + 1)
+            rescale(layer.mlp.fc2.weight.data, layer_id + 1)
+
+    def _init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            nn.init.trunc_normal_(m.weight, std=0.02)
+            if isinstance(m, nn.Linear) and m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.LayerNorm):
+            nn.init.constant_(m.bias, 0)
+            nn.init.constant_(m.weight, 1.0)
+
+    def get_classifier(self):
+        return self.head
+
+    def reset_classifier(self, num_classes, global_pool=""):
+        self.num_classes = num_classes
+        self.head = nn.Linear(self.embed_dim, num_classes) if num_classes > 0 else nn.Identity()
+
+    def forward_features(self, x):
+        if self.transformer_engine and self.use_fp8:
+            with te.fp8_autocast(enabled=True, fp8_recipe=self.fp8_recipe):
+                return self._forward_uncast(x)
+        return self._forward_uncast(x)
+
+    def _forward_uncast(self, x):
+        x = self.patch_embed(x)
+        batch_size, seq_len, _ = x.size()
+
+        cls_tokens = self.cls_token.expand(batch_size, -1, -1)
+        x = torch.cat((cls_tokens, x), dim=1)
+        if self.pos_embed is not None:
+            x = x + self.pos_embed
+        x = self.pos_drop(x)
+
+        rel_pos_bias = self.rel_pos_bias() if self.rel_pos_bias is not None else None
+        for blk in self.blocks:
+            if self.checkpoint_activations:
+                x = checkpoint.checkpoint(blk, x, rel_pos_bias)
+            else:
+                x = blk(x, rel_pos_bias)
+        return x
+
+    def forward(self, x):
+        x = self.forward_features(x)
+        return x
+
+    def get_intermediate_layers(self, x):
+        x = self.patch_embed(x)
+        batch_size, seq_len, _ = x.size()
+
+        cls_tokens = self.cls_token.expand(batch_size, -1, -1)
+        x = torch.cat((cls_tokens, x), dim=1)
+        if self.pos_embed is not None:
+            x = x + self.pos_embed
+        x = self.pos_drop(x)
+
+        features = []
+        rel_pos_bias = self.rel_pos_bias() if self.rel_pos_bias is not None else None
+        for blk in self.blocks:
+            x = blk(x, rel_pos_bias)
+            features.append(x)
+
+        return features
+
+    def get_num_layer(self, var_name=""):
+        if var_name in ("cls_token", "mask_token", "pos_embed"):
+            return 0
+        elif var_name.startswith("patch_embed"):
+            return 0
+        elif var_name.startswith("rel_pos_bias"):
+            return len(self.blocks) - 1
+        elif var_name.startswith("blocks"):
+            layer_id = int(var_name.split(".")[1])
+            return layer_id + 1
+        else:
+            return len(self.blocks)
+
+
+def interpolate_pos_embed(
+    pos_embed_key: str,
+    num_patches: int,
+    patch_embed_shape: torch.Size,
+    checkpoint_model: dict[str, torch.Tensor],
+    target_h: int = None,
+    target_w: int = None,
+) -> None:
+    if pos_embed_key in checkpoint_model:
+        pos_embed_checkpoint = checkpoint_model[pos_embed_key].float()
+        embedding_size = pos_embed_checkpoint.shape[-1]
+        num_extra_tokens = patch_embed_shape - num_patches
+        # height (== width) for the checkpoint position embedding
+        orig_size = int((pos_embed_checkpoint.shape[-2] - num_extra_tokens) ** 0.5)
+
+        # If target dimensions are provided, use them; otherwise assume square
+        if target_h is not None and target_w is not None:
+            new_h, new_w = target_h, target_w
+        else:
+            # height (== width) for the new position embedding (square assumption)
+            new_size = int(num_patches**0.5)
+            new_h, new_w = new_size, new_size
+
+        # class_token and dist_token are kept unchanged
+        if orig_size * orig_size != new_h * new_w:
+            logger.info("Positional interpolation from %dx%d to %dx%d" % (orig_size, orig_size, new_h, new_w))
+            extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens]
+            # only the position tokens are interpolated
+            pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:]
+            pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2)
+            pos_tokens = torch.nn.functional.interpolate(
+                pos_tokens, size=(new_h, new_w), mode="bicubic", align_corners=False
+            )
+            pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2)
+            new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1)
+            checkpoint_model[pos_embed_key] = new_pos_embed
+
+
+class PositionalEmbeddingHook:
+    def __init__(self, pos_embed_name, num_patches, patch_embed_shape, target_h=None, target_w=None):
+        self.pos_embed_name = pos_embed_name
+        self.num_patches = num_patches
+        self.patch_embed_shape = patch_embed_shape
+        self.target_h = target_h
+        self.target_w = target_w
+
+    def __call__(self, state_dict, prefix, *args, **kwargs) -> None:
+        logger.info("Calling `PositionalEmbeddingHook`")
+        pos_embed_key = f"{prefix}{self.pos_embed_name}"
+        interpolate_pos_embed(
+            pos_embed_key, self.num_patches, self.patch_embed_shape, state_dict, self.target_h, self.target_w
+        )
+
+
+class EvaViTG(VisionTransformer):
+    def __init__(
+        self,
+        img_size: Union[int, Tuple[int, int]] = 224,
+        drop_path_rate: float = 0.4,
+        pretrained: bool = False,
+        checkpoint_path: Optional[str] = None,
+        checkpoint_activations: bool = False,
+        checkpoint_attention: bool = False,
+        transformer_engine: bool = False,
+        use_fp8: bool = False,
+        **kwargs: Any,
+    ) -> None:
+        if not TRANSFORMER_ENGINE_AVAILABLE and transformer_engine:
+            raise ValueError(
+                "TransformerEngine is not available, "
+                "please install transformer-engine or set `transformer_engine=False` in config."
+            )
+        if use_fp8 and not transformer_engine:
+            raise ValueError("`transformer_engine` must be enabled for `use_fp8`.")
+        super().__init__(
+            img_size=img_size,
+            patch_size=14,
+            use_mean_pooling=False,
+            embed_dim=1408,
+            depth=39,
+            num_heads=1408 // 88,
+            mlp_ratio=4.3637,
+            qkv_bias=True,
+            drop_path_rate=drop_path_rate,
+            norm_layer=partial(nn.LayerNorm, eps=1e-6),
+            checkpoint_activations=checkpoint_activations,
+            checkpoint_attention=checkpoint_attention,
+            transformer_engine=transformer_engine,
+            use_fp8=use_fp8,
+        )
+        self.checkpoint_path = checkpoint_path
+
+        # compatibility with pre-trained checkpoints
+        self.register_pre_hooks()
+
+        # load pre-trained checkpoints
+        if pretrained:
+            self.load_checkpoint()
+
+    def load_checkpoint(self) -> None:
+        logger.info(f"Loading checkpoint from {self.checkpoint_path}")
+        state_dict = torch.load(self.checkpoint_path, map_location="cpu")
+        incompatible_keys = self.load_state_dict(state_dict, strict=False)
+        logger.info(f"Incompatible keys: {incompatible_keys}")
+        logger.info(f"Loaded visual encoder {type(self)} with state dict from {self.checkpoint_path}")
+
+    def register_pre_hooks(self) -> None:
+        """Register positional embedding interpolation when loading pre-trained checkpoints using different resolution."""
+        # Calculate target patch dimensions for non-square support
+        patch_h = self.patch_embed.patch_shape[0]
+        patch_w = self.patch_embed.patch_shape[1]
+
+        embed_hook = PositionalEmbeddingHook(
+            pos_embed_name="pos_embed",
+            num_patches=self.patch_embed.num_patches,
+            patch_embed_shape=self.pos_embed.shape[-2],
+            target_h=patch_h,
+            target_w=patch_w,
+        )
+        self._register_load_state_dict_pre_hook(embed_hook)
+
+    def _initialize_weights(self, m):
+        if isinstance(m, nn.Linear):
+            nn.init.trunc_normal_(m.weight, std=0.02)
+            if isinstance(m, nn.Linear) and m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.LayerNorm):
+            nn.init.constant_(m.bias, 0)
+            nn.init.constant_(m.weight, 1.0)

preprocessing_embed1.py

@@ -0,0 +1,133 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Processor class for Cosmos-Embed1
+"""
+
+from typing import List, Optional, Tuple, Union
+
+import numpy as np
+import torch
+import torchvision
+from transformers import AutoProcessor, BatchFeature
+from transformers.processing_utils import ProcessorMixin
+from transformers.utils import TensorType
+
+from .configuration_embed1 import CosmosEmbed1Config
+
+
+class CosmosEmbed1Processor(ProcessorMixin):
+    r"""
+    Constructs a processor which wraps a BertTokenizer tokenizer and a fast video resize function.
+
+    Args:
+        tokenizer ([`BertTokenizerFast`], *optional*):
+            The tokenizer is a required input for text processing.
+        config ([`CosmosEmbed1Config`], *optional*):
+            Needed for processing options.
+    """
+
+    attributes = ["tokenizer"]
+    tokenizer_class = ("BertTokenizer", "BertTokenizerFast")
+    config_class = CosmosEmbed1Config
+    chat_template = None
+
+    def __init__(
+        self,
+        tokenizer=None,
+        resolution: Union[int, Tuple[int, int]] = 448,
+        num_video_frames: int = 8,
+        max_txt_len: int = 128,
+        **kwargs,
+    ) -> None:
+        super().__init__(tokenizer, **kwargs)
+        self.resolution = resolution
+        self.num_video_frames = num_video_frames
+        self.max_txt_len = max_txt_len
+
+    def __call__(
+        self,
+        text: Optional[Union[str, List[str]]] = None,
+        videos: Optional[Union[np.ndarray, torch.Tensor]] = None,
+        return_tensors: Optional[Union[str, TensorType]] = TensorType.PYTORCH,
+        resolution: Union[int, Tuple[int, int]] = None,
+        num_video_frames: int = None,
+        max_txt_len: int = None,
+        **kwargs,
+    ) -> BatchFeature:
+        inputs = {}
+
+        if text is not None:
+            max_txt_len = max_txt_len if max_txt_len is not None else self.max_txt_len
+            tokenized = self.tokenizer(
+                text, return_tensors="pt", padding="max_length", truncation=True, max_length=max_txt_len, **kwargs
+            )
+            inputs["input_ids"] = tokenized.input_ids
+            inputs["attention_mask"] = tokenized.attention_mask.float()
+
+        if videos is not None:
+            if isinstance(videos, np.ndarray):
+                videos = torch.from_numpy(videos)
+            if not isinstance(videos, torch.Tensor) or videos.ndim != 5:
+                raise ValueError("Processor expects a numpy or torch tensor of shape BTCHW from [0-255].")
+            resolution = resolution if resolution is not None else self.resolution
+            if isinstance(resolution, int):
+                resolution = (resolution, resolution)
+            _, t, c, h, w = videos.shape
+            if c != 3:
+                raise ValueError(f"Expected tensor of shape BTCHW with RGB channels, got channel size {c}.")
+            num_video_frames = num_video_frames if num_video_frames is not None else self.num_video_frames
+            if t != num_video_frames:
+                raise ValueError(f"Expected tensor of shape BTCHW with {num_video_frames} frames, got {t}.")
+            if h != resolution[0] or w != resolution[1]:
+                videos = resize_video(videos, resolution)
+            if videos.dtype == torch.uint8:
+                videos = videos.float()
+            inputs["videos"] = videos / 255.0
+
+        if not inputs:
+            raise ValueError("Must pass either `text` or `videos` argument to __call__ function.")
+
+        return BatchFeature(inputs, tensor_type=return_tensors)
+
+
+def resize_video(video: torch.Tensor, size: Tuple[int, int]) -> torch.Tensor:
+    """Resize a video tensor (B, T, C, H, W) to a new height/width.
+
+    Args:
+        video (torch.Tensor): (B, T, C, H, W) uint8 or float32.
+        size (tuple): target (H', W') size.
+    Returns:
+        torch.Tensor: resized video of shape (B, T, C, H', W')
+    """
+    h, w = size
+    B, T, C, H, W = video.shape
+    video = video.view(B * T, C, H, W)
+    resize = torchvision.transforms.Resize(
+        (h, w),
+        antialias=True,
+        interpolation=torchvision.transforms.InterpolationMode.BILINEAR,
+    )
+    video = resize(video)
+    new_H, new_W = video.shape[-2:]
+    video = video.view(B, T, C, new_H, new_W)
+    return video
+
+
+AutoProcessor.register(CosmosEmbed1Config, CosmosEmbed1Processor)
+
+
+__all__ = ["CosmosEmbed1Processor"]

tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": true, "model_max_length": 512}