Add JoyFox-OmniFlow-9B

Initial complete model release.

.gitattributes

@@ -33,3 +33,28 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/HT_ref_audio.wav filter=lfs diff=lfs merge=lfs -text
+assets/Skiing.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/Trump_WEF_2018_10s.mp3 filter=lfs diff=lfs merge=lfs -text
+assets/audio_cases/assistant_response.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/audio_cases/elon_musk__000_assistant_audio.wav filter=lfs diff=lfs merge=lfs -text
+assets/audio_cases/elon_musk__system_ref_audio.wav filter=lfs diff=lfs merge=lfs -text
+assets/audio_cases/elon_musk_ref.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/audio_cases/elon_musk_response.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/audio_cases/hermione__000_assistant_audio.wav filter=lfs diff=lfs merge=lfs -text
+assets/audio_cases/hermione__system_ref_audio.wav filter=lfs diff=lfs merge=lfs -text
+assets/audio_cases/minicpm_assistant__000_assistant_audio.wav filter=lfs diff=lfs merge=lfs -text
+assets/audio_cases/minicpm_assistant__system_ref_audio.wav filter=lfs diff=lfs merge=lfs -text
+assets/audio_cases/paimon__000_assistant_audio.wav filter=lfs diff=lfs merge=lfs -text
+assets/audio_cases/paimon__system_ref_audio.wav filter=lfs diff=lfs merge=lfs -text
+assets/bajie.wav filter=lfs diff=lfs merge=lfs -text
+assets/fossil.png filter=lfs diff=lfs merge=lfs -text
+assets/haimianbaobao.wav filter=lfs diff=lfs merge=lfs -text
+assets/highway.png filter=lfs diff=lfs merge=lfs -text
+assets/nezha.wav filter=lfs diff=lfs merge=lfs -text
+assets/omni_duplex1.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/omni_duplex2.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/sunwukong.wav filter=lfs diff=lfs merge=lfs -text
+assets/system_ref_audio.wav filter=lfs diff=lfs merge=lfs -text
+assets/system_ref_audio_2.wav filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

Modelfile

@@ -0,0 +1,16 @@
+# ollama modelfile auto-generated by llamafactory
+
+FROM .
+
+TEMPLATE """{{ if .System }}<|im_start|>system
+{{ .System }}<|im_end|>
+{{ end }}{{ range .Messages }}{{ if eq .Role "user" }}<|im_start|>user
+{{ .Content }}<|im_end|>
+<|im_start|>assistant
+{{ else if eq .Role "assistant" }}{{ .Content }}<|im_end|>
+{{ end }}{{ end }}"""
+
+SYSTEM """You are a helpful assistant. You can accept audio and text input and output voice and text."""
+
+PARAMETER stop "<|im_end|>"
+PARAMETER num_ctx 4096

README.md

@@ -0,0 +1,264 @@
+---
+license: apache-2.0
+language:
+- en
+- zh
+base_model: openbmb/MiniCPM-o-4_5
+library_name: transformers
+pipeline_tag: any-to-any
+tags:
+- minicpm-o
+- full-duplex
+- omni-modal
+- audio
+- vision
+- tts
+- speech-generation
+- tool-calling
+- roleplay
+- code
+- math
+---
+
+# JoyFox-OmniFlow-9B
+
+JoyFox-OmniFlow-9B is a general-purpose, full-duplex multimodal model built on
+[OpenBMB MiniCPM-o 4.5](https://huggingface.co/openbmb/MiniCPM-o-4_5).
+
+The model is designed for:
+
+- English, Chinese, and multilingual conversations;
+- instruction following and complex question answering;
+- mathematical reasoning and code generation;
+- knowledge-intensive and long-form responses;
+- role-play, character-driven dialogue, and natural conversation;
+- agent workflows and tool-calling formats;
+- real-time vision, audio, text, and speech interaction.
+
+JoyFox-OmniFlow-9B is distributed as a complete BF16 Safetensors model that can
+be loaded directly with Hugging Face Transformers. The release includes the
+language model, vision encoder, audio encoder, multimodal projection layers,
+TTS module, voice-cloning support, token-to-waveform resources, and full-duplex
+streaming components.
+
+## Highlights
+
+- **Complete full-duplex model**: supports concurrent visual/audio input and text/speech output.
+- **Integrated TTS**: includes speech generation weights, token2wav resources, and reference audio assets.
+- **Broad language capabilities**: covers general instruction following, math, code, multilingual dialogue, role-play, and tool use.
+- **Omni-modal interaction**: combines language, vision, audio, multimodal projection, and streaming session modules.
+- **Voice cloning and character voices**: supports reference-audio-driven speech generation.
+- **Ready for GPU inference**: fully validated on an NVIDIA RTX PRO 6000 Blackwell GPU.
+- **Standard Transformers package**: no additional model-weight package is required at load time.
+
+## Capability Overview
+
+| Component | Capability |
+|---|---|
+| Language model | General dialogue, knowledge, instruction following, math, code, and role-play |
+| Vision module | Image, video-frame, and continuous visual-stream understanding |
+| Audio module | Speech, environmental audio, and continuous audio-stream understanding |
+| Multimodal projection | Alignment of language, vision, and audio representations |
+| TTS | Natural text-to-speech generation |
+| token2wav | Decoding speech tokens into waveform audio |
+| Full-duplex streaming | Simultaneous seeing, listening, reasoning, and speaking |
+| Voice cloning | Reference-audio-based voice control |
+
+## Model Package
+
+| Item | Value |
+|---|---|
+| Model class | `MiniCPMO` |
+| Model family | MiniCPM-o 4.5 |
+| Weight precision | BF16 |
+| Total parameters | 9,371,787,666 |
+| TTS parameters | 347,696,290 |
+| Weight keys | 1,414 |
+| TTS weight keys | 194 |
+| Safetensors shards | 5 |
+| Total weight size | 18,743,575,332 bytes |
+| Complete directory size | Approximately 18.67 GiB |
+| Audio module | Included |
+| Vision module | Included |
+| TTS module | Included |
+| Full-duplex streaming configuration | Enabled |
+
+## Integrity Validation
+
+The released model passed the following checks:
+
+- all 1,414 model weight keys are readable;
+- all 194 TTS weight tensors are readable;
+- the token2wav runtime assets are present;
+- `init_audio=true`;
+- `init_vision=true`;
+- `init_tts=true`;
+- `stream_input=true`;
+- zero missing keys during a complete Transformers load;
+- zero unexpected keys during a complete Transformers load;
+- zero mismatched keys during a complete Transformers load.
+
+### GPU Load Validation
+
+The complete BF16 model was successfully loaded on an NVIDIA RTX PRO 6000
+Blackwell Server Edition:
+
+- `llm`: CUDA
+- `vpm`: CUDA
+- `apm`: CUDA
+- `tts`: CUDA
+- `audio_projection_layer`: CUDA
+- `resampler`: CUDA
+- model-weight VRAM usage: approximately 17.56 GiB
+
+The VRAM figure above primarily reflects model weights and small framework
+overhead. Full-duplex inference requires additional memory for the KV cache,
+video frames, audio buffers, and token2wav.
+
+## Installation
+
+For text, image, or audio-understanding inference:
+
+```bash
+pip install "transformers==4.51.0" accelerate \
+  "torch>=2.3.0,<=2.8.0" "torchaudio<=2.8.0" \
+  "minicpmo-utils>=1.0.5"
+```
+
+For TTS or full-duplex streaming inference:
+
+```bash
+pip install "transformers==4.51.0" accelerate \
+  "torch>=2.3.0,<=2.8.0" "torchaudio<=2.8.0" \
+  "minicpmo-utils[all]>=1.0.5"
+```
+
+FFmpeg is also required for full-duplex video processing:
+
+```bash
+sudo apt update
+sudo apt install -y ffmpeg
+```
+
+The model structure was also validated locally with `transformers==4.57.0` and
+`torch==2.11.0+cu130`. For production full-duplex use, the MiniCPM-o 4.5
+officially recommended dependency versions are preferred.
+
+## GPU Quick Start
+
+```python
+import torch
+from transformers import AutoModel, AutoProcessor
+
+model_id = "joyfox/JoyFox-OmniFlow-9B"
+
+model = AutoModel.from_pretrained(
+    model_id,
+    trust_remote_code=True,
+    attn_implementation="sdpa",
+    torch_dtype=torch.bfloat16,
+    init_vision=True,
+    init_audio=True,
+    init_tts=True,
+)
+processor = AutoProcessor.from_pretrained(
+    model_id,
+    trust_remote_code=True,
+)
+
+model.eval().cuda()
+```
+
+## Enabling TTS and Full-Duplex Mode
+
+After installing `minicpmo-utils[all]`:
+
+```python
+# Initialize token2wav for speech-token-to-waveform decoding.
+model.init_tts()
+
+# Enter full-duplex mode.
+duplex_model = model.as_duplex()
+
+# Return to standard half-duplex mode when needed.
+model = duplex_model.as_simplex(reset_session=True)
+```
+
+Real-time video/audio chunking, `streaming_prefill`, `streaming_generate`, and
+output-video generation follow the standard MiniCPM-o 4.5 full-duplex API.
+
+## Speech Output Example
+
+```python
+import librosa
+import torch
+from huggingface_hub import hf_hub_download
+
+ref_audio_path = hf_hub_download(
+    repo_id=model_id,
+    filename="assets/HT_ref_audio.wav",
+)
+ref_audio, _ = librosa.load(ref_audio_path, sr=16000, mono=True)
+
+model.init_tts()
+system_message = model.get_sys_prompt(
+    ref_audio=ref_audio,
+    mode="omni",
+    language="en",
+)
+
+messages = [
+    system_message,
+    {
+        "role": "user",
+        "content": ["Introduce yourself in a natural and concise way."],
+    },
+]
+
+with torch.inference_mode():
+    answer = model.chat(
+        msgs=messages,
+        do_sample=True,
+        temperature=0.7,
+        max_new_tokens=1024,
+        use_tts_template=True,
+        enable_thinking=False,
+        omni_mode=True,
+        generate_audio=True,
+        output_audio_path="output.wav",
+    )
+
+print(answer)
+```
+
+## Intended Use
+
+JoyFox-OmniFlow-9B is suitable for:
+
+- general English, Chinese, and multilingual assistants;
+- mathematical, coding, knowledge, and complex instruction tasks;
+- role-play and character-driven dialogue;
+- agent and tool-calling applications;
+- multimodal assistants with visual and audio input;
+- half-duplex conversations with speech output;
+- real-time full-duplex video and audio interaction;
+- reference-audio-based voice cloning and character speech.
+
+## Limitations
+
+- This release does not report independent external benchmark results.
+- Response style depends on the system prompt, chat template, and decoding configuration.
+- Production tool-calling systems should validate schemas, arguments, and execution permissions.
+- The model may produce inaccurate, biased, or inappropriate content; high-risk use cases require human review.
+- Full-duplex quality depends on audio sampling, video chunking, system prompts, reference voices, and runtime implementation.
+- Full-duplex TTS requires the additional runtime dependencies included with `minicpmo-utils[all]`.
+
+## License and Acknowledgements
+
+This model is built on MiniCPM-o 4.5 and is subject to the applicable upstream
+license requirements. The model card declares Apache-2.0 metadata. Before
+deployment, redistribution, or commercial use, review the original MiniCPM-o
+4.5 license and the terms applicable to your intended use.
+
+We thank OpenBMB, LLaMA-Factory, and the open-source community for the model,
+tooling, and supporting resources used in this project.

added_tokens.json

@@ -0,0 +1,107 @@
+{
+  "</answer>": 151686,
+  "</box>": 151674,
+  "</focus>": 151688,
+  "</image>": 151670,
+  "</image_id>": 151682,
+  "</image_save_to>": 151696,
+  "</line>": 151690,
+  "</perception>": 151692,
+  "</point>": 151678,
+  "</quad>": 151676,
+  "</ref>": 151672,
+  "</slice>": 151680,
+  "</source_image>": 151694,
+  "</think>": 151668,
+  "</tool_call>": 151658,
+  "</tool_response>": 151666,
+  "</unit>": 151684,
+  "<answer>": 151685,
+  "<box>": 151673,
+  "<focus>": 151687,
+  "<image>": 151669,
+  "<image_id>": 151681,
+  "<image_save_to>": 151695,
+  "<line>": 151689,
+  "<perception>": 151691,
+  "<point>": 151677,
+  "<quad>": 151675,
+  "<ref>": 151671,
+  "<slice>": 151679,
+  "<source_image>": 151693,
+  "<think>": 151667,
+  "<tool_call>": 151657,
+  "<tool_response>": 151665,
+  "<unit>": 151683,
+  "<|audio_end|>": 151699,
+  "<|audio_start|>": 151697,
+  "<|audio|>": 151698,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|chunk_bos|>": 151719,
+  "<|chunk_eos|>": 151718,
+  "<|chunk_tts_bos|>": 151720,
+  "<|chunk_tts_eos|>": 151721,
+  "<|emotion_end|>": 151711,
+  "<|emotion_start|>": 151710,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|interrupt|>": 151707,
+  "<|listen|>": 151705,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|pitch_end|>": 151715,
+  "<|pitch_start|>": 151714,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|speak|>": 151706,
+  "<|speed_end|>": 151713,
+  "<|speed_start|>": 151712,
+  "<|spk_bos|>": 151700,
+  "<|spk_eos|>": 151702,
+  "<|spk|>": 151701,
+  "<|timbre_10|>": 151726,
+  "<|timbre_11|>": 151727,
+  "<|timbre_12|>": 151728,
+  "<|timbre_13|>": 151729,
+  "<|timbre_14|>": 151730,
+  "<|timbre_15|>": 151731,
+  "<|timbre_16|>": 151732,
+  "<|timbre_17|>": 151733,
+  "<|timbre_18|>": 151734,
+  "<|timbre_19|>": 151735,
+  "<|timbre_20|>": 151736,
+  "<|timbre_21|>": 151737,
+  "<|timbre_22|>": 151738,
+  "<|timbre_23|>": 151739,
+  "<|timbre_24|>": 151740,
+  "<|timbre_25|>": 151741,
+  "<|timbre_26|>": 151742,
+  "<|timbre_27|>": 151743,
+  "<|timbre_28|>": 151744,
+  "<|timbre_29|>": 151745,
+  "<|timbre_30|>": 151746,
+  "<|timbre_31|>": 151747,
+  "<|timbre_7|>": 151723,
+  "<|timbre_8|>": 151724,
+  "<|timbre_9|>": 151725,
+  "<|tts_bos|>": 151703,
+  "<|tts_eos|>": 151704,
+  "<|tts_pad|>": 151722,
+  "<|turn_bos|>": 151716,
+  "<|turn_eos|>": 151717,
+  "<|vad_end|>": 151709,
+  "<|vad_start|>": 151708,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

assets/HT_ref_audio.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb8f06ba5080cdf548969138881fb8ad8b04e2516108f4e08ba0363b68b613ea
+size 192590

assets/Skiing.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:479ace116d6ac92487ad90f415b3ef817cd019bba4521043ef0d5faaa1a8415d
+size 8534409

assets/Trump_WEF_2018_10s.mp3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4fb796c2bb95538eab22d9b68a31add560305b6d5ccbf150c3f96e7671b6db64
+size 161053

assets/audio_cases/assistant_response.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d46268e3beb78b4287a896a09b2df0f895629ecbdcdc4cf3c519fe040084860f
+size 269504

assets/audio_cases/elon_musk__000_assistant_audio.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff6dced2e686b743f39eee36aef1410913c8d31459bb564dbbf8f81ef8a2da88
+size 1762604

assets/audio_cases/elon_musk__system_ref_audio.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c4109b2d685e1923ed66433eb08c92047a1f67510629a27edf49af4e5c606dd
+size 539032

assets/audio_cases/elon_musk_ref.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:206c48ec4b08898fbfbfb1ea1fec058750c39c97cd2e5088b15b61fb194d6f7e
+size 165179

assets/audio_cases/elon_musk_response.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d83c3a977fd444351710c9ba2c347c90efd49ed990276c5309867374df373a4f
+size 388534

assets/audio_cases/hermione__000_assistant_audio.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfd18a2ee9e27fdcd65e2d9bf0fe0b3c9ebc28cda6c43279c12e077889c2b88d
+size 1534124

assets/audio_cases/hermione__system_ref_audio.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46bd82796ce5ebc15b29bc21c1de38284cfa972b2f27cca1f17086516a757dee
+size 197322

assets/audio_cases/minicpm_assistant__000_assistant_audio.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe3b793a6436682d1e684e0f4c158718208ed38926880e01ee84f5aa61c78331
+size 1246124

assets/audio_cases/minicpm_assistant__system_ref_audio.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad576b50fd2f53ad4bb317b97ed33bc7027071e7e0811a2a2110d995614f1f42
+size 192556

assets/audio_cases/paimon__000_assistant_audio.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ddd9d8358950a8dab385b8031cb8371a832460f3db4d91e399373705158f346
+size 697004

assets/audio_cases/paimon__system_ref_audio.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9625e7115ff1f47cf842fecbb31db82057ff1863c080870539c31107aa93b9d
+size 479304

assets/audio_cases/readme.txt

@@ -0,0 +1 @@
+This directory stores audio wav files for hf readme page.

assets/bajie.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16aa8ca3da7dad53680bac68cfde569e11e2f9ef5387b3ae60d68626e94db9d7
+size 636512

assets/haimianbaobao.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27405cf5977f04d3f248693f9b978654fad4f02f931d9746006bffb0ed7b66e1
+size 343120

assets/nezha.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe5d8932013ff30a9d8114a6d62e5342999d8c2ddf8509ecfeb1ead71cacf432
+size 457802

assets/omni_duplex1.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31622e1efd9a7b197a340266037b45aeec13b3b27f010f1ea1d22d9c6e69405f
+size 7295040

assets/omni_duplex2.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c04eaef27a821e18db3686fdcc9d4b1dafd441e08f6a0e12d4075b81cb04517e
+size 29285216

assets/sunwukong.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bdb6c175bd3b2bb02fe7a8cf0ced847d41ab552b2e318bc3c26ee477255eff7
+size 644650

assets/system_ref_audio.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c4109b2d685e1923ed66433eb08c92047a1f67510629a27edf49af4e5c606dd
+size 539032

assets/system_ref_audio_2.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a65d17099239711601cb17e28b7aa7b6149acd153b92fd0366c23902bd4f687
+size 341192

assets/token2wav/campplus.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6ac6a63997761ae2997373e2ee1c47040854b4b759ea41ec48e4e42df0f4d73
+size 28303423

assets/token2wav/flow.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15ccff24256ff61537c7f8b51e025116b83405f3fb017b54b008fc97da115446
+size 623466603

assets/token2wav/flow.yaml

@@ -0,0 +1,34 @@
+flow: !new:cosyvoice2.flow.flow.CausalMaskedDiffWithXvec
+    input_size: 512
+    output_size: 80
+    spk_embed_dim: 192
+    output_type: 'mel'
+    vocab_size: 6561
+    encoder: !new:cosyvoice2.transformer.upsample_encoder_v2.UpsampleConformerEncoderV2
+        input_size: 512
+        output_size: 512
+        input_layer: 'linear'
+        pre_lookahead_len: 3
+        num_blocks: 6
+        num_up_blocks: 4
+        up_stride: 2
+        up_scale_factor: 2
+        attention_heads: 8
+        pos_enc_layer_type: 'rel_pos_espnet'
+        selfattention_layer_type: 'rel_selfattn'
+        key_bias: true
+        linear_units: 2048
+        dropout_rate: 0.1
+        positional_dropout_rate: 0.1
+        attention_dropout_rate: 0.1
+        normalize_before: True
+    decoder: !new:cosyvoice2.flow.flow_matching.CausalConditionalCFM
+        inference_cfg_rate: 0.7
+        estimator: !new:cosyvoice2.flow.decoder_dit.DiT
+            in_channels: 320
+            out_channels: 80
+            mlp_ratio: 4.0
+            depth: 16
+            num_heads: 8
+            head_dim: 64
+            hidden_size: 512

assets/token2wav/hift.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3386cc880324d4e98e05987b99107f49e40ed925b8ecc87c1f4939432d429879
+size 83390254

assets/token2wav/speech_tokenizer_v2_25hz.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d43342aa12163a80bf07bffb94c9de2e120a8df2f9917cd2f642e7f4219c6f71
+size 496082973

chat_template.jinja

@@ -0,0 +1,88 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set content = message.content %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is defined and message.reasoning_content is not none %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in message.content %}
+                {%- set content = message.content.split('</think>')[-1].lstrip('\n') %}
+                {%- set reasoning_content = message.content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+    {%- if use_tts_template is defined and use_tts_template is true %}
+        {{- '<|tts_bos|>' }}
+    {%- endif %}
+{%- endif %}

config.json

@@ -0,0 +1,324 @@
+{
+  "architectures": [
+    "MiniCPMO"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "audio_chunk_length": 1.0,
+  "audio_config": {
+    "_attn_implementation_autoset": true,
+    "_name_or_path": "openai/whisper-medium",
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "apply_spec_augment": false,
+    "architectures": [
+      "MiniCPMWhisperEncoder"
+    ],
+    "attention_dropout": 0.0,
+    "begin_suppress_tokens": [
+      220,
+      50257
+    ],
+    "bos_token_id": 50257,
+    "classifier_proj_size": 256,
+    "d_model": 1024,
+    "decoder_attention_heads": 16,
+    "decoder_ffn_dim": 4096,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 24,
+    "decoder_start_token_id": 50258,
+    "dropout": 0.0,
+    "dtype": "float32",
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 50257,
+    "forced_decoder_ids": [
+      [
+        1,
+        50259
+      ],
+      [
+        2,
+        50359
+      ],
+      [
+        3,
+        50363
+      ]
+    ],
+    "init_std": 0.02,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_length": 448,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "model_type": "whisper",
+    "num_hidden_layers": 24,
+    "num_mel_bins": 80,
+    "pad_token_id": 50257,
+    "scale_embedding": false,
+    "suppress_tokens": [
+      1,
+      2,
+      7,
+      8,
+      9,
+      10,
+      14,
+      25,
+      26,
+      27,
+      28,
+      29,
+      31,
+      58,
+      59,
+      60,
+      61,
+      62,
+      63,
+      90,
+      91,
+      92,
+      93,
+      359,
+      503,
+      522,
+      542,
+      873,
+      893,
+      902,
+      918,
+      922,
+      931,
+      1350,
+      1853,
+      1982,
+      2460,
+      2627,
+      3246,
+      3253,
+      3268,
+      3536,
+      3846,
+      3961,
+      4183,
+      4667,
+      6585,
+      6647,
+      7273,
+      9061,
+      9383,
+      10428,
+      10929,
+      11938,
+      12033,
+      12331,
+      12562,
+      13793,
+      14157,
+      14635,
+      15265,
+      15618,
+      16553,
+      16604,
+      18362,
+      18956,
+      20075,
+      21675,
+      22520,
+      26130,
+      26161,
+      26435,
+      28279,
+      29464,
+      31650,
+      32302,
+      32470,
+      36865,
+      42863,
+      47425,
+      49870,
+      50254,
+      50258,
+      50358,
+      50359,
+      50360,
+      50361,
+      50362
+    ],
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51865
+  },
+  "audio_pool_step": 5,
+  "auto_map": {
+    "AutoConfig": "configuration_minicpmo.MiniCPMOConfig",
+    "AutoModel": "modeling_minicpmo.MiniCPMO",
+    "AutoModelForCausalLM": "modeling_minicpmo.MiniCPMO"
+  },
+  "batch_vision_input": true,
+  "bos_token_id": 151643,
+  "drop_vision_last_layer": false,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "image_size": 448,
+  "init_audio": true,
+  "init_tts": true,
+  "init_vision": true,
+  "initializer_range": 0.02,
+  "intermediate_size": 12288,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "listen_speak_type": "asr",
+  "max_position_embeddings": 40960,
+  "max_window_layers": 36,
+  "model_type": "minicpmo",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 8,
+  "patch_size": 14,
+  "query_num": 64,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000,
+  "slice_config": {
+    "max_slice_nums": 1,
+    "model_type": "minicpmv",
+    "patch_size": 14,
+    "scale_resolution": 448
+  },
+  "slice_mode": true,
+  "sliding_window": null,
+  "stream_input": true,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.0",
+  "tts_config": {
+    "_attn_implementation_autoset": true,
+    "attention_type": "full_attention",
+    "attn_implementation": "sdpa",
+    "audio_bos_token_id": 151687,
+    "audio_tokenizer_sample_rate": 16000,
+    "audio_tokenizer_type": "s3tokenizer",
+    "aug_layer_loss_weight": false,
+    "aug_loss_weight": false,
+    "backbone_model": "llama",
+    "condition_type": "hidden_text_merge",
+    "cosyvoice_config_path": null,
+    "cosyvoice_model_dir": null,
+    "filter_tts_loss": false,
+    "hidden_act": "silu",
+    "hidden_size": 768,
+    "interleaved": false,
+    "intermediate_size": 3072,
+    "llm_dim": 4096,
+    "llm_dim_model_base": 256,
+    "llm_down_scale": false,
+    "llm_hidden_size": 4096,
+    "llm_intermediate_size": 768,
+    "long_weight": 0.1,
+    "max_position_embeddings": 4096,
+    "model_type": "minicpmtts",
+    "normalize_projected_hidden": true,
+    "num_attention_heads": 12,
+    "num_audio_tokens": 6562,
+    "num_hidden_layers": 20,
+    "num_key_value_heads": 12,
+    "num_mel_bins": 100,
+    "num_text_tokens": 152064,
+    "num_vq": 1,
+    "projector_type": "mlp",
+    "recomputed_chunks": 1,
+    "s3_stream_chunk_size": 25,
+    "s3_stream_generate": false,
+    "s3_stream_n_timesteps": 10,
+    "s3_stream_prelook_size": 3,
+    "short_weight": 0.1,
+    "streaming": false,
+    "streaming_audio_chunk_size": 50,
+    "streaming_sliding_window": false,
+    "streaming_sliding_window_audio_frame_rate": 50,
+    "streaming_sliding_window_audio_init_text_length": 10,
+    "streaming_sliding_window_audio_window_size": 300,
+    "streaming_sliding_window_average_speed": 5,
+    "streaming_sliding_window_fast_speed": 7,
+    "streaming_sliding_window_max_text_len": 500,
+    "streaming_sliding_window_slow_speed": 3,
+    "streaming_sliding_window_text_window_size": 50,
+    "streaming_text_chunk_max": 7,
+    "streaming_text_chunk_min": 3,
+    "streaming_text_reserved_len": 300,
+    "text_eos_token_id": 151692,
+    "tts_filter_loss_fix": false,
+    "use_llm_hidden_state": false,
+    "use_text": true,
+    "window_size": 2
+  },
+  "use_cache": true,
+  "use_image_id": true,
+  "use_sliding_window": false,
+  "version": "4.5",
+  "vision_batch_size": 16,
+  "vision_config": {
+    "_attn_implementation_autoset": true,
+    "attention_dropout": 0.0,
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 1152,
+    "image_size": 980,
+    "intermediate_size": 4304,
+    "layer_norm_eps": 1e-06,
+    "model_type": "siglip_vision_model",
+    "num_attention_heads": 16,
+    "num_channels": 3,
+    "num_hidden_layers": 27,
+    "patch_size": 14
+  },
+  "vocab_size": 151748,
+  "torch_dtype": "bfloat16"
+}

configuration.json

@@ -0,0 +1 @@
+{"framework":"Pytorch","task":"any-to-any"}

configuration_minicpmo.py

@@ -0,0 +1,260 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright 2026 The OpenBMB Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from typing import Union
+
+from transformers import PretrainedConfig
+from transformers import Qwen3Config
+from transformers import WhisperConfig
+from transformers.utils import logging
+
+from .modeling_navit_siglip import SiglipVisionConfig
+
+logger = logging.get_logger(__name__)
+
+
+class MiniCPMVSliceConfig(PretrainedConfig):
+    model_type = "minicpmv"
+
+    def __init__(
+        self,
+        patch_size=14,
+        max_slice_nums=9,
+        scale_resolution=448,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.patch_size = patch_size
+        self.max_slice_nums = max_slice_nums
+        self.scale_resolution = scale_resolution
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
+        cls._set_token_in_kwargs(kwargs)
+
+        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+
+        if config_dict.get("model_type") == "minicpmv":
+            config_dict = config_dict["slice_config"]
+
+        if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
+            logger.warning(
+                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
+                f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
+            )
+
+        return cls.from_dict(config_dict, **kwargs)
+
+
+class MiniCPMTTSConfig(PretrainedConfig):
+    model_type = "minicpmtts"
+
+    def __init__(
+        self,
+        llm_dim: int = 2560,
+        llm_intermediate_size: int = 768,
+        llm_down_scale: bool = False,
+        llm_dim_model_base: int = 256,
+        projector_type: str = "mlp",
+        hidden_act: str = "silu",
+        aug_loss_weight: bool = False,
+        aug_layer_loss_weight: bool = False,
+        filter_tts_loss: bool = False,
+        tts_filter_loss_fix: bool = False,
+        long_weight: float = 0.1,
+        short_weight: float = 0.1,
+        hidden_size: int = 768,
+        intermediate_size: int = 3072,
+        num_attention_heads: int = 12,
+        num_hidden_layers: int = 20,
+        num_key_value_heads: int = 12,
+        max_position_embeddings: int = 4096,
+        num_audio_tokens: int = 4097,
+        num_text_tokens: int = 21178,
+        num_mel_bins: int = 100,
+        num_vq: int = 1,
+        use_llm_hidden_state: bool = False,
+        audio_bos_token_id: int = 21132,
+        text_eos_token_id: int = 21133,
+        use_text: bool = True,
+        streaming: bool = False,
+        streaming_text_chunk_min: int = 3,
+        streaming_text_chunk_max: int = 7,
+        streaming_text_reserved_len: int = 300,
+        streaming_audio_chunk_size: int = 50,
+        attn_implementation: str = "sdpa",
+        condition_type: str = "llm_hidden",
+        backbone_model: str = "llama",
+        audio_tokenizer_type: str = "wavtokenizer",
+        audio_tokenizer_sample_rate: int = 24000,
+        streaming_sliding_window: bool = False,
+        streaming_sliding_window_max_text_len: int = 500,
+        streaming_sliding_window_average_speed: int = 5,
+        streaming_sliding_window_fast_speed: int = 7,
+        streaming_sliding_window_slow_speed: int = 3,
+        streaming_sliding_window_audio_frame_rate: int = 50,
+        streaming_sliding_window_audio_init_text_length: int = 10,
+        streaming_sliding_window_audio_window_size: int = 300,
+        normalize_projected_hidden: bool = False,
+        interleaved: bool = False,
+        attention_type: str = "sliding_recompute",
+        recomputed_chunks: int = 1,
+        window_size: int = 2,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.llm_dim = llm_dim
+        self.llm_hidden_size = llm_dim
+        self.llm_intermediate_size = llm_intermediate_size
+        self.llm_down_scale = llm_down_scale
+        self.llm_dim_model_base = llm_dim_model_base
+        self.projector_type = projector_type
+        self.aug_loss_weight = aug_loss_weight
+        self.aug_layer_loss_weight = aug_layer_loss_weight
+        self.tts_filter_loss_fix = tts_filter_loss_fix
+        self.filter_tts_loss = filter_tts_loss
+        self.long_weight = long_weight
+        self.short_weight = short_weight
+        self.hidden_act = hidden_act
+
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_attention_heads = num_attention_heads
+        self.num_hidden_layers = num_hidden_layers
+        self.num_key_value_heads = num_key_value_heads
+        self.max_position_embeddings = max_position_embeddings
+        self.num_audio_tokens = num_audio_tokens
+        self.num_text_tokens = num_text_tokens
+        self.num_mel_bins = num_mel_bins
+        self.num_vq = num_vq
+        self.use_llm_hidden_state = use_llm_hidden_state
+        self.audio_bos_token_id = audio_bos_token_id
+        self.text_eos_token_id = text_eos_token_id
+        self.use_text = use_text
+        self.streaming = streaming
+        self.streaming_text_chunk_min = streaming_text_chunk_min
+        self.streaming_text_chunk_max = streaming_text_chunk_max
+        self.streaming_text_reserved_len = streaming_text_reserved_len
+        self.streaming_audio_chunk_size = streaming_audio_chunk_size
+        self.attn_implementation = attn_implementation
+        self.condition_type = condition_type
+        self.backbone_model = backbone_model
+        self.audio_tokenizer_type = audio_tokenizer_type
+        self.audio_tokenizer_sample_rate = audio_tokenizer_sample_rate
+
+        self.streaming_sliding_window = streaming_sliding_window
+        self.streaming_sliding_window_max_text_len = streaming_sliding_window_max_text_len
+        self.streaming_sliding_window_average_speed = streaming_sliding_window_average_speed
+        self.streaming_sliding_window_fast_speed = streaming_sliding_window_fast_speed
+        self.streaming_sliding_window_slow_speed = streaming_sliding_window_slow_speed
+        self.streaming_sliding_window_audio_frame_rate = streaming_sliding_window_audio_frame_rate
+        self.streaming_sliding_window_audio_init_text_length = streaming_sliding_window_audio_init_text_length
+        self.streaming_sliding_window_audio_window_size = streaming_sliding_window_audio_window_size
+
+        self.normalize_projected_hidden = normalize_projected_hidden
+
+        self.interleaved = interleaved
+        self.attention_type = attention_type
+        self.recomputed_chunks = recomputed_chunks
+        self.window_size = window_size
+
+
+class MiniCPMOConfig(Qwen3Config):
+    model_type = "minicpmo"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    default_vision_config = {
+        "hidden_size": 1152,
+        "image_size": 980,
+        "intermediate_size": 4304,
+        "model_type": "siglip",
+        "num_attention_heads": 16,
+        "num_hidden_layers": 27,
+        "patch_size": 14,
+    }
+
+    def __init__(
+        self,
+        use_cache=True,
+        query_num=64,
+        image_size=448,
+        drop_vision_last_layer=True,
+        batch_vision_input=True,
+        slice_config=None,
+        vision_config=None,
+        audio_config=None,
+        tts_config=None,
+        use_image_id=True,
+        vision_batch_size=16,
+        audio_pool_step=5,
+        audio_chunk_length=1.0,
+        stream_input=False,
+        listen_speak_type="asr",
+        init_vision=True,
+        init_audio=True,
+        init_tts=True,
+        **kwargs,
+    ):
+        self.use_cache = use_cache
+        self.query_num = query_num
+        self.image_size = image_size
+        self.drop_vision_last_layer = drop_vision_last_layer
+        self.batch_vision_input = batch_vision_input
+        self.use_image_id = use_image_id
+        self.vision_batch_size = vision_batch_size
+        self.audio_pool_step = audio_pool_step
+        self.audio_chunk_length = audio_chunk_length
+        self.stream_input = stream_input
+        self.listen_speak_type = listen_speak_type
+
+        self.init_vision = init_vision
+        self.init_audio = init_audio
+        self.init_tts = init_tts
+
+        if slice_config is None:
+            self.slice_config = MiniCPMVSliceConfig(max_slice_nums=1)
+        else:
+            self.slice_config = MiniCPMVSliceConfig(**slice_config)
+        self.slice_mode = True
+
+        # same as HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit add tgt_sizes
+        if vision_config is None:
+            self.vision_config = SiglipVisionConfig(**self.default_vision_config)
+            logger.info("vision_config is None, using default vision config")
+        elif isinstance(vision_config, dict):
+            self.vision_config = SiglipVisionConfig(**vision_config)
+        elif isinstance(vision_config, SiglipVisionConfig):
+            self.vision_config = vision_config
+
+        if audio_config is None:
+            self.audio_config = WhisperConfig()
+        elif isinstance(audio_config, dict):
+            self.audio_config = WhisperConfig(**audio_config)
+        elif isinstance(audio_config, WhisperConfig):
+            self.audio_config = audio_config
+
+        if tts_config is None:
+            self.tts_config = MiniCPMTTSConfig()
+        elif isinstance(tts_config, dict):
+            self.tts_config = MiniCPMTTSConfig(**tts_config)
+        elif isinstance(tts_config, MiniCPMTTSConfig):
+            self.tts_config = tts_config
+
+        self.patch_size = self.vision_config.patch_size
+
+        super().__init__(**kwargs)

generation_config.json

@@ -0,0 +1,13 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "temperature": 0.6,
+  "top_k": 20,
+  "top_p": 0.95,
+  "transformers_version": "4.57.0"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:851d5dd243ff02bfe8bd97b3b8f43dc499c3f94b4bf1b91798f992697938b54a
+size 4900718032

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:537e2b3eac1f6e4192ac075e4a8ef721c091769144f5801e6d8f3b361b641fdd
+size 4915960936

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed4f427cfa50c84b87f57f5617f7b7d017619c8aae6be60b6599581158daf5b6
+size 4983069056

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17ef5669ee1163db56515c0d72f24c5c38e79c1ef96dd75a8886f7354ca2f1a6
+size 3248573944

model-tts.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:902e46ed220539517708cc95017e3ed594025d22df5815be236cb15ca587772f
+size 695415076

modeling_navit_siglip.py

@@ -0,0 +1,981 @@
+# coding=utf-8
+# Copyright 2024 Google AI and The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch Siglip model."""
+# Copied from  HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit and add tgt_sizes
+
+
+import math
+import os
+import warnings
+from dataclasses import dataclass
+from typing import Optional
+from typing import Tuple
+from typing import Union
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+import torch.utils.checkpoint
+from torch import nn
+from torch.nn.init import _calculate_fan_in_and_fan_out
+from transformers.activations import ACT2FN
+from transformers.configuration_utils import PretrainedConfig
+from transformers.modeling_attn_mask_utils import _prepare_4d_attention_mask
+from transformers.modeling_outputs import BaseModelOutput
+from transformers.modeling_outputs import BaseModelOutputWithPooling
+from transformers.modeling_utils import PreTrainedModel
+from transformers.utils import add_start_docstrings
+from transformers.utils import add_start_docstrings_to_model_forward
+from transformers.utils import is_flash_attn_2_available
+from transformers.utils import logging
+from transformers.utils import ModelOutput
+from transformers.utils import replace_return_docstrings
+
+logger = logging.get_logger(__name__)
+
+
+class SiglipVisionConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`SiglipVisionModel`]. It is used to instantiate a
+    Siglip vision encoder according to the specified arguments, defining the model architecture. Instantiating a
+    configuration with the defaults will yield a similar configuration to that of the vision encoder of the Siglip
+    [google/siglip-base-patch16-224](https://huggingface.co/google/siglip-base-patch16-224) architecture.
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+    Args:
+        hidden_size (`int`, *optional*, defaults to 768):
+            Dimensionality of the encoder layers and the pooler layer.
+        intermediate_size (`int`, *optional*, defaults to 3072):
+            Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+        num_hidden_layers (`int`, *optional*, defaults to 12):
+            Number of hidden layers in the Transformer encoder.
+        num_attention_heads (`int`, *optional*, defaults to 12):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        num_channels (`int`, *optional*, defaults to 3):
+            Number of channels in the input images.
+        image_size (`int`, *optional*, defaults to 224):
+            The size (resolution) of each image.
+        patch_size (`int`, *optional*, defaults to 16):
+            The size (resolution) of each patch.
+        hidden_act (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
+            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+            `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported.
+        layer_norm_eps (`float`, *optional*, defaults to 1e-06):
+            The epsilon used by the layer normalization layers.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio for the attention probabilities.
+    Example:
+    ```python
+    >>> from transformers import SiglipVisionConfig, SiglipVisionModel
+    >>> # Initializing a SiglipVisionConfig with google/siglip-base-patch16-224 style configuration
+    >>> configuration = SiglipVisionConfig()
+    >>> # Initializing a SiglipVisionModel (with random weights) from the google/siglip-base-patch16-224 style configuration
+    >>> model = SiglipVisionModel(configuration)
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "siglip_vision_model"
+
+    def __init__(
+        self,
+        hidden_size=768,
+        intermediate_size=3072,
+        num_hidden_layers=12,
+        num_attention_heads=12,
+        num_channels=3,
+        image_size=224,
+        patch_size=16,
+        hidden_act="gelu_pytorch_tanh",
+        layer_norm_eps=1e-6,
+        attention_dropout=0.0,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_channels = num_channels
+        self.patch_size = patch_size
+        self.image_size = image_size
+        self.attention_dropout = attention_dropout
+        self.layer_norm_eps = layer_norm_eps
+        self.hidden_act = hidden_act
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
+        cls._set_token_in_kwargs(kwargs)
+
+        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+
+        # get the vision config dict if we are loading from SiglipConfig
+        if config_dict.get("model_type") == "siglip":
+            config_dict = config_dict["vision_config"]
+
+        if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
+            logger.warning(
+                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
+                f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
+            )
+
+        return cls.from_dict(config_dict, **kwargs)
+
+
+_CHECKPOINT_FOR_DOC = "google/siglip-base-patch16-224"
+
+SIGLIP_PRETRAINED_MODEL_ARCHIVE_LIST = [
+    "google/siglip-base-patch16-224",
+    # See all SigLIP models at https://huggingface.co/models?filter=siglip
+]
+
+if is_flash_attn_2_available():
+    from flash_attn import flash_attn_func
+    from flash_attn import flash_attn_varlen_func
+    from flash_attn.bert_padding import index_first_axis  # noqa
+    from flash_attn.bert_padding import pad_input
+    from flash_attn.bert_padding import unpad_input
+
+
+# Copied from transformers.models.llama.modeling_llama._get_unpad_data
+def _get_unpad_data(attention_mask):
+    seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
+    indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
+    max_seqlen_in_batch = seqlens_in_batch.max().item()
+    cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0))
+    return (
+        indices,
+        cu_seqlens,
+        max_seqlen_in_batch,
+    )
+
+
+def _trunc_normal_(tensor, mean, std, a, b):
+    # Cut & paste from PyTorch official master until it's in a few official releases - RW
+    # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf
+    def norm_cdf(x):
+        # Computes standard normal cumulative distribution function
+        return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0
+
+    if (mean < a - 2 * std) or (mean > b + 2 * std):
+        warnings.warn(
+            "mean is more than 2 std from [a, b] in nn.init.trunc_normal_. "
+            "The distribution of values may be incorrect.",
+            stacklevel=2,
+        )
+
+    # Values are generated by using a truncated uniform distribution and
+    # then using the inverse CDF for the normal distribution.
+    # Get upper and lower cdf values
+    l = norm_cdf((a - mean) / std)
+    u = norm_cdf((b - mean) / std)
+
+    # Uniformly fill tensor with values from [l, u], then translate to
+    # [2l-1, 2u-1].
+    tensor.uniform_(2 * l - 1, 2 * u - 1)
+
+    # Use inverse cdf transform for normal distribution to get truncated
+    # standard normal
+    if tensor.dtype in [torch.float16, torch.bfloat16]:
+        # The `erfinv_` op is not (yet?) defined in float16+cpu, bfloat16+gpu
+        og_dtype = tensor.dtype
+        tensor = tensor.to(torch.float32)
+        tensor.erfinv_()
+        tensor = tensor.to(og_dtype)
+    else:
+        tensor.erfinv_()
+
+    # Transform to proper mean, std
+    tensor.mul_(std * math.sqrt(2.0))
+    tensor.add_(mean)
+
+    # Clamp to ensure it's in the proper range
+    if tensor.dtype == torch.float16:
+        # The `clamp_` op is not (yet?) defined in float16+cpu
+        tensor = tensor.to(torch.float32)
+        tensor.clamp_(min=a, max=b)
+        tensor = tensor.to(torch.float16)
+    else:
+        tensor.clamp_(min=a, max=b)
+
+
+def trunc_normal_tf_(
+    tensor: torch.Tensor,
+    mean: float = 0.0,
+    std: float = 1.0,
+    a: float = -2.0,
+    b: float = 2.0,
+) -> torch.Tensor:
+    """Fills the input Tensor with values drawn from a truncated
+    normal distribution. The values are effectively drawn from the
+    normal distribution :math:`\\mathcal{N}(\text{mean}, \text{std}^2)`
+    with values outside :math:`[a, b]` redrawn until they are within
+    the bounds. The method used for generating the random values works
+    best when :math:`a \\leq \text{mean} \\leq b`.
+    NOTE: this 'tf' variant behaves closer to Tensorflow / JAX impl where the
+    bounds [a, b] are applied when sampling the normal distribution with mean=0, std=1.0
+    and the result is subsquently scaled and shifted by the mean and std args.
+    Args:
+        tensor: an n-dimensional `torch.Tensor`
+        mean: the mean of the normal distribution
+        std: the standard deviation of the normal distribution
+        a: the minimum cutoff value
+        b: the maximum cutoff value
+    """
+    with torch.no_grad():
+        _trunc_normal_(tensor, 0, 1.0, a, b)
+        tensor.mul_(std).add_(mean)
+
+
+def variance_scaling_(tensor, scale=1.0, mode="fan_in", distribution="normal"):
+    fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
+    if mode == "fan_in":
+        denom = fan_in
+    elif mode == "fan_out":
+        denom = fan_out
+    elif mode == "fan_avg":
+        denom = (fan_in + fan_out) / 2
+
+    variance = scale / denom
+
+    if distribution == "truncated_normal":
+        # constant is stddev of standard normal truncated to (-2, 2)
+        trunc_normal_tf_(tensor, std=math.sqrt(variance) / 0.87962566103423978)
+    elif distribution == "normal":
+        with torch.no_grad():
+            tensor.normal_(std=math.sqrt(variance))
+    elif distribution == "uniform":
+        bound = math.sqrt(3 * variance)
+        with torch.no_grad():
+            tensor.uniform_(-bound, bound)
+    else:
+        raise ValueError(f"invalid distribution {distribution}")
+
+
+def lecun_normal_(tensor):
+    variance_scaling_(tensor, mode="fan_in", distribution="truncated_normal")
+
+
+def default_flax_embed_init(tensor):
+    variance_scaling_(tensor, mode="fan_in", distribution="normal")
+
+
+@dataclass
+# Copied from transformers.models.clip.modeling_clip.CLIPVisionModelOutput with CLIP->Siglip
+class SiglipVisionModelOutput(ModelOutput):
+    """
+    Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states.
+    Args:
+        image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`):
+            The image embeddings obtained by applying the projection layer to the pooler_output.
+        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
+            Sequence of hidden-states at the output of the last layer of the model.
+        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
+            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
+            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
+            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
+        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
+            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+            sequence_length)`.
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
+    """
+
+    image_embeds: Optional[torch.FloatTensor] = None
+    last_hidden_state: torch.FloatTensor = None
+    hidden_states: Optional[Tuple[torch.FloatTensor]] = None
+    attentions: Optional[Tuple[torch.FloatTensor]] = None
+
+
+class SiglipVisionEmbeddings(nn.Module):
+    def __init__(self, config: SiglipVisionConfig):
+        super().__init__()
+        self.config = config
+        self.embed_dim = config.hidden_size
+        self.image_size = config.image_size
+        self.patch_size = config.patch_size
+
+        self.patch_embedding = nn.Conv2d(
+            in_channels=config.num_channels,
+            out_channels=self.embed_dim,
+            kernel_size=self.patch_size,
+            stride=self.patch_size,
+            padding="valid",
+        )
+
+        self.num_patches_per_side = self.image_size // self.patch_size
+        self.num_patches = self.num_patches_per_side**2
+        self.num_positions = self.num_patches
+        self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim)
+
+    def forward(
+        self,
+        pixel_values: torch.FloatTensor,
+        patch_attention_mask: torch.BoolTensor,
+        tgt_sizes: Optional[torch.IntTensor] = None,
+    ) -> torch.Tensor:
+        batch_size = pixel_values.size(0)
+
+        patch_embeds = self.patch_embedding(pixel_values)
+        embeddings = patch_embeds.flatten(2).transpose(1, 2)
+
+        max_im_h, max_im_w = pixel_values.size(2), pixel_values.size(3)
+        max_nb_patches_h, max_nb_patches_w = (
+            max_im_h // self.patch_size,
+            max_im_w // self.patch_size,
+        )
+        boundaries = torch.arange(1 / self.num_patches_per_side, 1.0, 1 / self.num_patches_per_side)
+        position_ids = torch.full(
+            size=(
+                batch_size,
+                max_nb_patches_h * max_nb_patches_w,
+            ),
+            fill_value=0,
+        )
+
+        for batch_idx, p_attn_mask in enumerate(patch_attention_mask):
+            if tgt_sizes is not None:
+                nb_patches_h = tgt_sizes[batch_idx][0]
+                nb_patches_w = tgt_sizes[batch_idx][1]
+            else:
+                nb_patches_h = p_attn_mask[:, 0].sum()
+                nb_patches_w = p_attn_mask[0].sum()
+
+            fractional_coords_h = torch.arange(0, 1 - 1e-6, 1 / nb_patches_h)
+            fractional_coords_w = torch.arange(0, 1 - 1e-6, 1 / nb_patches_w)
+
+            bucket_coords_h = torch.bucketize(fractional_coords_h, boundaries, right=True)
+            bucket_coords_w = torch.bucketize(fractional_coords_w, boundaries, right=True)
+
+            pos_ids = (bucket_coords_h[:, None] * self.num_patches_per_side + bucket_coords_w).flatten()
+            position_ids[batch_idx][p_attn_mask.view(-1).cpu()] = pos_ids
+
+        position_ids = position_ids.to(self.position_embedding.weight.device)
+
+        embeddings = embeddings + self.position_embedding(position_ids)
+        return embeddings
+
+
+class SiglipAttention(nn.Module):
+    """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+    # Copied from transformers.models.clip.modeling_clip.CLIPAttention.__init__
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.embed_dim = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.head_dim = self.embed_dim // self.num_heads
+        if self.head_dim * self.num_heads != self.embed_dim:
+            raise ValueError(
+                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:"
+                f" {self.num_heads})."
+            )
+        self.scale = self.head_dim**-0.5
+        self.dropout = config.attention_dropout
+
+        self.k_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.v_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.q_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.out_proj = nn.Linear(self.embed_dim, self.embed_dim)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        output_attentions: Optional[bool] = False,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        """Input shape: Batch x Time x Channel"""
+
+        batch_size, q_len, _ = hidden_states.size()
+
+        query_states = self.q_proj(hidden_states)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+
+        query_states = query_states.view(batch_size, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(batch_size, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(batch_size, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+
+        k_v_seq_len = key_states.shape[-2]
+        attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) * self.scale
+
+        if attn_weights.size() != (batch_size, self.num_heads, q_len, k_v_seq_len):
+            raise ValueError(
+                f"Attention weights should be of size {(batch_size, self.num_heads, q_len, k_v_seq_len)}, but is"
+                f" {attn_weights.size()}"
+            )
+
+        if attention_mask is not None:
+            if attention_mask.size() != (batch_size, 1, q_len, k_v_seq_len):
+                raise ValueError(
+                    f"Attention mask should be of size {(batch_size, 1, q_len, k_v_seq_len)}, but is {attention_mask.size()}"
+                )
+            attn_weights = attn_weights + attention_mask
+
+        # upcast attention to fp32
+        attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
+        attn_weights = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training)
+        attn_output = torch.matmul(attn_weights, value_states)
+
+        if attn_output.size() != (batch_size, self.num_heads, q_len, self.head_dim):
+            raise ValueError(
+                f"`attn_output` should be of size {(batch_size, self.num_heads, q_len, self.head_dim)}, but is"
+                f" {attn_output.size()}"
+            )
+
+        attn_output = attn_output.transpose(1, 2).contiguous()
+        attn_output = attn_output.reshape(batch_size, q_len, self.embed_dim)
+
+        attn_output = self.out_proj(attn_output)
+
+        return attn_output, attn_weights
+
+
+class SiglipFlashAttention2(SiglipAttention):
+    """
+    Llama flash attention module. This module inherits from `LlamaAttention` as the weights of the module stays
+    untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
+    flash attention and deal with padding tokens in case the input contains any of them.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.is_causal = False  # Hack to make sure we don't use a causal mask
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.LongTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Tuple[torch.Tensor]] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+        **kwargs,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        output_attentions = False
+
+        bsz, q_len, _ = hidden_states.size()
+
+        query_states = self.q_proj(hidden_states)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+
+        # Flash attention requires the input to have the shape
+        # batch_size x seq_length x head_dim x hidden_dim
+        # therefore we just need to keep the original shape
+        query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+
+        kv_seq_len = key_states.shape[-2]
+        if past_key_value is not None:
+            kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
+        # cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+        # query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
+
+        # if past_key_value is not None:
+        #     cache_kwargs = {"sin": sin, "cos": cos}  # Specific to RoPE models
+        #     key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
+
+        # TODO: These transpose are quite inefficient but Flash Attention requires the layout [batch_size, sequence_length, num_heads, head_dim]. We would need to refactor the KV cache
+        # to be able to avoid many of these transpose/reshape/view.
+        query_states = query_states.transpose(1, 2)
+        key_states = key_states.transpose(1, 2)
+        value_states = value_states.transpose(1, 2)
+
+        dropout_rate = self.dropout if self.training else 0.0
+
+        # In PEFT, usually we cast the layer norms in float32 for training stability reasons
+        # therefore the input hidden states gets silently casted in float32. Hence, we need
+        # cast them back in the correct dtype just to be sure everything works as expected.
+        # This might slowdown training & inference so it is recommended to not cast the LayerNorms
+        # in fp32. (LlamaRMSNorm handles it correctly)
+
+        input_dtype = query_states.dtype
+        if input_dtype == torch.float32:
+            if torch.is_autocast_enabled():
+                target_dtype = torch.get_autocast_gpu_dtype()
+            # Handle the case where the model is quantized
+            elif hasattr(self.config, "_pre_quantization_dtype"):
+                target_dtype = self.config._pre_quantization_dtype
+            else:
+                target_dtype = self.q_proj.weight.dtype
+
+            logger.warning_once(
+                "The input hidden states seems to be silently casted in float32, this might be related to the fact"
+                " you have upcasted embedding or layer norm layers in float32. We will cast back the input in"
+                f" {target_dtype}."
+            )
+
+            query_states = query_states.to(target_dtype)
+            key_states = key_states.to(target_dtype)
+            value_states = value_states.to(target_dtype)
+
+        attn_output = self._flash_attention_forward(
+            query_states,
+            key_states,
+            value_states,
+            attention_mask,
+            q_len,
+            dropout=dropout_rate,
+        )
+
+        attn_output = attn_output.reshape(bsz, q_len, self.embed_dim).contiguous()
+        attn_output = self.out_proj(attn_output)
+
+        if not output_attentions:
+            attn_weights = None
+
+        return attn_output, attn_weights
+
+    def _flash_attention_forward(
+        self,
+        query_states,
+        key_states,
+        value_states,
+        attention_mask,
+        query_length,
+        dropout=0.0,
+        softmax_scale=None,
+    ):
+        """
+        Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token
+        first unpad the input, then computes the attention scores and pad the final attention scores.
+        Args:
+            query_states (`torch.Tensor`):
+                Input query states to be passed to Flash Attention API
+            key_states (`torch.Tensor`):
+                Input key states to be passed to Flash Attention API
+            value_states (`torch.Tensor`):
+                Input value states to be passed to Flash Attention API
+            attention_mask (`torch.Tensor`):
+                The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the
+                position of padding tokens and 1 for the position of non-padding tokens.
+            dropout (`int`, *optional*):
+                Attention dropout
+            softmax_scale (`float`, *optional*):
+                The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim)
+        """
+
+        # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__.
+        causal = self.is_causal and query_length != 1
+
+        # Contains at least one padding token in the sequence
+        if attention_mask is not None:
+            batch_size = query_states.shape[0]
+            (
+                query_states,
+                key_states,
+                value_states,
+                indices_q,
+                cu_seq_lens,
+                max_seq_lens,
+            ) = self._upad_input(query_states, key_states, value_states, attention_mask, query_length)
+
+            cu_seqlens_q, cu_seqlens_k = cu_seq_lens
+            max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
+
+            attn_output_unpad = flash_attn_varlen_func(
+                query_states,
+                key_states,
+                value_states,
+                cu_seqlens_q=cu_seqlens_q,
+                cu_seqlens_k=cu_seqlens_k,
+                max_seqlen_q=max_seqlen_in_batch_q,
+                max_seqlen_k=max_seqlen_in_batch_k,
+                dropout_p=dropout,
+                softmax_scale=softmax_scale,
+                causal=causal,
+            )
+
+            attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length)
+        else:
+            attn_output = flash_attn_func(
+                query_states,
+                key_states,
+                value_states,
+                dropout,
+                softmax_scale=softmax_scale,
+                causal=causal,
+            )
+
+        return attn_output
+
+    def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length):
+        indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask)
+        batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape
+
+        key_layer = index_first_axis(
+            key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim),
+            indices_k,
+        )
+        value_layer = index_first_axis(
+            value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim),
+            indices_k,
+        )
+        if query_length == kv_seq_len:
+            query_layer = index_first_axis(
+                query_layer.reshape(batch_size * kv_seq_len, self.num_heads, head_dim),
+                indices_k,
+            )
+            cu_seqlens_q = cu_seqlens_k
+            max_seqlen_in_batch_q = max_seqlen_in_batch_k
+            indices_q = indices_k
+        elif query_length == 1:
+            max_seqlen_in_batch_q = 1
+            cu_seqlens_q = torch.arange(
+                batch_size + 1, dtype=torch.int32, device=query_layer.device
+            )  # There is a memcpy here, that is very bad.
+            indices_q = cu_seqlens_q[:-1]
+            query_layer = query_layer.squeeze(1)
+        else:
+            # The -q_len: slice assumes left padding.
+            attention_mask = attention_mask[:, -query_length:]
+            query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask)
+
+        return (
+            query_layer,
+            key_layer,
+            value_layer,
+            indices_q,
+            (cu_seqlens_q, cu_seqlens_k),
+            (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
+        )
+
+
+# Copied from transformers.models.clip.modeling_clip.CLIPMLP with CLIP->Siglip
+class SiglipMLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.activation_fn = ACT2FN[config.hidden_act]
+        self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
+        self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_states = self.fc1(hidden_states)
+        hidden_states = self.activation_fn(hidden_states)
+        hidden_states = self.fc2(hidden_states)
+        return hidden_states
+
+
+# Copied from transformers.models.clip.modeling_clip.CLIPEncoderLayer with CLIP->Siglip
+class SiglipEncoderLayer(nn.Module):
+    def __init__(self, config: SiglipVisionConfig):
+        super().__init__()
+        self.embed_dim = config.hidden_size
+        self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2"
+        self.self_attn = SiglipAttention(config) if not self._use_flash_attention_2 else SiglipFlashAttention2(config)
+        self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+        self.mlp = SiglipMLP(config)
+        self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: torch.Tensor,
+        output_attentions: Optional[bool] = False,
+    ) -> Tuple[torch.FloatTensor]:
+        """
+        Args:
+            hidden_states (`torch.FloatTensor`):
+                Input to the layer of shape `(batch, seq_len, embed_dim)`.
+            attention_mask (`torch.FloatTensor`):
+                Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values.
+            output_attentions (`bool`, *optional*, defaults to `False`):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+        """
+        residual = hidden_states
+
+        hidden_states = self.layer_norm1(hidden_states)
+        hidden_states, attn_weights = self.self_attn(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            output_attentions=output_attentions,
+        )
+        hidden_states = residual + hidden_states
+
+        residual = hidden_states
+        hidden_states = self.layer_norm2(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + hidden_states
+
+        outputs = (hidden_states,)
+
+        if output_attentions:
+            outputs += (attn_weights,)
+
+        return outputs
+
+
+class SiglipPreTrainedModel(PreTrainedModel):
+    """
+    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
+    models.
+    """
+
+    config_class = SiglipVisionConfig
+    base_model_prefix = "siglip"
+    supports_gradient_checkpointing = True
+
+    def _init_weights(self, module):
+        """Initialize the weights"""
+
+        if isinstance(module, SiglipVisionEmbeddings):
+            width = self.config.hidden_size
+            nn.init.normal_(module.position_embedding.weight, std=1 / np.sqrt(width))
+        elif isinstance(module, nn.Embedding):
+            default_flax_embed_init(module.weight)
+        elif isinstance(module, SiglipAttention):
+            nn.init.normal_(module.q_proj.weight)
+            nn.init.normal_(module.k_proj.weight)
+            nn.init.normal_(module.v_proj.weight)
+            nn.init.normal_(module.out_proj.weight)
+            nn.init.zeros_(module.q_proj.bias)
+            nn.init.zeros_(module.k_proj.bias)
+            nn.init.zeros_(module.v_proj.bias)
+            nn.init.zeros_(module.out_proj.bias)
+        elif isinstance(module, SiglipMLP):
+            nn.init.normal_(module.fc1.weight)
+            nn.init.normal_(module.fc2.weight)
+            nn.init.normal_(module.fc1.bias, std=1e-6)
+            nn.init.normal_(module.fc2.bias, std=1e-6)
+        elif isinstance(module, (nn.Linear, nn.Conv2d)):
+            lecun_normal_(module.weight)
+            if module.bias is not None:
+                nn.init.zeros_(module.bias)
+        elif isinstance(module, nn.LayerNorm):
+            module.bias.data.zero_()
+            module.weight.data.fill_(1.0)
+
+
+SIGLIP_START_DOCSTRING = r"""
+    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+    etc.)
+    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+    and behavior.
+    Parameters:
+        config ([`SiglipVisionConfig`]): Model configuration class with all the parameters of the model.
+            Initializing with a config file does not load the weights associated with the model, only the
+            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+
+
+SIGLIP_VISION_INPUTS_DOCSTRING = r"""
+    Args:
+        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
+            Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
+            [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.
+        output_attentions (`bool`, *optional*):
+            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+            tensors for more detail.
+        output_hidden_states (`bool`, *optional*):
+            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+            more detail.
+        return_dict (`bool`, *optional*):
+            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+
+# Copied from transformers.models.clip.modeling_clip.CLIPEncoder with CLIP->Siglip
+class SiglipEncoder(nn.Module):
+    """
+    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
+    [`SiglipEncoderLayer`].
+    Args:
+        config: SiglipConfig
+    """
+
+    def __init__(self, config: SiglipVisionConfig):
+        super().__init__()
+        self.config = config
+        self.layers = nn.ModuleList([SiglipEncoderLayer(config) for _ in range(config.num_hidden_layers)])
+        self.gradient_checkpointing = False
+
+    # Ignore copy
+    def forward(
+        self,
+        inputs_embeds,
+        attention_mask: Optional[torch.Tensor] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, BaseModelOutput]:
+        r"""
+        Args:
+            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
+                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+                than the model's internal embedding lookup matrix.
+            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+                - 1 for tokens that are **not masked**,
+                - 0 for tokens that are **masked**.
+                [What are attention masks?](../glossary#attention-mask)
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+            output_hidden_states (`bool`, *optional*):
+                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
+                for more detail.
+            return_dict (`bool`, *optional*):
+                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+        """
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        encoder_states = () if output_hidden_states else None
+        all_attentions = () if output_attentions else None
+
+        hidden_states = inputs_embeds
+        for encoder_layer in self.layers:
+            if output_hidden_states:
+                encoder_states = encoder_states + (hidden_states,)
+            if self.gradient_checkpointing and self.training:
+                layer_outputs = self._gradient_checkpointing_func(
+                    encoder_layer.__call__,
+                    hidden_states,
+                    attention_mask,
+                    output_attentions,
+                )
+            else:
+                layer_outputs = encoder_layer(
+                    hidden_states,
+                    attention_mask,
+                    output_attentions=output_attentions,
+                )
+
+            hidden_states = layer_outputs[0]
+
+            if output_attentions:
+                all_attentions = all_attentions + (layer_outputs[1],)
+
+        if output_hidden_states:
+            encoder_states = encoder_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
+        return BaseModelOutput(
+            last_hidden_state=hidden_states,
+            hidden_states=encoder_states,
+            attentions=all_attentions,
+        )
+
+
+@add_start_docstrings(
+    """The vision model from SigLIP without any head or projection on top.""",
+    SIGLIP_START_DOCSTRING,
+)
+class SiglipVisionTransformer(SiglipPreTrainedModel):
+    config_class = SiglipVisionConfig
+    main_input_name = "pixel_values"
+    _supports_flash_attn_2 = True
+    _no_split_modules = []
+
+    def __init__(self, config: SiglipVisionConfig):
+        super().__init__(config)
+        self.config = config
+        embed_dim = config.hidden_size
+
+        self.embeddings = SiglipVisionEmbeddings(config)
+        self.encoder = SiglipEncoder(config)
+        self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
+        self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2"
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self) -> nn.Module:
+        return self.embeddings.patch_embedding
+
+    @add_start_docstrings_to_model_forward(SIGLIP_VISION_INPUTS_DOCSTRING)
+    @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=SiglipVisionConfig)
+    def forward(
+        self,
+        pixel_values,
+        patch_attention_mask: Optional[torch.BoolTensor] = None,
+        tgt_sizes: Optional[torch.IntTensor] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, BaseModelOutputWithPooling]:
+        r"""
+        Returns:
+        """
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        batch_size = pixel_values.size(0)
+        if patch_attention_mask is None:
+            patch_attention_mask = torch.ones(
+                size=(
+                    batch_size,
+                    pixel_values.size(2) // self.config.patch_size,
+                    pixel_values.size(3) // self.config.patch_size,
+                ),
+                dtype=torch.bool,
+                device=pixel_values.device,
+            )
+
+        hidden_states = self.embeddings(
+            pixel_values=pixel_values,
+            patch_attention_mask=patch_attention_mask,
+            tgt_sizes=tgt_sizes,
+        )
+
+        patch_attention_mask = patch_attention_mask.view(batch_size, -1)
+        # The call to `_upad_input` in `_flash_attention_forward` is expensive
+        # So when the `patch_attention_mask` is full of 1s (i.e. attending to the whole sequence),
+        # avoiding passing the attention_mask, which is equivalent to attending to the full sequence
+        if not torch.any(~patch_attention_mask):
+            attention_mask = None
+        else:
+            attention_mask = (
+                _prepare_4d_attention_mask(patch_attention_mask, hidden_states.dtype)
+                if not self._use_flash_attention_2
+                else patch_attention_mask
+            )
+
+        encoder_outputs = self.encoder(
+            inputs_embeds=hidden_states,
+            attention_mask=attention_mask,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        last_hidden_state = encoder_outputs[0]
+        last_hidden_state = self.post_layernorm(last_hidden_state)
+
+        if not return_dict:
+            return (last_hidden_state, None) + encoder_outputs[1:]
+
+        return BaseModelOutputWithPooling(
+            last_hidden_state=last_hidden_state,
+            pooler_output=None,
+            hidden_states=encoder_outputs.hidden_states,
+            attentions=encoder_outputs.attentions,
+        )

preprocessor_config.json

@@ -0,0 +1,49 @@
+{
+  "audio_pool_step": 5,
+  "auto_map": {
+    "AutoFeatureExtractor": "processing_minicpmo.MiniCPMAAudioProcessor",
+    "AutoImageProcessor": "processing_minicpmo.MiniCPMVImageProcessor",
+    "AutoProcessor": "processing_minicpmo.MiniCPMOProcessor"
+  },
+  "chunk_length": 30,
+  "dither": 0.0,
+  "dynamic_log_norm": true,
+  "dynamic_range_db": 8.0,
+  "feature_extractor_type": "MiniCPMAAudioProcessor",
+  "feature_size": 80,
+  "hop_length": 160,
+  "im_end": "</image>",
+  "im_id_end": "</image_id>",
+  "im_id_start": "<image_id>",
+  "im_start": "<image>",
+  "image_feature_size": 64,
+  "image_processor_type": "MiniCPMVImageProcessor",
+  "log_floor_db": -10.0,
+  "max_slice_nums": 9,
+  "n_fft": 400,
+  "n_samples": 480000,
+  "nb_max_frames": 3000,
+  "norm_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "norm_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "patch_size": 14,
+  "processor_class": "MiniCPMOProcessor",
+  "return_attention_mask": false,
+  "sampling_rate": 16000,
+  "scale_resolution": 448,
+  "slice_end": "</slice>",
+  "slice_mode": true,
+  "slice_start": "<slice>",
+  "unk": "<unk>",
+  "use_image_id": true,
+  "version": 4.5
+}