Add checkpoint files via LFS

.gitattributes

@@ -0,0 +1,13 @@
+checkpoint/vad/silero_vad.onnx filter=lfs diff=lfs merge=lfs -text
+checkpoint/vad/silero_vad_v4.onnx filter=lfs diff=lfs merge=lfs -text
+checkpoint/omni-o/omni-o-hf/model.safetensors filter=lfs diff=lfs merge=lfs -text
+checkpoint/omni-o/omni-o-moe-hf/model.safetensors filter=lfs diff=lfs merge=lfs -text
+checkpoint/omni-o/omni-o.pth filter=lfs diff=lfs merge=lfs -text
+checkpoint/omni-o/omni-o_moe.pth filter=lfs diff=lfs merge=lfs -text
+checkpoint/siglip/model.safetensors filter=lfs diff=lfs merge=lfs -text
+checkpoint/sensevoice/chn_jpn_yue_eng_ko_spectok.bpe.model filter=lfs diff=lfs merge=lfs -text
+checkpoint/sensevoice/model.pt filter=lfs diff=lfs merge=lfs -text
+checkpoint/mimi/model.safetensors filter=lfs diff=lfs merge=lfs -text
+checkpoint/campplus/campplus_cn_common.pt filter=lfs diff=lfs merge=lfs -text
+checkpoint/omni/native_hf/model.safetensors filter=lfs diff=lfs merge=lfs -text
+checkpoint/omni/omni.pth filter=lfs diff=lfs merge=lfs -text

checkpoint/campplus/.mv

@@ -0,0 +1 @@
+Revision:master,CreatedAt:1777630305

checkpoint/campplus/README.md

@@ -0,0 +1,25 @@
+---
+license: apache-2.0
+language:
+  - zh
+tags:
+  - speaker-verification
+  - speaker-recognition
+  - speaker-embedding
+  - speech
+  - audio
+  - campplus
+pipeline_tag: audio-classification
+---
+
+# CAMPPlus Speaker Encoder
+
+This directory contains a local copy of the CAMPPlus speaker verification model, originally released through ModelScope / 3D-Speaker. The model is used to extract speaker embeddings from speech, while the original model architecture, weights, authorship, and license remain attributed to the original authors.
+
+The upstream 3D-Speaker project is released under the `Apache-2.0` license. When using, redistributing, or citing these files, please keep the original author information, model link, and license notice.
+
+Original links:
+
+- ModelScope: https://www.modelscope.cn/models/iic/speech_campplus_sv_zh-cn_16k-common
+- 3D-Speaker: https://github.com/modelscope/3D-Speaker
+- License: https://www.apache.org/licenses/LICENSE-2.0

checkpoint/campplus/campplus_cn_common.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55ffb1a55d04bac4a9b7ed80497cb731909985c9b55d120fce04b851a697886c
+size 14173135

checkpoint/campplus/config.yaml

@@ -0,0 +1,23 @@
+# This is an example that demonstrates how to configure a model file.
+# You can modify the configuration according to your own requirements.
+
+# to print the register_table:
+# from funasr.register import tables
+# tables.print()
+
+# network architecture
+model: CAMPPlus
+model_conf:
+    feat_dim: 80
+    embedding_size: 192
+    growth_rate: 32
+    bn_size: 4
+    init_channels: 128
+    config_str: 'batchnorm-relu'
+    memory_efficient: True
+    output_level: 'segment'
+
+# frontend related
+frontend: WavFrontend
+frontend_conf:
+    fs: 16000

checkpoint/campplus/configuration.json

@@ -0,0 +1,23 @@
+{
+    "framework": "pytorch",
+    "task": "speaker-verification",
+    "model_config": "config.yaml",
+    "model_file": "campplus_cn_common.bin",
+    "model": {
+        "type": "cam++-sv",
+        "model_config": {
+            "sample_rate": 16000,
+            "fbank_dim": 80,
+            "emb_size": 192
+        },
+        "pretrained_model": "campplus_cn_common.bin",
+        "yesOrno_thr": 0.31
+    },
+    "pipeline": {
+        "type": "speaker-verification"
+    },
+    "file_path_metas": {
+        "init_param":"campplus_cn_common.bin", 
+        "config":"config.yaml"
+    }
+}

checkpoint/mimi/.mv

@@ -0,0 +1 @@
+Revision:master,CreatedAt:1777630495

checkpoint/mimi/README.md

@@ -0,0 +1,23 @@
+---
+license: cc-by-4.0
+language:
+  - en
+tags:
+  - audio
+  - codec
+  - feature-extraction
+  - mimi
+  - fp16
+pipeline_tag: feature-extraction
+---
+
+# Mimi FP16
+
+This directory contains an FP16 converted version of [Kyutai Mimi](https://huggingface.co/kyutai/mimi), used as an audio codec for encoding and decoding. The original model was released by Kyutai, and the model architecture, weights, authorship, and license remain attributed to the original authors.
+
+The original Mimi model is released under the `CC-BY-4.0` license. The files here are only converted to FP16 to reduce memory usage and make local usage more convenient. When using, redistributing, or citing these files, please keep the original Kyutai attribution, model link, and license notice.
+
+Original links:
+
+- Hugging Face: https://huggingface.co/kyutai/mimi
+- License: https://creativecommons.org/licenses/by/4.0/

checkpoint/mimi/config.json

@@ -0,0 +1,49 @@
+{
+  "architectures": [
+    "MimiModel"
+  ],
+  "attention_dropout": 0.0,
+  "audio_channels": 1,
+  "codebook_dim": 256,
+  "codebook_size": 2048,
+  "compress": 2,
+  "dilation_growth_rate": 2,
+  "frame_rate": 12.5,
+  "head_dim": 64,
+  "hidden_act": "gelu",
+  "hidden_size": 512,
+  "initializer_range": 0.02,
+  "intermediate_size": 2048,
+  "kernel_size": 7,
+  "last_kernel_size": 3,
+  "layer_scale_initial_scale": 0.01,
+  "max_position_embeddings": 8000,
+  "model_type": "mimi",
+  "norm_eps": 1e-05,
+  "normalize": false,
+  "num_attention_heads": 8,
+  "num_filters": 64,
+  "num_hidden_layers": 8,
+  "num_key_value_heads": 8,
+  "num_residual_layers": 1,
+  "num_semantic_quantizers": 1,
+  "pad_mode": "constant",
+  "residual_kernel_size": 3,
+  "rope_theta": 10000.0,
+  "sampling_rate": 24000,
+  "sliding_window": 250,
+  "torch_dtype": "float16",
+  "transformers_version": "4.45.0.dev0",
+  "trim_right_ratio": 1.0,
+  "upsample_groups": 512,
+  "upsampling_ratios": [
+    8,
+    6,
+    5,
+    4
+  ],
+  "use_cache": false,
+  "use_causal_conv": true,
+  "use_conv_shortcut": false,
+  "vector_quantization_hidden_dimension": 256
+}

checkpoint/mimi/configuration.json

@@ -0,0 +1 @@
+{"framework":"Pytorch","task":"text-to-speech"}

checkpoint/mimi/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7542ee039d3025d5089cf227d21df64b6b8eff08fcd376a11a1fbd178dd9d3f5
+size 192346842

checkpoint/mimi/preprocessor_config.json

@@ -0,0 +1,10 @@
+{
+  "chunk_length_s": null,
+  "feature_extractor_type": "EncodecFeatureExtractor",
+  "feature_size": 1,
+  "overlap": null,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "return_attention_mask": true,
+  "sampling_rate": 24000
+}

checkpoint/omni-o/.mv

@@ -0,0 +1 @@
+Revision:master,CreatedAt:1778047169

checkpoint/omni-o/omni-o-hf/chat_template.jinja

@@ -0,0 +1,85 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if true %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if open_thinking is defined and open_thinking is true %}
+        {{- '<think>\n' }}
+    {%- else %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

checkpoint/omni-o/omni-o-hf/config.json

@@ -0,0 +1,60 @@
+{
+  "architectures": [
+    "VAM"
+  ],
+  "audio_hidden_size": 512,
+  "audio_ids": [
+    16
+  ],
+  "audio_pad_token": 2049,
+  "audio_special_token": "<|audio_pad|>",
+  "audio_spk_token": 2051,
+  "audio_stop_token": 2050,
+  "audio_vocab_size": 2112,
+  "auto_map": {
+    "AutoConfig": "modeling_omni_o.VAMConfig",
+    "AutoModelForCausalLM": "modeling_omni_o.VAM"
+  },
+  "bos_token_id": 1,
+  "bridge_layer": 3,
+  "dropout": 0.0,
+  "dtype": "float16",
+  "eos_token_id": 2,
+  "flash_attn": true,
+  "head_dim": 96,
+  "hidden_act": "silu",
+  "hidden_size": 768,
+  "image_hidden_size": 768,
+  "image_ids": [
+    12
+  ],
+  "image_special_token": "<|image_pad|>",
+  "image_token_len": 64,
+  "inference_rope_scaling": false,
+  "intermediate_size": 2432,
+  "max_position_embeddings": 32768,
+  "model_type": "omni-o",
+  "moe_intermediate_size": 2432,
+  "norm_topk_prob": true,
+  "num_attention_heads": 8,
+  "num_experts": 4,
+  "num_experts_per_tok": 1,
+  "num_hidden_layers": 8,
+  "num_key_value_heads": 4,
+  "num_talker_hidden_layers": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": null,
+  "rope_theta": 1000000.0,
+  "router_aux_loss_coef": 0.0005,
+  "spk_emb_size": 192,
+  "talker_hidden_size": 768,
+  "think_end_ids": [
+    26,
+    234,
+    234
+  ],
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.1",
+  "use_moe": false,
+  "vocab_size": 6400
+}

checkpoint/omni-o/omni-o-hf/config.py

@@ -0,0 +1,24 @@
+from models.lm.config import LMConfig
+
+
+class VAMConfig(LMConfig):
+    model_type = "omni-o"
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.num_talker_hidden_layers = kwargs.get("num_talker_hidden_layers", 4)
+        self.talker_hidden_size = kwargs.get("talker_hidden_size", 768)
+        self.audio_ids = kwargs.get("audio_ids", [16])
+        self.audio_special_token = kwargs.get("audio_special_token", "<|audio_pad|>")
+        self.audio_hidden_size = kwargs.get("audio_hidden_size", 512)
+        self.audio_vocab_size = kwargs.get("audio_vocab_size", 2112)
+        self.audio_pad_token = kwargs.get("audio_pad_token", 2049)
+        self.audio_stop_token = kwargs.get("audio_stop_token", 2050)
+        self.audio_spk_token = kwargs.get("audio_spk_token", 2051)
+        self.spk_emb_size = kwargs.get("spk_emb_size", 192)
+        self.think_end_ids = kwargs.get("think_end_ids", [26, 234, 234])
+        self.image_ids = kwargs.get("image_ids", [12])
+        self.image_special_token = kwargs.get("image_special_token", "<|image_pad|>")
+        self.image_hidden_size = kwargs.get("image_hidden_size", 768)
+        self.image_token_len = kwargs.get("image_token_len", 64)
+        self.bridge_layer = kwargs.get("bridge_layer", self.num_hidden_layers // 2 - 1)

checkpoint/omni-o/omni-o-hf/generation_config.json

@@ -0,0 +1,8 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.14.1"
+}

checkpoint/omni-o/omni-o-hf/model.py

@@ -0,0 +1,415 @@
+import os
+import math
+import torch
+import warnings
+import logging
+import contextlib
+import io
+from torch import nn
+from torch.nn import functional as F
+from transformers.modeling_outputs import MoeCausalLMOutputWithPast
+from transformers import SiglipVisionModel, SiglipImageProcessor, logging as hf_logging
+
+from core import RMSNorm, precompute_freqs_cis, Block, MOEFeedForward
+from models.lm.config import LMConfig
+from models.lm.model import LMForCausalLM
+from models.vam.config import VAMConfig
+from encoders.audio import SenseVoiceAudioEncoder, SenseVoiceAudioProcessor
+from encoders.vision import SiglipVisionEncoder
+from projectors import MMVisionProjector, MMAudioProjector
+
+
+class TalkerHead(nn.Module):
+    def __init__(self, in_features, out_features, num_layers=8, rank=256):
+        super().__init__()
+        self.num_layers = num_layers
+        self.base = nn.Linear(in_features, out_features, bias=False)
+        self.adapters = nn.ModuleList([
+            nn.Sequential(nn.Linear(in_features, rank, bias=False), nn.GELU(), nn.Linear(rank, out_features, bias=False))
+            for _ in range(num_layers)
+        ])
+
+    def forward(self, x):
+        base_out = self.base(x)
+        return [base_out + adapter(x) for adapter in self.adapters]
+
+
+class TalkerEmbedding(nn.Module):
+    def __init__(self, num_embeddings, embedding_dim, num_layers=8, rank=256):
+        super().__init__()
+        self.num_layers = num_layers
+        self.base = nn.Embedding(num_embeddings, embedding_dim)
+        self.adapters = nn.ModuleList([
+            nn.Sequential(nn.Embedding(num_embeddings, rank), nn.GELU(), nn.Linear(rank, embedding_dim, bias=False))
+            for _ in range(num_layers)
+        ])
+
+    def forward(self, x):
+        base_out = self.base(x)
+        return sum(base_out[:, i, :] + self.adapters[i](x[:, i, :]) for i in range(len(self.adapters))) / self.num_layers
+
+
+class TalkerModule(nn.Module):
+    def __init__(self, config: VAMConfig):
+        super().__init__()
+        self.talker_config = LMConfig(hidden_size=config.talker_hidden_size, use_moe=config.use_moe)
+        self.layers = nn.ModuleList([Block(l, self.talker_config) for l in range(config.num_talker_hidden_layers)])
+        self.norm = RMSNorm(config.talker_hidden_size, eps=config.rms_norm_eps)
+        self.lm_head = TalkerHead(config.talker_hidden_size, config.audio_vocab_size)
+        self.embed_tokens = TalkerEmbedding(config.audio_vocab_size, config.talker_hidden_size)
+        self.codec_proj = nn.Sequential(
+            nn.Linear(config.talker_hidden_size, config.talker_hidden_size),
+            nn.GELU(),
+            nn.Linear(config.talker_hidden_size, config.talker_hidden_size),
+            RMSNorm(config.talker_hidden_size, eps=config.rms_norm_eps),
+        )
+        self.embed_proj = nn.Sequential(
+            nn.Linear(config.hidden_size, config.hidden_size),
+            nn.GELU(),
+            nn.Linear(config.hidden_size, config.talker_hidden_size),
+            RMSNorm(config.talker_hidden_size, eps=config.rms_norm_eps),
+        )
+        self.text_scale, self.audio_scale = nn.Parameter(torch.tensor(3.0)), nn.Parameter(torch.tensor(1.0))
+        self.spk_proj = nn.Linear(config.spk_emb_size, config.talker_hidden_size, bias=False)
+        freqs_cos, freqs_sin = precompute_freqs_cis(
+            dim=self.talker_config.head_dim, end=config.max_position_embeddings,
+            rope_base=config.rope_theta, rope_scaling=config.rope_scaling
+        )
+        self.register_buffer("freqs_cos", freqs_cos, persistent=False)
+        self.register_buffer("freqs_sin", freqs_sin, persistent=False)
+
+
+class VAM(LMForCausalLM):
+    config_class = VAMConfig
+
+    def __init__(self, config: VAMConfig = None, audio_encoder_path: str = None, vision_model_path: str = None):
+        config = config or VAMConfig()
+        super().__init__(config)
+        object.__setattr__(self, 'thinker', self.model)
+        object.__setattr__(self.model, 'lm_head', self.lm_head)
+        self.talker = TalkerModule(config)
+        self.audio_proj = MMAudioProjector(config.audio_hidden_size, config.hidden_size)
+        self.vision_proj = MMVisionProjector(config.image_hidden_size, config.hidden_size, target_tokens=config.image_token_len)
+        self.audio_pad_token, self.audio_stop_token, self.audio_spk_token = config.audio_pad_token, config.audio_stop_token, config.audio_spk_token
+        meta_init = any(p.device.type == 'meta' for p in self.parameters())
+        if meta_init:
+            object.__setattr__(self, 'audio_encoder', None)
+            object.__setattr__(self, 'audio_processor', None)
+            object.__setattr__(self, 'vision_encoder', None)
+            object.__setattr__(self, 'vision_processor', None)
+        else:
+            audio_enc = SenseVoiceAudioEncoder(audio_encoder_path) if audio_encoder_path else SenseVoiceAudioEncoder()
+            object.__setattr__(self, 'audio_encoder', audio_enc)
+            object.__setattr__(self, 'audio_processor', audio_enc.processor)
+            vision_enc = SiglipVisionEncoder(vision_model_path) if vision_model_path else SiglipVisionEncoder()
+            object.__setattr__(self, 'vision_encoder', vision_enc)
+            object.__setattr__(self, 'vision_processor', vision_enc.processor)
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
+        audio_encoder_path = kwargs.pop('audio_encoder_path', None)
+        vision_model_path = kwargs.pop('vision_model_path', None)
+        model = super().from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+        if audio_encoder_path and model.audio_encoder is None:
+            enc, proc = cls.load_sensevoice(audio_encoder_path)
+            object.__setattr__(model, 'audio_encoder', enc)
+            object.__setattr__(model, 'audio_processor', proc)
+        if vision_model_path and model.vision_encoder is None:
+            enc, proc = cls.load_vision(vision_model_path)
+            object.__setattr__(model, 'vision_encoder', enc)
+            object.__setattr__(model, 'vision_processor', proc)
+        return model
+
+    @staticmethod
+    def load_sensevoice(path):
+        if not os.path.exists(path):
+            warnings.warn(f"[VAM] SenseVoice path not found: {path}")
+            return None, None
+        logging.getLogger().setLevel(logging.ERROR)
+        hf_logging.set_verbosity_error()
+        with contextlib.redirect_stdout(io.StringIO()):
+            from funasr import AutoModel
+            m = AutoModel(model=path, trust_remote_code=True, disable_update=True, device="cpu")
+        encoder, frontend = m.model.encoder, m.kwargs["frontend"]
+        for p in encoder.parameters():
+            p.requires_grad = False
+        return encoder.eval().float(), SenseVoiceAudioProcessor(frontend.eval())
+
+    @staticmethod
+    def load_vision(path):
+        if path is None or not os.path.exists(path):
+            warnings.warn(f"[VAM] Vision model path not found: {path}. vision_encoder will be None!")
+            return None, None
+        hf_logging.set_verbosity_error()
+        try:
+            model = SiglipVisionModel.from_pretrained(path)
+        except (RuntimeError, ValueError):
+            return None, None
+        processor = SiglipImageProcessor.from_pretrained(path)
+        for p in model.parameters():
+            p.requires_grad = False
+        return model.eval(), processor
+
+    @torch.compiler.disable
+    def encode_audio_inputs(self, audio_inputs, audio_lens=None):
+        if (audio_inputs is None) or (self.audio_encoder is None) or (not audio_inputs.any()):
+            return None
+        batch_mask = audio_inputs.flatten(1).any(1)
+        enc_dtype = next(self.audio_encoder.parameters()).dtype
+        valid_fbank = audio_inputs[batch_mask].to(dtype=enc_dtype)
+        if audio_lens is not None:
+            valid_lens = audio_lens[batch_mask].to(valid_fbank.device)
+        else:
+            valid_lens = torch.tensor([valid_fbank.size(1)] * valid_fbank.size(0), device=valid_fbank.device)
+        with torch.no_grad():
+            emb, _ = self.audio_encoder.model(valid_fbank, valid_lens)
+        proj_dtype = next(self.audio_proj.parameters()).dtype
+        emb_list = [self.audio_proj(emb[i, :max(1, min(valid_lens[i].item(), emb.size(1)))].unsqueeze(0).to(proj_dtype)).squeeze(0) for i in range(emb.size(0))]
+        if batch_mask.all():
+            return emb_list
+        out = [None] * audio_inputs.size(0)
+        j = 0
+        for i in range(audio_inputs.size(0)):
+            if batch_mask[i]:
+                out[i] = emb_list[j]
+                j += 1
+        return out
+
+    @torch.compiler.disable
+    def inject_audio_features(self, tokens, h, audio_feats, seqlen):
+        if audio_feats is None or not self.config.audio_ids:
+            return h
+        marker = self.config.audio_ids[0]
+        out = []
+        for b in range(h.size(0)):
+            hb, seq, i = h[b], tokens[b].tolist(), 0
+            af = audio_feats[b] if audio_feats[b] is not None else None
+            while i < len(seq):
+                if seq[i] == marker:
+                    start = i
+                    while i < len(seq) and seq[i] == marker:
+                        i += 1
+                    if af is not None:
+                        inject_len = min(af.size(0), i - start)
+                        hb = torch.cat((hb[:start], af[:inject_len], hb[start + inject_len:]), dim=0)
+                        af = None
+                else:
+                    i += 1
+            out.append(hb)
+        return torch.stack(out)
+
+    @torch.compiler.disable
+    def get_image_embeddings(self, image_inputs):
+        if hasattr(image_inputs, 'keys'):
+            image_inputs = {k: (v.squeeze(1) if v.ndim > 2 and v.shape[1] == 1 else v) for k, v in image_inputs.items()}
+            pixel_attention_mask = image_inputs.get('pixel_attention_mask')
+            if pixel_attention_mask is not None and not pixel_attention_mask.any():
+                pv = image_inputs['pixel_values']
+                return pv.new_zeros(pv.size(0), pv.size(1), self.config.image_hidden_size)
+        with torch.no_grad():
+            outputs = self.vision_encoder.model(**image_inputs)
+        return outputs.last_hidden_state
+
+    @torch.compiler.disable
+    def encode_image_inputs(self, pixel_values):
+        if pixel_values is None or self.vision_encoder is None:
+            return None
+        mask = pixel_values.flatten(1).any(1)
+        if not mask.any():
+            return pixel_values.new_zeros(pixel_values.size(0), self.config.image_token_len, self.config.hidden_size)
+        with torch.no_grad():
+            emb = self.vision_encoder.model(pixel_values=pixel_values[mask]).last_hidden_state
+        if emb.dim() == 2:
+            emb = emb.unsqueeze(0)
+        emb = self.vision_proj(emb)
+        if mask.all():
+            return emb
+        idx = mask.nonzero().view(-1, 1, 1).expand_as(emb)
+        return emb.new_zeros(pixel_values.size(0), *emb.shape[1:]).scatter(0, idx, emb)
+
+    @torch.compiler.disable
+    def count_vision_proj(self, tokens, h, vision_tensors=None, seqlen=512):
+        if vision_tensors is None or not self.config.image_ids:
+            return h
+        marker, vf = self.config.image_ids[0], vision_tensors
+        if vf.dim() == 3:
+            vf = vf.unsqueeze(1)
+        out = []
+        for b in range(h.size(0)):
+            hb, seq, k, i = h[b], tokens[b].tolist(), 0, 0
+            while i < len(seq):
+                if seq[i] == marker:
+                    start = i
+                    while i < len(seq) and seq[i] == marker:
+                        i += 1
+                    if k < vf.size(1):
+                        hb = torch.cat((hb[:start], vf[b][k][:i - start], hb[i:]), dim=0)[:seqlen]
+                        k += 1
+                else:
+                    i += 1
+            out.append(hb)
+        return torch.stack(out)
+
+    def forward(self, input_ids, attention_mask=None, past_key_values=None, use_cache=False, logits_to_keep=0,
+                audio_inputs=None, audio_lens=None, pixel_values=None, **args):
+        if len(input_ids.shape) == 2:
+            batch_size, seq_length = input_ids.shape
+            text_ids = input_ids
+            audio_ids = torch.full((batch_size, 8, seq_length), self.audio_pad_token, dtype=torch.long, device=input_ids.device)
+        else:
+            batch_size, _, seq_length = input_ids.shape
+            text_ids, audio_ids = input_ids[:, 8, :], input_ids[:, :8, :]
+        if hasattr(past_key_values, 'layers'):
+            past_key_values = None
+        n_thinker, n_talker = len(self.thinker.layers), len(self.talker.layers)
+        past_key_values = past_key_values or ([None] * (n_thinker + n_talker))
+        start_pos = past_key_values[0][0].shape[1] if past_key_values[0] is not None else 0
+        if self.thinker.freqs_cos[0, 0] == 0:
+            freqs_cos, freqs_sin = precompute_freqs_cis(dim=self.config.head_dim, end=self.config.max_position_embeddings, rope_base=self.config.rope_theta, rope_scaling=self.config.rope_scaling)
+            self.thinker.freqs_cos, self.thinker.freqs_sin = freqs_cos.to(input_ids.device), freqs_sin.to(input_ids.device)
+        if self.talker.freqs_cos[0, 0] == 0:
+            freqs_cos, freqs_sin = precompute_freqs_cis(dim=self.talker.talker_config.head_dim, end=self.config.max_position_embeddings, rope_base=self.config.rope_theta, rope_scaling=self.config.rope_scaling)
+            self.talker.freqs_cos, self.talker.freqs_sin = freqs_cos.to(input_ids.device), freqs_sin.to(input_ids.device)
+        presents = []
+
+        hidden_states = self.thinker.dropout(self.thinker.embed_tokens(text_ids))
+        position_embeddings = (self.thinker.freqs_cos[start_pos:start_pos + seq_length], self.thinker.freqs_sin[start_pos:start_pos + seq_length])
+        if audio_inputs is not None and start_pos == 0:
+            audio_features = self.encode_audio_inputs(audio_inputs, audio_lens)
+            hidden_states = self.inject_audio_features(text_ids, hidden_states, audio_features, seq_length)
+        if pixel_values is not None and start_pos == 0:
+            if hasattr(pixel_values, 'keys'):
+                img_emb = self.get_image_embeddings(pixel_values).to(hidden_states.dtype)
+                vision_tensors = self.vision_proj(img_emb)
+            else:
+                if len(pixel_values.shape) == 6:
+                    pixel_values = pixel_values.squeeze(2)
+                if len(pixel_values.shape) == 4:
+                    pixel_values = pixel_values.unsqueeze(1)
+                bs, num, c, im_h, im_w = pixel_values.shape
+                stack_dim = 1 if bs > 1 else 0
+                vision_tensors = torch.stack([self.encode_image_inputs(pixel_values[:, i, :, :, :]) for i in range(num)], dim=stack_dim)
+            hidden_states = self.count_vision_proj(tokens=text_ids, h=hidden_states, vision_tensors=vision_tensors, seqlen=seq_length)
+        bridge_states = hidden_states
+        for i, (layer, past_key_value) in enumerate(zip(self.thinker.layers, past_key_values[:n_thinker])):
+            hidden_states, present = layer(hidden_states, position_embeddings, past_key_value=past_key_value, use_cache=use_cache, attention_mask=attention_mask)
+            presents.append(present)
+            if i == self.config.bridge_layer:
+                bridge_states = hidden_states
+        h_thinker = self.thinker.norm(hidden_states)
+
+        talker_emb = self.talker.embed_tokens(audio_ids)
+        spk_emb = args.get('spk_emb', None)
+        if spk_emb is not None:
+            spk_mask = (audio_ids[:, 0, :] == self.audio_spk_token).unsqueeze(-1)
+            talker_emb = torch.where(spk_mask, self.talker.spk_proj(spk_emb).unsqueeze(1), talker_emb)
+        hidden_states = self.talker.embed_proj(bridge_states) * self.talker.text_scale + self.talker.codec_proj(talker_emb) * self.talker.audio_scale
+        talker_pos_emb = (self.talker.freqs_cos[start_pos:start_pos + seq_length], self.talker.freqs_sin[start_pos:start_pos + seq_length])
+        for layer, past_key_value in zip(self.talker.layers, past_key_values[n_thinker:]):
+            hidden_states, present = layer(hidden_states, talker_pos_emb, past_key_value=past_key_value, use_cache=use_cache, attention_mask=attention_mask)
+            presents.append(present)
+        h_talker = self.talker.norm(hidden_states)
+
+        slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
+        aux_loss = sum(l.mlp.aux_loss for l in list(self.thinker.layers) + list(self.talker.layers) if isinstance(l.mlp, MOEFeedForward))
+        aux_loss += sum(p.sum() for p in self.audio_proj.parameters()) * 0 + sum(p.sum() for p in self.vision_proj.parameters()) * 0 + sum(p.sum() for p in self.talker.lm_head.adapters.parameters()) * 0 + sum(p.sum() for p in self.talker.spk_proj.parameters()) * 0
+        text_logits = self.thinker.lm_head(h_thinker[:, slice_indices, :])
+        audio_logits = self.talker.lm_head(h_talker[:, slice_indices, :])
+
+        out = MoeCausalLMOutputWithPast(aux_loss=aux_loss, logits=text_logits, past_key_values=presents)
+        out.audio_logits = audio_logits
+        return out
+
+    @torch.inference_mode()
+    def generate(self, input_ids, eos_token_id=2, max_new_tokens=1024, temperature=0.75, top_p=0.90,
+                 stream=False, rp=1., use_cache=True, return_audio_codes=False, **args):
+        if stream:
+            return self.stream_generate(input_ids, eos_token_id, max_new_tokens, temperature, top_p, rp, use_cache, return_audio_codes, **args)
+        tokens = list(self.stream_generate(input_ids, eos_token_id, max_new_tokens, temperature, top_p, rp, use_cache, return_audio_codes, **args))
+        if tokens:
+            for text_out, _ in reversed(tokens):
+                if text_out is not None:
+                    return text_out
+            return tokens[-1]
+        return input_ids
+
+    def stream_generate(self, input_ids, eos_token_id, max_new_tokens, temperature, top_p, rp, use_cache, return_audio_codes=False, **args):
+        start_pos, past_kvs, text_finished, first_finished = input_ids.shape[1], None, False, True
+        audio_codes = [[] for _ in range(8)]
+        audio_stop_pos = [None] * 8
+        audio_buffer = torch.full((1, 8, start_pos), self.audio_pad_token, dtype=torch.long, device=input_ids.device)
+        spk_emb = args.get('spk_emb', None)
+        ref_codes = args.get('ref_codes', None)
+        ref_len = ref_codes.shape[2] if ref_codes is not None else 0
+        spk_reserve = 1 if spk_emb is not None else 0
+        fill_end = start_pos
+        fill_start = max(spk_reserve, start_pos - ref_len)
+        if ref_codes is not None and fill_start < fill_end:
+            audio_buffer[:, :, fill_start:fill_end] = ref_codes[:, :, -(fill_end - fill_start):]
+        if spk_emb is not None and fill_start > 0:
+            audio_buffer[:, :, fill_start - 1] = self.audio_spk_token
+        think_end_step, generated_tokens = None, ([] if args.get('open_thinking', False) else None)
+        while input_ids.shape[1] < start_pos + max_new_tokens:
+            if past_kvs is None or not use_cache:
+                out = self.forward(torch.cat((audio_buffer, input_ids.unsqueeze(1)), dim=1), past_key_values=past_kvs, use_cache=use_cache, **args)
+            else:
+                out = self.forward(torch.cat((audio_buffer[:, :, -1:], input_ids[:, -1:].unsqueeze(1)), dim=1), past_key_values=past_kvs, use_cache=use_cache, **args)
+            past_kvs = out.past_key_values
+
+            logits = out.logits[0, -1, :].clone() / (temperature + 1e-9)
+            if rp != 1.0:
+                seen = list(set(input_ids[0].tolist()))
+                score = logits[seen]
+                logits[seen] = torch.where(score > 0, score / rp, score * rp)
+            if top_p and top_p < 1.0:
+                sorted_l, sorted_i = torch.sort(logits, descending=True)
+                mask = torch.cumsum(F.softmax(sorted_l, dim=-1), dim=-1) > top_p
+                mask[1:], mask[0] = mask[:-1].clone(), False
+                logits[sorted_i[mask]] = -float('Inf')
+            text_token = torch.multinomial(F.softmax(logits, dim=-1), 1).item()
+
+            if text_finished:
+                text_token = args.get('enter_token_id', 201) if first_finished else args.get('pad_token_id', 0)
+                first_finished = False
+
+            step = input_ids.shape[1] - start_pos
+            audio_step = step - 1
+            if generated_tokens is not None:
+                generated_tokens.append(text_token)
+                if not think_end_step and generated_tokens[-len(self.config.think_end_ids):] == list(self.config.think_end_ids):
+                    think_end_step = step + 2
+                audio_step = (step - think_end_step) if think_end_step else -1
+            for i, al in enumerate(out.audio_logits):
+                if audio_step < i:
+                    audio_codes[i].append(self.audio_pad_token)
+                else:
+                    logits_i = al[0, -1, :].clone() / 0.2
+                    for prev_code in audio_codes[i][-3:]:
+                        score = logits_i[prev_code]
+                        logits_i[prev_code] = torch.where(score > 0, score / 1.05, score * 1.05)
+                    top_val, top_idx = logits_i.topk(50)
+                    code = top_idx[torch.multinomial(F.softmax(top_val, dim=-1), 1)].item()
+                    audio_codes[i].append(code)
+                    if audio_stop_pos[i] is None and code >= 2048:
+                        audio_stop_pos[i] = len(audio_codes[i]) - 1
+
+            if text_finished and all(audio_stop_pos[i] is not None for i in range(8)):
+                break
+
+            input_ids = torch.cat((input_ids, torch.tensor([[text_token]], device=input_ids.device)), dim=1)
+            audio_buffer = torch.cat((audio_buffer, torch.full((1, 8, 1), self.audio_pad_token, dtype=torch.long, device=input_ids.device)), dim=2)
+            for i in range(min(audio_step + 1, 8)):
+                audio_buffer[0, i, -1] = audio_codes[i][-1]
+
+            audio_frame = None
+            if return_audio_codes and audio_step >= 7:
+                frame = [audio_codes[i][step - 7 + i] for i in range(8)]
+                active_layers = sum(1 for i in range(8) if audio_stop_pos[i] is None or step - 7 + i < audio_stop_pos[i])
+                if active_layers >= 8:
+                    audio_frame = frame
+            if not text_finished:
+                yield input_ids[:, start_pos:], audio_frame
+                if text_token == eos_token_id:
+                    text_finished = True
+            else:
+                yield None, audio_frame

checkpoint/omni-o/omni-o-hf/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cda7e3a62c6db47795fb04730b85eb0b09780d13e86d12a4281d81c3964a937c
+size 226283380

checkpoint/omni-o/omni-o-hf/modeling_omni_o.py

@@ -0,0 +1,3 @@
+import sys, os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "src"))
+from models.vam import VAM, VAMConfig

checkpoint/omni-o/omni-o-hf/tokenizer_config.json

@@ -0,0 +1,33 @@
+{
+  "add_prefix_space": false,
+  "audio_bos_token": "<|audio_start|>",
+  "audio_eos_token": "<|audio_end|>",
+  "audio_token": "<|audio_pad|>",
+  "backend": "tokenizers",
+  "bos_token": "<|im_start|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": {},
+  "image_token": "<|image_pad|>",
+  "is_local": true,
+  "legacy": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "model_specific_special_tokens": {
+    "audio_bos_token": "<|audio_start|>",
+    "audio_eos_token": "<|audio_end|>",
+    "audio_token": "<|audio_pad|>",
+    "image_token": "<|image_pad|>",
+    "video_token": "<|video_pad|>",
+    "vision_bos_token": "<|vision_start|>",
+    "vision_eos_token": "<|vision_end|>"
+  },
+  "pad_token": "<|endoftext|>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<|endoftext|>",
+  "video_token": "<|video_pad|>",
+  "vision_bos_token": "<|vision_start|>",
+  "vision_eos_token": "<|vision_end|>"
+}

checkpoint/omni-o/omni-o-moe-hf/chat_template.jinja

@@ -0,0 +1,85 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if true %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if open_thinking is defined and open_thinking is true %}
+        {{- '<think>\n' }}
+    {%- else %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

checkpoint/omni-o/omni-o-moe-hf/config.json

@@ -0,0 +1,60 @@
+{
+  "architectures": [
+    "VAM"
+  ],
+  "audio_hidden_size": 512,
+  "audio_ids": [
+    16
+  ],
+  "audio_pad_token": 2049,
+  "audio_special_token": "<|audio_pad|>",
+  "audio_spk_token": 2051,
+  "audio_stop_token": 2050,
+  "audio_vocab_size": 2112,
+  "auto_map": {
+    "AutoConfig": "modeling_omni_o.VAMConfig",
+    "AutoModelForCausalLM": "modeling_omni_o.VAM"
+  },
+  "bos_token_id": 1,
+  "bridge_layer": 3,
+  "dropout": 0.0,
+  "dtype": "float16",
+  "eos_token_id": 2,
+  "flash_attn": true,
+  "head_dim": 96,
+  "hidden_act": "silu",
+  "hidden_size": 768,
+  "image_hidden_size": 768,
+  "image_ids": [
+    12
+  ],
+  "image_special_token": "<|image_pad|>",
+  "image_token_len": 64,
+  "inference_rope_scaling": false,
+  "intermediate_size": 2432,
+  "max_position_embeddings": 32768,
+  "model_type": "omni-o",
+  "moe_intermediate_size": 2432,
+  "norm_topk_prob": true,
+  "num_attention_heads": 8,
+  "num_experts": 4,
+  "num_experts_per_tok": 1,
+  "num_hidden_layers": 8,
+  "num_key_value_heads": 4,
+  "num_talker_hidden_layers": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": null,
+  "rope_theta": 1000000.0,
+  "router_aux_loss_coef": 0.0005,
+  "spk_emb_size": 192,
+  "talker_hidden_size": 768,
+  "think_end_ids": [
+    26,
+    234,
+    234
+  ],
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.1",
+  "use_moe": true,
+  "vocab_size": 6400
+}

checkpoint/omni-o/omni-o-moe-hf/config.py

@@ -0,0 +1,24 @@
+from models.lm.config import LMConfig
+
+
+class VAMConfig(LMConfig):
+    model_type = "omni-o"
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.num_talker_hidden_layers = kwargs.get("num_talker_hidden_layers", 4)
+        self.talker_hidden_size = kwargs.get("talker_hidden_size", 768)
+        self.audio_ids = kwargs.get("audio_ids", [16])
+        self.audio_special_token = kwargs.get("audio_special_token", "<|audio_pad|>")
+        self.audio_hidden_size = kwargs.get("audio_hidden_size", 512)
+        self.audio_vocab_size = kwargs.get("audio_vocab_size", 2112)
+        self.audio_pad_token = kwargs.get("audio_pad_token", 2049)
+        self.audio_stop_token = kwargs.get("audio_stop_token", 2050)
+        self.audio_spk_token = kwargs.get("audio_spk_token", 2051)
+        self.spk_emb_size = kwargs.get("spk_emb_size", 192)
+        self.think_end_ids = kwargs.get("think_end_ids", [26, 234, 234])
+        self.image_ids = kwargs.get("image_ids", [12])
+        self.image_special_token = kwargs.get("image_special_token", "<|image_pad|>")
+        self.image_hidden_size = kwargs.get("image_hidden_size", 768)
+        self.image_token_len = kwargs.get("image_token_len", 64)
+        self.bridge_layer = kwargs.get("bridge_layer", self.num_hidden_layers // 2 - 1)

checkpoint/omni-o/omni-o-moe-hf/generation_config.json

@@ -0,0 +1,8 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.14.1"
+}

checkpoint/omni-o/omni-o-moe-hf/model.py

@@ -0,0 +1,415 @@
+import os
+import math
+import torch
+import warnings
+import logging
+import contextlib
+import io
+from torch import nn
+from torch.nn import functional as F
+from transformers.modeling_outputs import MoeCausalLMOutputWithPast
+from transformers import SiglipVisionModel, SiglipImageProcessor, logging as hf_logging
+
+from core import RMSNorm, precompute_freqs_cis, Block, MOEFeedForward
+from models.lm.config import LMConfig
+from models.lm.model import LMForCausalLM
+from models.vam.config import VAMConfig
+from encoders.audio import SenseVoiceAudioEncoder, SenseVoiceAudioProcessor
+from encoders.vision import SiglipVisionEncoder
+from projectors import MMVisionProjector, MMAudioProjector
+
+
+class TalkerHead(nn.Module):
+    def __init__(self, in_features, out_features, num_layers=8, rank=256):
+        super().__init__()
+        self.num_layers = num_layers
+        self.base = nn.Linear(in_features, out_features, bias=False)
+        self.adapters = nn.ModuleList([
+            nn.Sequential(nn.Linear(in_features, rank, bias=False), nn.GELU(), nn.Linear(rank, out_features, bias=False))
+            for _ in range(num_layers)
+        ])
+
+    def forward(self, x):
+        base_out = self.base(x)
+        return [base_out + adapter(x) for adapter in self.adapters]
+
+
+class TalkerEmbedding(nn.Module):
+    def __init__(self, num_embeddings, embedding_dim, num_layers=8, rank=256):
+        super().__init__()
+        self.num_layers = num_layers
+        self.base = nn.Embedding(num_embeddings, embedding_dim)
+        self.adapters = nn.ModuleList([
+            nn.Sequential(nn.Embedding(num_embeddings, rank), nn.GELU(), nn.Linear(rank, embedding_dim, bias=False))
+            for _ in range(num_layers)
+        ])
+
+    def forward(self, x):
+        base_out = self.base(x)
+        return sum(base_out[:, i, :] + self.adapters[i](x[:, i, :]) for i in range(len(self.adapters))) / self.num_layers
+
+
+class TalkerModule(nn.Module):
+    def __init__(self, config: VAMConfig):
+        super().__init__()
+        self.talker_config = LMConfig(hidden_size=config.talker_hidden_size, use_moe=config.use_moe)
+        self.layers = nn.ModuleList([Block(l, self.talker_config) for l in range(config.num_talker_hidden_layers)])
+        self.norm = RMSNorm(config.talker_hidden_size, eps=config.rms_norm_eps)
+        self.lm_head = TalkerHead(config.talker_hidden_size, config.audio_vocab_size)
+        self.embed_tokens = TalkerEmbedding(config.audio_vocab_size, config.talker_hidden_size)
+        self.codec_proj = nn.Sequential(
+            nn.Linear(config.talker_hidden_size, config.talker_hidden_size),
+            nn.GELU(),
+            nn.Linear(config.talker_hidden_size, config.talker_hidden_size),
+            RMSNorm(config.talker_hidden_size, eps=config.rms_norm_eps),
+        )
+        self.embed_proj = nn.Sequential(
+            nn.Linear(config.hidden_size, config.hidden_size),
+            nn.GELU(),
+            nn.Linear(config.hidden_size, config.talker_hidden_size),
+            RMSNorm(config.talker_hidden_size, eps=config.rms_norm_eps),
+        )
+        self.text_scale, self.audio_scale = nn.Parameter(torch.tensor(3.0)), nn.Parameter(torch.tensor(1.0))
+        self.spk_proj = nn.Linear(config.spk_emb_size, config.talker_hidden_size, bias=False)
+        freqs_cos, freqs_sin = precompute_freqs_cis(
+            dim=self.talker_config.head_dim, end=config.max_position_embeddings,
+            rope_base=config.rope_theta, rope_scaling=config.rope_scaling
+        )
+        self.register_buffer("freqs_cos", freqs_cos, persistent=False)
+        self.register_buffer("freqs_sin", freqs_sin, persistent=False)
+
+
+class VAM(LMForCausalLM):
+    config_class = VAMConfig
+
+    def __init__(self, config: VAMConfig = None, audio_encoder_path: str = None, vision_model_path: str = None):
+        config = config or VAMConfig()
+        super().__init__(config)
+        object.__setattr__(self, 'thinker', self.model)
+        object.__setattr__(self.model, 'lm_head', self.lm_head)
+        self.talker = TalkerModule(config)
+        self.audio_proj = MMAudioProjector(config.audio_hidden_size, config.hidden_size)
+        self.vision_proj = MMVisionProjector(config.image_hidden_size, config.hidden_size, target_tokens=config.image_token_len)
+        self.audio_pad_token, self.audio_stop_token, self.audio_spk_token = config.audio_pad_token, config.audio_stop_token, config.audio_spk_token
+        meta_init = any(p.device.type == 'meta' for p in self.parameters())
+        if meta_init:
+            object.__setattr__(self, 'audio_encoder', None)
+            object.__setattr__(self, 'audio_processor', None)
+            object.__setattr__(self, 'vision_encoder', None)
+            object.__setattr__(self, 'vision_processor', None)
+        else:
+            audio_enc = SenseVoiceAudioEncoder(audio_encoder_path) if audio_encoder_path else SenseVoiceAudioEncoder()
+            object.__setattr__(self, 'audio_encoder', audio_enc)
+            object.__setattr__(self, 'audio_processor', audio_enc.processor)
+            vision_enc = SiglipVisionEncoder(vision_model_path) if vision_model_path else SiglipVisionEncoder()
+            object.__setattr__(self, 'vision_encoder', vision_enc)
+            object.__setattr__(self, 'vision_processor', vision_enc.processor)
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
+        audio_encoder_path = kwargs.pop('audio_encoder_path', None)
+        vision_model_path = kwargs.pop('vision_model_path', None)
+        model = super().from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+        if audio_encoder_path and model.audio_encoder is None:
+            enc, proc = cls.load_sensevoice(audio_encoder_path)
+            object.__setattr__(model, 'audio_encoder', enc)
+            object.__setattr__(model, 'audio_processor', proc)
+        if vision_model_path and model.vision_encoder is None:
+            enc, proc = cls.load_vision(vision_model_path)
+            object.__setattr__(model, 'vision_encoder', enc)
+            object.__setattr__(model, 'vision_processor', proc)
+        return model
+
+    @staticmethod
+    def load_sensevoice(path):
+        if not os.path.exists(path):
+            warnings.warn(f"[VAM] SenseVoice path not found: {path}")
+            return None, None
+        logging.getLogger().setLevel(logging.ERROR)
+        hf_logging.set_verbosity_error()
+        with contextlib.redirect_stdout(io.StringIO()):
+            from funasr import AutoModel
+            m = AutoModel(model=path, trust_remote_code=True, disable_update=True, device="cpu")
+        encoder, frontend = m.model.encoder, m.kwargs["frontend"]
+        for p in encoder.parameters():
+            p.requires_grad = False
+        return encoder.eval().float(), SenseVoiceAudioProcessor(frontend.eval())
+
+    @staticmethod
+    def load_vision(path):
+        if path is None or not os.path.exists(path):
+            warnings.warn(f"[VAM] Vision model path not found: {path}. vision_encoder will be None!")
+            return None, None
+        hf_logging.set_verbosity_error()
+        try:
+            model = SiglipVisionModel.from_pretrained(path)
+        except (RuntimeError, ValueError):
+            return None, None
+        processor = SiglipImageProcessor.from_pretrained(path)
+        for p in model.parameters():
+            p.requires_grad = False
+        return model.eval(), processor
+
+    @torch.compiler.disable
+    def encode_audio_inputs(self, audio_inputs, audio_lens=None):
+        if (audio_inputs is None) or (self.audio_encoder is None) or (not audio_inputs.any()):
+            return None
+        batch_mask = audio_inputs.flatten(1).any(1)
+        enc_dtype = next(self.audio_encoder.parameters()).dtype
+        valid_fbank = audio_inputs[batch_mask].to(dtype=enc_dtype)
+        if audio_lens is not None:
+            valid_lens = audio_lens[batch_mask].to(valid_fbank.device)
+        else:
+            valid_lens = torch.tensor([valid_fbank.size(1)] * valid_fbank.size(0), device=valid_fbank.device)
+        with torch.no_grad():
+            emb, _ = self.audio_encoder.model(valid_fbank, valid_lens)
+        proj_dtype = next(self.audio_proj.parameters()).dtype
+        emb_list = [self.audio_proj(emb[i, :max(1, min(valid_lens[i].item(), emb.size(1)))].unsqueeze(0).to(proj_dtype)).squeeze(0) for i in range(emb.size(0))]
+        if batch_mask.all():
+            return emb_list
+        out = [None] * audio_inputs.size(0)
+        j = 0
+        for i in range(audio_inputs.size(0)):
+            if batch_mask[i]:
+                out[i] = emb_list[j]
+                j += 1
+        return out
+
+    @torch.compiler.disable
+    def inject_audio_features(self, tokens, h, audio_feats, seqlen):
+        if audio_feats is None or not self.config.audio_ids:
+            return h
+        marker = self.config.audio_ids[0]
+        out = []
+        for b in range(h.size(0)):
+            hb, seq, i = h[b], tokens[b].tolist(), 0
+            af = audio_feats[b] if audio_feats[b] is not None else None
+            while i < len(seq):
+                if seq[i] == marker:
+                    start = i
+                    while i < len(seq) and seq[i] == marker:
+                        i += 1
+                    if af is not None:
+                        inject_len = min(af.size(0), i - start)
+                        hb = torch.cat((hb[:start], af[:inject_len], hb[start + inject_len:]), dim=0)
+                        af = None
+                else:
+                    i += 1
+            out.append(hb)
+        return torch.stack(out)
+
+    @torch.compiler.disable
+    def get_image_embeddings(self, image_inputs):
+        if hasattr(image_inputs, 'keys'):
+            image_inputs = {k: (v.squeeze(1) if v.ndim > 2 and v.shape[1] == 1 else v) for k, v in image_inputs.items()}
+            pixel_attention_mask = image_inputs.get('pixel_attention_mask')
+            if pixel_attention_mask is not None and not pixel_attention_mask.any():
+                pv = image_inputs['pixel_values']
+                return pv.new_zeros(pv.size(0), pv.size(1), self.config.image_hidden_size)
+        with torch.no_grad():
+            outputs = self.vision_encoder.model(**image_inputs)
+        return outputs.last_hidden_state
+
+    @torch.compiler.disable
+    def encode_image_inputs(self, pixel_values):
+        if pixel_values is None or self.vision_encoder is None:
+            return None
+        mask = pixel_values.flatten(1).any(1)
+        if not mask.any():
+            return pixel_values.new_zeros(pixel_values.size(0), self.config.image_token_len, self.config.hidden_size)
+        with torch.no_grad():
+            emb = self.vision_encoder.model(pixel_values=pixel_values[mask]).last_hidden_state
+        if emb.dim() == 2:
+            emb = emb.unsqueeze(0)
+        emb = self.vision_proj(emb)
+        if mask.all():
+            return emb
+        idx = mask.nonzero().view(-1, 1, 1).expand_as(emb)
+        return emb.new_zeros(pixel_values.size(0), *emb.shape[1:]).scatter(0, idx, emb)
+
+    @torch.compiler.disable
+    def count_vision_proj(self, tokens, h, vision_tensors=None, seqlen=512):
+        if vision_tensors is None or not self.config.image_ids:
+            return h
+        marker, vf = self.config.image_ids[0], vision_tensors
+        if vf.dim() == 3:
+            vf = vf.unsqueeze(1)
+        out = []
+        for b in range(h.size(0)):
+            hb, seq, k, i = h[b], tokens[b].tolist(), 0, 0
+            while i < len(seq):
+                if seq[i] == marker:
+                    start = i
+                    while i < len(seq) and seq[i] == marker:
+                        i += 1
+                    if k < vf.size(1):
+                        hb = torch.cat((hb[:start], vf[b][k][:i - start], hb[i:]), dim=0)[:seqlen]
+                        k += 1
+                else:
+                    i += 1
+            out.append(hb)
+        return torch.stack(out)
+
+    def forward(self, input_ids, attention_mask=None, past_key_values=None, use_cache=False, logits_to_keep=0,
+                audio_inputs=None, audio_lens=None, pixel_values=None, **args):
+        if len(input_ids.shape) == 2:
+            batch_size, seq_length = input_ids.shape
+            text_ids = input_ids
+            audio_ids = torch.full((batch_size, 8, seq_length), self.audio_pad_token, dtype=torch.long, device=input_ids.device)
+        else:
+            batch_size, _, seq_length = input_ids.shape
+            text_ids, audio_ids = input_ids[:, 8, :], input_ids[:, :8, :]
+        if hasattr(past_key_values, 'layers'):
+            past_key_values = None
+        n_thinker, n_talker = len(self.thinker.layers), len(self.talker.layers)
+        past_key_values = past_key_values or ([None] * (n_thinker + n_talker))
+        start_pos = past_key_values[0][0].shape[1] if past_key_values[0] is not None else 0
+        if self.thinker.freqs_cos[0, 0] == 0:
+            freqs_cos, freqs_sin = precompute_freqs_cis(dim=self.config.head_dim, end=self.config.max_position_embeddings, rope_base=self.config.rope_theta, rope_scaling=self.config.rope_scaling)
+            self.thinker.freqs_cos, self.thinker.freqs_sin = freqs_cos.to(input_ids.device), freqs_sin.to(input_ids.device)
+        if self.talker.freqs_cos[0, 0] == 0:
+            freqs_cos, freqs_sin = precompute_freqs_cis(dim=self.talker.talker_config.head_dim, end=self.config.max_position_embeddings, rope_base=self.config.rope_theta, rope_scaling=self.config.rope_scaling)
+            self.talker.freqs_cos, self.talker.freqs_sin = freqs_cos.to(input_ids.device), freqs_sin.to(input_ids.device)
+        presents = []
+
+        hidden_states = self.thinker.dropout(self.thinker.embed_tokens(text_ids))
+        position_embeddings = (self.thinker.freqs_cos[start_pos:start_pos + seq_length], self.thinker.freqs_sin[start_pos:start_pos + seq_length])
+        if audio_inputs is not None and start_pos == 0:
+            audio_features = self.encode_audio_inputs(audio_inputs, audio_lens)
+            hidden_states = self.inject_audio_features(text_ids, hidden_states, audio_features, seq_length)
+        if pixel_values is not None and start_pos == 0:
+            if hasattr(pixel_values, 'keys'):
+                img_emb = self.get_image_embeddings(pixel_values).to(hidden_states.dtype)
+                vision_tensors = self.vision_proj(img_emb)
+            else:
+                if len(pixel_values.shape) == 6:
+                    pixel_values = pixel_values.squeeze(2)
+                if len(pixel_values.shape) == 4:
+                    pixel_values = pixel_values.unsqueeze(1)
+                bs, num, c, im_h, im_w = pixel_values.shape
+                stack_dim = 1 if bs > 1 else 0
+                vision_tensors = torch.stack([self.encode_image_inputs(pixel_values[:, i, :, :, :]) for i in range(num)], dim=stack_dim)
+            hidden_states = self.count_vision_proj(tokens=text_ids, h=hidden_states, vision_tensors=vision_tensors, seqlen=seq_length)
+        bridge_states = hidden_states
+        for i, (layer, past_key_value) in enumerate(zip(self.thinker.layers, past_key_values[:n_thinker])):
+            hidden_states, present = layer(hidden_states, position_embeddings, past_key_value=past_key_value, use_cache=use_cache, attention_mask=attention_mask)
+            presents.append(present)
+            if i == self.config.bridge_layer:
+                bridge_states = hidden_states
+        h_thinker = self.thinker.norm(hidden_states)
+
+        talker_emb = self.talker.embed_tokens(audio_ids)
+        spk_emb = args.get('spk_emb', None)
+        if spk_emb is not None:
+            spk_mask = (audio_ids[:, 0, :] == self.audio_spk_token).unsqueeze(-1)
+            talker_emb = torch.where(spk_mask, self.talker.spk_proj(spk_emb).unsqueeze(1), talker_emb)
+        hidden_states = self.talker.embed_proj(bridge_states) * self.talker.text_scale + self.talker.codec_proj(talker_emb) * self.talker.audio_scale
+        talker_pos_emb = (self.talker.freqs_cos[start_pos:start_pos + seq_length], self.talker.freqs_sin[start_pos:start_pos + seq_length])
+        for layer, past_key_value in zip(self.talker.layers, past_key_values[n_thinker:]):
+            hidden_states, present = layer(hidden_states, talker_pos_emb, past_key_value=past_key_value, use_cache=use_cache, attention_mask=attention_mask)
+            presents.append(present)
+        h_talker = self.talker.norm(hidden_states)
+
+        slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
+        aux_loss = sum(l.mlp.aux_loss for l in list(self.thinker.layers) + list(self.talker.layers) if isinstance(l.mlp, MOEFeedForward))
+        aux_loss += sum(p.sum() for p in self.audio_proj.parameters()) * 0 + sum(p.sum() for p in self.vision_proj.parameters()) * 0 + sum(p.sum() for p in self.talker.lm_head.adapters.parameters()) * 0 + sum(p.sum() for p in self.talker.spk_proj.parameters()) * 0
+        text_logits = self.thinker.lm_head(h_thinker[:, slice_indices, :])
+        audio_logits = self.talker.lm_head(h_talker[:, slice_indices, :])
+
+        out = MoeCausalLMOutputWithPast(aux_loss=aux_loss, logits=text_logits, past_key_values=presents)
+        out.audio_logits = audio_logits
+        return out
+
+    @torch.inference_mode()
+    def generate(self, input_ids, eos_token_id=2, max_new_tokens=1024, temperature=0.75, top_p=0.90,
+                 stream=False, rp=1., use_cache=True, return_audio_codes=False, **args):
+        if stream:
+            return self.stream_generate(input_ids, eos_token_id, max_new_tokens, temperature, top_p, rp, use_cache, return_audio_codes, **args)
+        tokens = list(self.stream_generate(input_ids, eos_token_id, max_new_tokens, temperature, top_p, rp, use_cache, return_audio_codes, **args))
+        if tokens:
+            for text_out, _ in reversed(tokens):
+                if text_out is not None:
+                    return text_out
+            return tokens[-1]
+        return input_ids
+
+    def stream_generate(self, input_ids, eos_token_id, max_new_tokens, temperature, top_p, rp, use_cache, return_audio_codes=False, **args):
+        start_pos, past_kvs, text_finished, first_finished = input_ids.shape[1], None, False, True
+        audio_codes = [[] for _ in range(8)]
+        audio_stop_pos = [None] * 8
+        audio_buffer = torch.full((1, 8, start_pos), self.audio_pad_token, dtype=torch.long, device=input_ids.device)
+        spk_emb = args.get('spk_emb', None)
+        ref_codes = args.get('ref_codes', None)
+        ref_len = ref_codes.shape[2] if ref_codes is not None else 0
+        spk_reserve = 1 if spk_emb is not None else 0
+        fill_end = start_pos
+        fill_start = max(spk_reserve, start_pos - ref_len)
+        if ref_codes is not None and fill_start < fill_end:
+            audio_buffer[:, :, fill_start:fill_end] = ref_codes[:, :, -(fill_end - fill_start):]
+        if spk_emb is not None and fill_start > 0:
+            audio_buffer[:, :, fill_start - 1] = self.audio_spk_token
+        think_end_step, generated_tokens = None, ([] if args.get('open_thinking', False) else None)
+        while input_ids.shape[1] < start_pos + max_new_tokens:
+            if past_kvs is None or not use_cache:
+                out = self.forward(torch.cat((audio_buffer, input_ids.unsqueeze(1)), dim=1), past_key_values=past_kvs, use_cache=use_cache, **args)
+            else:
+                out = self.forward(torch.cat((audio_buffer[:, :, -1:], input_ids[:, -1:].unsqueeze(1)), dim=1), past_key_values=past_kvs, use_cache=use_cache, **args)
+            past_kvs = out.past_key_values
+
+            logits = out.logits[0, -1, :].clone() / (temperature + 1e-9)
+            if rp != 1.0:
+                seen = list(set(input_ids[0].tolist()))
+                score = logits[seen]
+                logits[seen] = torch.where(score > 0, score / rp, score * rp)
+            if top_p and top_p < 1.0:
+                sorted_l, sorted_i = torch.sort(logits, descending=True)
+                mask = torch.cumsum(F.softmax(sorted_l, dim=-1), dim=-1) > top_p
+                mask[1:], mask[0] = mask[:-1].clone(), False
+                logits[sorted_i[mask]] = -float('Inf')
+            text_token = torch.multinomial(F.softmax(logits, dim=-1), 1).item()
+
+            if text_finished:
+                text_token = args.get('enter_token_id', 201) if first_finished else args.get('pad_token_id', 0)
+                first_finished = False
+
+            step = input_ids.shape[1] - start_pos
+            audio_step = step - 1
+            if generated_tokens is not None:
+                generated_tokens.append(text_token)
+                if not think_end_step and generated_tokens[-len(self.config.think_end_ids):] == list(self.config.think_end_ids):
+                    think_end_step = step + 2
+                audio_step = (step - think_end_step) if think_end_step else -1
+            for i, al in enumerate(out.audio_logits):
+                if audio_step < i:
+                    audio_codes[i].append(self.audio_pad_token)
+                else:
+                    logits_i = al[0, -1, :].clone() / 0.2
+                    for prev_code in audio_codes[i][-3:]:
+                        score = logits_i[prev_code]
+                        logits_i[prev_code] = torch.where(score > 0, score / 1.05, score * 1.05)
+                    top_val, top_idx = logits_i.topk(50)
+                    code = top_idx[torch.multinomial(F.softmax(top_val, dim=-1), 1)].item()
+                    audio_codes[i].append(code)
+                    if audio_stop_pos[i] is None and code >= 2048:
+                        audio_stop_pos[i] = len(audio_codes[i]) - 1
+
+            if text_finished and all(audio_stop_pos[i] is not None for i in range(8)):
+                break
+
+            input_ids = torch.cat((input_ids, torch.tensor([[text_token]], device=input_ids.device)), dim=1)
+            audio_buffer = torch.cat((audio_buffer, torch.full((1, 8, 1), self.audio_pad_token, dtype=torch.long, device=input_ids.device)), dim=2)
+            for i in range(min(audio_step + 1, 8)):
+                audio_buffer[0, i, -1] = audio_codes[i][-1]
+
+            audio_frame = None
+            if return_audio_codes and audio_step >= 7:
+                frame = [audio_codes[i][step - 7 + i] for i in range(8)]
+                active_layers = sum(1 for i in range(8) if audio_stop_pos[i] is None or step - 7 + i < audio_stop_pos[i])
+                if active_layers >= 8:
+                    audio_frame = frame
+            if not text_finished:
+                yield input_ids[:, start_pos:], audio_frame
+                if text_token == eos_token_id:
+                    text_finished = True
+            else:
+                yield None, audio_frame

checkpoint/omni-o/omni-o-moe-hf/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37f1f8e4fd120026b82bdef41dc2fc8d1a16328cf027d1f600327c88cec5b36f
+size 629811308

checkpoint/omni-o/omni-o-moe-hf/modeling_omni_o.py

@@ -0,0 +1,3 @@
+import sys, os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "src"))
+from models.vam import VAM, VAMConfig

checkpoint/omni-o/omni-o-moe-hf/tokenizer_config.json

@@ -0,0 +1,33 @@
+{
+  "add_prefix_space": false,
+  "audio_bos_token": "<|audio_start|>",
+  "audio_eos_token": "<|audio_end|>",
+  "audio_token": "<|audio_pad|>",
+  "backend": "tokenizers",
+  "bos_token": "<|im_start|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": {},
+  "image_token": "<|image_pad|>",
+  "is_local": true,
+  "legacy": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "model_specific_special_tokens": {
+    "audio_bos_token": "<|audio_start|>",
+    "audio_eos_token": "<|audio_end|>",
+    "audio_token": "<|audio_pad|>",
+    "image_token": "<|image_pad|>",
+    "video_token": "<|video_pad|>",
+    "vision_bos_token": "<|vision_start|>",
+    "vision_eos_token": "<|vision_end|>"
+  },
+  "pad_token": "<|endoftext|>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<|endoftext|>",
+  "video_token": "<|video_pad|>",
+  "vision_bos_token": "<|vision_start|>",
+  "vision_eos_token": "<|vision_end|>"
+}

checkpoint/omni-o/omni-o.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a05a617858453aa9490bb0ca9e22503e51dd8718348bc06d3a549ffecb589f8
+size 236155558

checkpoint/omni-o/omni-o_moe.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cef510dab06320b1e3991c2bec5ddf2263d1b0c89d1fa4070b2567e4fef6d1a
+size 639710162

checkpoint/omni/native_hf/chat_template.jinja

@@ -0,0 +1,85 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if true %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if open_thinking is defined and open_thinking is true %}
+        {{- '<think>\n' }}
+    {%- else %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

checkpoint/omni/native_hf/config.json

@@ -0,0 +1,33 @@
+{
+  "architectures": [
+    "LMForCausalLM"
+  ],
+  "bos_token_id": 1,
+  "dropout": 0.0,
+  "dtype": "float16",
+  "eos_token_id": 2,
+  "flash_attn": true,
+  "head_dim": 96,
+  "hidden_act": "silu",
+  "hidden_size": 768,
+  "inference_rope_scaling": false,
+  "intermediate_size": 2432,
+  "max_position_embeddings": 32768,
+  "max_seq_len": 8192,
+  "model_type": "omni",
+  "moe_intermediate_size": 2432,
+  "norm_topk_prob": true,
+  "num_attention_heads": 8,
+  "num_experts": 4,
+  "num_experts_per_tok": 1,
+  "num_hidden_layers": 8,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "router_aux_loss_coef": 0.0005,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.1",
+  "use_moe": false,
+  "vocab_size": 6400,
+  "rope_scaling": null
+}

checkpoint/omni/native_hf/generation_config.json

@@ -0,0 +1,8 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.14.1"
+}

checkpoint/omni/native_hf/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3adf69402b5d22e693151cabadc12528f923c4ba6bf343738aaf13f0892162e8
+size 127834168

checkpoint/omni/native_hf/tokenizer_config.json

@@ -0,0 +1,33 @@
+{
+  "add_prefix_space": false,
+  "audio_bos_token": "<|audio_start|>",
+  "audio_eos_token": "<|audio_end|>",
+  "audio_token": "<|audio_pad|>",
+  "backend": "tokenizers",
+  "bos_token": "<|im_start|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": {},
+  "image_token": "<|image_pad|>",
+  "is_local": true,
+  "legacy": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "model_specific_special_tokens": {
+    "audio_bos_token": "<|audio_start|>",
+    "audio_eos_token": "<|audio_end|>",
+    "audio_token": "<|audio_pad|>",
+    "image_token": "<|image_pad|>",
+    "video_token": "<|video_pad|>",
+    "vision_bos_token": "<|vision_start|>",
+    "vision_eos_token": "<|vision_end|>"
+  },
+  "pad_token": "<|endoftext|>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<|endoftext|>",
+  "video_token": "<|video_pad|>",
+  "vision_bos_token": "<|vision_start|>",
+  "vision_eos_token": "<|vision_end|>"
+}

checkpoint/omni/omni.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b8c49f6c9d965092e651cafeeaeb8705558632b3fd4ac8ab319cbc5a3cbc4a0
+size 137683346

checkpoint/sensevoice/.mv

@@ -0,0 +1 @@
+Revision:master,CreatedAt:1777630553

checkpoint/sensevoice/README.md

@@ -0,0 +1,27 @@
+---
+license: apache-2.0
+language:
+  - zh
+  - en
+  - yue
+  - ja
+  - ko
+tags:
+  - automatic-speech-recognition
+  - speech
+  - audio
+  - encoder
+  - sensevoice
+pipeline_tag: automatic-speech-recognition
+---
+
+# SenseVoiceSmall
+
+This directory contains a local copy of [SenseVoiceSmall](https://www.modelscope.cn/iic/SenseVoiceSmall), released by the FunAudioLLM team. The model is used as a speech encoder / speech recognition component, while the original model architecture, weights, authorship, and license remain attributed to the original authors.
+
+The original SenseVoiceSmall model is released under the `Apache-2.0` license. When using, redistributing, or citing these files, please keep the original author information, model link, and license notice.
+
+Original links:
+
+- ModelScope: https://www.modelscope.cn/iic/SenseVoiceSmall
+- License: https://www.apache.org/licenses/LICENSE-2.0

checkpoint/sensevoice/am.mvn

@@ -0,0 +1,8 @@
+<Nnet> 
+<Splice> 560 560
+[ 0 ]
+<AddShift> 560 560 
+<LearnRateCoef> 0 [ -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 ]
+<Rescale> 560 560
+<LearnRateCoef> 0 [ 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 ]
+</Nnet> 

checkpoint/sensevoice/chn_jpn_yue_eng_ko_spectok.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa87f86064c3730d799ddf7af3c04659151102cba548bce325cf06ba4da4e6a8
+size 377341

checkpoint/sensevoice/config.yaml

@@ -0,0 +1,97 @@
+encoder: SenseVoiceEncoderSmall
+encoder_conf:
+    output_size: 512
+    attention_heads: 4
+    linear_units: 2048
+    num_blocks: 50
+    tp_blocks: 20
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: pe
+    pos_enc_class: SinusoidalPositionEncoder
+    normalize_before: true
+    kernel_size: 11
+    sanm_shfit: 0
+    selfattention_layer_type: sanm
+
+
+model: SenseVoiceSmall
+model_conf:
+    length_normalized_loss: true
+    sos: 1
+    eos: 2
+    ignore_id: -1
+
+tokenizer: SentencepiecesTokenizer
+tokenizer_conf:
+  bpemodel: null
+  unk_symbol: <unk>
+  split_with_space: true
+
+frontend: WavFrontend
+frontend_conf:
+    fs: 16000
+    window: hamming
+    n_mels: 80
+    frame_length: 25
+    frame_shift: 10
+    lfr_m: 7
+    lfr_n: 6
+    cmvn_file: null
+
+
+dataset: SenseVoiceCTCDataset
+dataset_conf:
+  index_ds: IndexDSJsonl
+  batch_sampler: EspnetStyleBatchSampler
+  data_split_num: 32
+  batch_type: token
+  batch_size: 14000
+  max_token_length: 2000
+  min_token_length: 60
+  max_source_length: 2000
+  min_source_length: 60
+  max_target_length: 200
+  min_target_length: 0
+  shuffle: true
+  num_workers: 4
+  sos: ${model_conf.sos}
+  eos: ${model_conf.eos}
+  IndexDSJsonl: IndexDSJsonl
+  retry: 20
+
+train_conf:
+  accum_grad: 1
+  grad_clip: 5
+  max_epoch: 20
+  keep_nbest_models: 10
+  avg_nbest_model: 10
+  log_interval: 100
+  resume: true
+  validate_interval: 10000
+  save_checkpoint_interval: 10000
+
+optim: adamw
+optim_conf:
+  lr: 0.00002
+scheduler: warmuplr
+scheduler_conf:
+  warmup_steps: 25000
+
+specaug: SpecAugLFR
+specaug_conf:
+    apply_time_warp: false
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 30
+    lfr_rate: 6
+    num_freq_mask: 1
+    apply_time_mask: true
+    time_mask_width_range:
+    - 0
+    - 12
+    num_time_mask: 1

checkpoint/sensevoice/configuration.json

@@ -0,0 +1,14 @@
+{
+  "framework": "pytorch",
+  "task" : "auto-speech-recognition",
+  "model": {"type" : "funasr"},
+  "pipeline": {"type":"funasr-pipeline"},
+  "model_name_in_hub": {
+    "ms":"", 
+    "hf":""},
+  "file_path_metas": {
+    "init_param":"model.pt", 
+    "config":"config.yaml",
+    "tokenizer_conf": {"bpemodel": "chn_jpn_yue_eng_ko_spectok.bpe.model"},
+    "frontend_conf":{"cmvn_file": "am.mvn"}}
+}