Upload 5 files

.gitattributes

@@ -48,3 +48,4 @@ engines_linux/libaudiocpp_engine.so filter=lfs diff=lfs merge=lfs -text
 engines_linux/libcublas.so.13 filter=lfs diff=lfs merge=lfs -text
 engines_linux/libcublasLt.so.13 filter=lfs diff=lfs merge=lfs -text
 engines_linux/libcudart.so.13 filter=lfs diff=lfs merge=lfs -text
+models/higgs-bf16/tokenizer.json filter=lfs diff=lfs merge=lfs -text

models/higgs-bf16/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

models/higgs-bf16/config.json

@@ -0,0 +1,112 @@
+{
+  "_hidden_size": 2560,
+  "_vocab_size": 151936,
+  "architectures": [
+    "HiggsMultimodalQwen3ForConditionalGeneration"
+  ],
+  "audio_encoder_config": {
+    "_name_or_path": "",
+    "architectures": null,
+    "chunk_size_feed_forward": 0,
+    "dtype": null,
+    "encoder_type": "discrete",
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "max_chunk_size": 50,
+    "mel_per_sample": 8,
+    "model_type": "higgs_audio_encoder",
+    "num_codebooks": 8,
+    "out_dim": 2560,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "problem_type": null,
+    "qwen3_aut_config": null,
+    "return_dict": true,
+    "tie_word_embeddings": true,
+    "use_delay_pattern": true,
+    "vocab_size": 1026,
+    "whisper_config": null
+  },
+  "audio_token_id": -100,
+  "ignore_index": -100,
+  "model_type": "higgs_multimodal_qwen3",
+  "text_config": {
+    "_name_or_path": "/ceph/models/Qwen3-4B-Base",
+    "architectures": [
+      "Qwen3ForCausalLM"
+    ],
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": 151643,
+    "dtype": "bfloat16",
+    "eos_token_id": 151643,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 2560,
+    "initializer_range": 0.02,
+    "intermediate_size": 9728,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 32768,
+    "max_window_layers": 36,
+    "model_type": "qwen3",
+    "num_attention_heads": 32,
+    "num_hidden_layers": 36,
+    "num_key_value_heads": 8,
+    "pad_token_id": null,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 1000000,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "transformers_version": "5.5.0"
+}

models/higgs-bf16/higgs_audio_v2_tokenizer_config.json

@@ -0,0 +1,77 @@
+{
+  "acoustic_model_config": {
+    "codebook_dim": 8,
+    "codebook_loss_weight": 1.0,
+    "codebook_size": 1024,
+    "commitment_loss_weight": 0.25,
+    "decoder_hidden_size": 1024,
+    "downsampling_ratios": [8, 5, 4, 2, 3],
+    "encoder_hidden_size": 64,
+    "hidden_size": 256,
+    "hop_length": 960,
+    "model_type": "dac",
+    "n_codebooks": 9,
+    "quantizer_dropout": 0,
+    "sampling_rate": 16000,
+    "upsampling_ratios": [8, 5, 4, 2, 3]
+  },
+  "architectures": ["HiggsAudioV2TokenizerModel"],
+  "block_dilations": [1, 1],
+  "channel_ratios": [1, 1],
+  "codebook_dim": 64,
+  "codebook_size": 1024,
+  "downsample_factor": 320,
+  "dtype": "float32",
+  "initializer_range": 0.02,
+  "kernel_size": 3,
+  "model_type": "higgs_audio_v2_tokenizer",
+  "sample_rate": 24000,
+  "semantic_model_config": {
+    "activation_dropout": 0.1,
+    "apply_spec_augment": true,
+    "attention_dropout": 0.1,
+    "bos_token_id": 1,
+    "classifier_proj_size": 256,
+    "conv_bias": false,
+    "conv_dim": [512, 512, 512, 512, 512, 512, 512],
+    "conv_kernel": [10, 3, 3, 3, 3, 2, 2],
+    "conv_pos_batch_norm": false,
+    "conv_stride": [5, 2, 2, 2, 2, 2, 2],
+    "ctc_loss_reduction": "sum",
+    "ctc_zero_infinity": false,
+    "do_stable_layer_norm": false,
+    "eos_token_id": 2,
+    "feat_extract_activation": "gelu",
+    "feat_extract_norm": "group",
+    "feat_proj_dropout": 0.0,
+    "feat_proj_layer_norm": true,
+    "final_dropout": 0.1,
+    "hidden_act": "gelu",
+    "hidden_dropout": 0.1,
+    "hidden_size": 768,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_norm_eps": 1e-05,
+    "layerdrop": 0.1,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.0,
+    "model_type": "hubert",
+    "num_attention_heads": 12,
+    "num_conv_pos_embedding_groups": 16,
+    "num_conv_pos_embeddings": 128,
+    "num_feat_extract_layers": 7,
+    "num_hidden_layers": 12,
+    "pad_token_id": 0,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 32
+  },
+  "semantic_sample_rate": 16000,
+  "strides": [1, 1],
+  "target_bandwidths": [0.5, 1, 1.5, 2],
+  "transformers_version": "5.3.0.dev0",
+  "unit_kernel_size": 3
+}

models/higgs-bf16/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb883de2de5adc5113f1f02b54830a0ea7cd6ef191cde65c41aceb3737d4d1c1
+size 11433924

models/higgs-bf16/tokenizer_config.json

@@ -0,0 +1,78 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|asr|>",
+    "<|streaming_asr|>",
+    "<|tts|>",
+    "<|streaming_tts|>",
+    "<|audio_cont_txt|>",
+    "<|audio|>",
+    "<|audio_end|>",
+    "<|text|>",
+    "<|text_end|>",
+    "<|eoc|>",
+    "<|user|>",
+    "<|assistant|>",
+    "<|system|>",
+    "<|await_audio|>",
+    "<|ref_audio|>",
+    "<|ref_text|>",
+    "<|emotion:elation|>",
+    "<|emotion:amusement|>",
+    "<|emotion:enthusiasm|>",
+    "<|emotion:determination|>",
+    "<|emotion:pride|>",
+    "<|emotion:contentment|>",
+    "<|emotion:affection|>",
+    "<|emotion:relief|>",
+    "<|emotion:contemplation|>",
+    "<|emotion:confusion|>",
+    "<|emotion:surprise|>",
+    "<|emotion:awe|>",
+    "<|emotion:longing|>",
+    "<|emotion:arousal|>",
+    "<|emotion:anger|>",
+    "<|emotion:fear|>",
+    "<|emotion:disgust|>",
+    "<|emotion:bitterness|>",
+    "<|emotion:sadness|>",
+    "<|emotion:shame|>",
+    "<|emotion:helplessness|>",
+    "<|env:music|>",
+    "<|env:noise|>",
+    "<|style:singing|>",
+    "<|style:shouting|>",
+    "<|style:whispering|>",
+    "<|sfx:cough|>",
+    "<|sfx:laughter|>",
+    "<|sfx:crying|>",
+    "<|sfx:screaming|>",
+    "<|sfx:burping|>",
+    "<|sfx:humming|>",
+    "<|sfx:sigh|>",
+    "<|sfx:sniff|>",
+    "<|sfx:sneeze|>",
+    "<|prosody:speed_very_slow|>",
+    "<|prosody:speed_slow|>",
+    "<|prosody:speed_fast|>",
+    "<|prosody:speed_very_fast|>",
+    "<|prosody:pitch_low|>",
+    "<|prosody:pitch_high|>",
+    "<|prosody:pause|>",
+    "<|prosody:long_pause|>",
+    "<|chatml|>",
+    "<|prosody:expressive_high|>",
+    "<|prosody:expressive_low|>"
+  ],
+  "is_local": true,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}