Upload MuseSparkForConditionalGeneration

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,9 @@
+---
+library_name: transformers
+tags:
+- trl
+---
+
+# Tiny MuseSparkForConditionalGeneration
+
+This is a minimal model built for unit tests in the [TRL](https://github.com/huggingface/trl) library.

chat_template.jinja

@@ -0,0 +1,227 @@
+{%- set tools = tools | default(none) -%}
+{%- set add_generation_prompt = add_generation_prompt | default(false) -%}
+{%- set reasoning_effort = reasoning_effort | default('medium') -%}
+{%- set cutoff_date = cutoff_date | default('2026-01-04') -%}
+{%- set current_date = current_date | default(strftime_now('%Y-%m-%d') if strftime_now is defined else none) -%}
+{%- set reasoning_effort_map = {'minimal': 1, 'low': 16, 'medium': 64, 'high': 256, 'xhigh': 512} -%}
+
+{#- Neutralise control sequences in caller-supplied text so it cannot be
+    tokenized as structure. "<|" is broken because every special token has the
+    form "<|...|>"; the bare media placeholders are broken because the
+    multimodal processor rewrites them positionally and a literal one in text
+    would desynchronise placeholders from the attached media. -#}
+{%- macro esc(value) -%}
+    {{- value | string
+        | replace('<|', '< |')
+        | replace('<image>', '< image>')
+        | replace('<video>', '< video>')
+        | replace('<audio>', '< audio>') -}}
+{%- endmacro -%}
+
+{%- macro esc_header(value) -%}
+    {{- esc(value) | replace('\r', ' ') | replace('\n', ' ') -}}
+{%- endmacro -%}
+
+{%- macro esc_attr(value) -%}
+    {{- esc_header(value)
+        | replace('&', '&amp;')
+        | replace('"', '&quot;')
+        | replace("'", '&#39;')
+        | replace('<', '&lt;')
+        | replace('>', '&gt;') -}}
+{%- endmacro -%}
+
+{%- macro render_content(content) -%}
+    {%- if content is string -%}
+        {{- esc(content) -}}
+    {%- elif content is not none -%}
+        {%- for part in content -%}
+            {%- set part_type = part.get('type', '') -%}
+            {%- if part_type in ['image', 'input_image', 'image_url'] -%}
+                {{- '<|patch|>' -}}
+            {%- elif part_type in ['video', 'input_video', 'video_url'] -%}
+                {{- '<|video|>' -}}
+            {%- elif part_type in ['audio', 'input_audio', 'audio_url'] -%}
+                {{- '<|audio|>' -}}
+            {%- elif part_type in ['text', 'input_text', 'output_text'] -%}
+                {{- esc(part.get('text', '')) -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro render_atem(tc) -%}
+    {%- set args = tc.function.arguments -%}
+    {%- if args is not mapping -%}
+        {{- raise_exception('This chat template requires tool_call.function.arguments to be a dict (mapping). The jinja sandbox has no JSON parser, so deserialize the arguments string before calling apply_chat_template.') -}}
+    {%- endif -%}
+    {{- '<atem:function_calls>\n<atem:invoke name="' -}}{{- esc_attr(tc.function.name) -}}{{- '">\n' -}}
+    {%- for k, v in args.items() -%}
+        {{- '<atem:parameter name="' -}}{{- esc_attr(k) -}}{{- '">' -}}
+        {%- if v is boolean -%}
+            {%- if v -%}
+                true
+            {%- else -%}
+                false
+            {%- endif -%}
+        {%- elif v is none -%}
+            null
+        {%- elif v is mapping or (v is iterable and v is not string) -%}
+            {{- esc(v | tojson) -}}
+        {%- else -%}
+            {{- esc(v) -}}
+        {%- endif -%}
+        {{- '</atem:parameter>\n' -}}
+    {%- endfor -%}
+    {{- '</atem:invoke>\n</atem:function_calls>' -}}
+{%- endmacro -%}
+
+{%- macro render_tool_defs(tools) -%}
+    {{- 'In this environment you have access to a set of tools you can use to answer the user\'s question.\n\n' -}}
+    {{- 'You can invoke a function by writing a "<atem:function_calls>" block like the following as part of your reply to the user:\n' -}}
+    {{- '<atem:function_calls>\n<atem:invoke name="$FUNCTION_NAME">\n<atem:parameter name="$PARAMETER_NAME">$PARAMETER_VALUE</atem:parameter>\n...\n</atem:invoke>\n</atem:function_calls>\n\n' -}}
+    {{- 'String and scalar parameters should be specified as is, while lists and objects should use JSON format. Note that spaces for string values are not stripped. The output is not expected to be valid XML and is parsed with regular expressions.\n' -}}
+    {{- 'Here are the functions available in JSONSchema format:\n' -}}
+    {{- '// Tool metadata\n{"name": "", "description": ""}\n' -}}
+    {{- '// Function schemas' -}}
+    {%- for tool in tools -%}
+        {%- set fn = tool.function if tool.function is defined else tool -%}
+        {{- '\n{"name": ' -}}{{- fn.name | tojson -}}
+        {{- ', "description": ' -}}{{- (fn.description if fn.description is defined and fn.description else '') | tojson -}}
+        {{- ', "parameters": ' -}}{{- (fn.parameters if fn.parameters is defined and fn.parameters else {}) | tojson -}}{{- '}' -}}
+    {%- endfor -%}
+    {{- '\n\n' -}}
+    {{- 'Here are examples of how to call functions in the tool set. To make parallel tool calls, emit each call as its own message (one tool call per message) in consecutive messages, as shown below:\n' -}}
+    {{- '(If the tool namespace is not specified, invoke the function directly as `example_function_name_1` rather than `example_tool_name_1.example_function_name_1`)\n\n' -}}
+    {{- 'to=example_tool_name_1.example_function_name_1\n\n' -}}
+    {{- '<atem:function_calls>\n<atem:invoke name="example_tool_name_1.example_function_name_1">\n' -}}
+    {{- '<atem:parameter name="example_parameter_1">value_1</atem:parameter>\n' -}}
+    {{- '</atem:invoke>\n</atem:function_calls>\n\n' -}}
+    {{- 'to=example_tool_name_2.example_function_name_2\n\n' -}}
+    {{- '<atem:function_calls>\n<atem:invoke name="example_tool_name_2.example_function_name_2">\n' -}}
+    {{- '<atem:parameter name="example_parameter_2">This is the value for the second parameter\nthat can span\n"multiple" lines\n</atem:parameter>\n' -}}
+    {{- '<atem:parameter name="example_parameter_3">value_3</atem:parameter>\n' -}}
+    {{- '</atem:invoke>\n</atem:function_calls>' -}}
+{%- endmacro -%}
+
+{%- macro render_valid_recipients(tools) -%}
+    {%- set rns = namespace(recipients=[]) -%}
+    {{- 'Use the appropriate recipient for each message:\n\n' -}}
+    {%- set rns.recipients = rns.recipients + ['"self"'] -%}
+    {{- '- "self": private reasoning and tool planning.\n' -}}
+    {%- set rns.recipients = rns.recipients + ['"commentary"'] -%}
+    {{- '- "commentary": user-visible intermediate updates while the assistant will continue working, including messages sent to the user before or between tool calls.\n' -}}
+    {%- if tools -%}
+        {%- set first_tool = tools | first -%}
+        {%- set first_fn = (first_tool.function if first_tool.function is defined else first_tool).name -%}
+        {%- set rns.recipients = rns.recipients + ['"[func_name] (e.g., ' ~ esc_header(first_fn) ~ ')"'] -%}
+        {{- '- "tool" or "[tool name]": the name of the tool to call.\n' -}}
+    {%- endif -%}
+    {%- set rns.recipients = rns.recipients + ['"user"'] -%}
+    {{- '- "user": messages that end the assistant turn, such as a completed response or clarification question that waits for the user\'s reply; do not use for pre-tool updates or partial responses.\n' -}}
+    {{- '\n# Valid recipients: ' ~ rns.recipients | join(', ') ~ '.' -}}
+{%- endmacro -%}
+
+{{- bos_token -}}
+{%- set ns = namespace(instructions=none, rest=messages) -%}
+{%- if messages and messages[0]['role'] in ['system', 'developer'] -%}
+    {%- set ns.instructions = render_content(messages[0]['content'])
+          | replace('Reasoning effort', 'Reasoning strength')
+          | replace('Reasoning Effort', 'Reasoning Strength')
+          | replace('reasoning effort', 'reasoning strength')
+          | replace('REASONING EFFORT', 'REASONING STRENGTH') -%}
+    {%- set ns.rest = messages[1:] -%}
+{%- endif -%}
+{%- set sys = namespace(parts=['You are a helpful AI assistant.', 'Knowledge cutoff: ' ~ cutoff_date ~ '.']) -%}
+{%- if current_date -%}
+    {%- set sys.parts = sys.parts + ['Current date: ' ~ current_date ~ '.'] -%}
+{%- endif -%}
+{%- if ns.instructions -%}
+    {%- set sys.parts = sys.parts + [ns.instructions] -%}
+{%- endif -%}
+{%- if tools -%}
+    {%- set sys.parts = sys.parts + [render_tool_defs(tools) | trim] -%}
+{%- endif -%}
+{%- if reasoning_effort in reasoning_effort_map and 'reasoning strength:' not in ((ns.instructions if ns.instructions else '') | lower) -%}
+    {%- set sys.parts = sys.parts + ['Reasoning strength: ' ~ reasoning_effort_map[reasoning_effort] ~ '.'] -%}
+{%- endif -%}
+{%- set sys.parts = sys.parts + [render_valid_recipients(tools) | trim] -%}
+{{- '<|start|>system<|message|>' ~ sys.parts | join('\n\n') ~ '<|eot|>' -}}
+{%- for message in ns.rest -%}
+    {%- set role = message['role'] -%}
+    {%- set end_token = '<|eom|>' if (not loop.last and ns.rest[loop.index0 + 1]['role'] == role) else '<|eot|>' -%}
+    {%- if role in ['system', 'developer'] -%}
+        {{- '<|start|>' + role + '<|message|>' -}}
+        {{- render_content(message['content']) -}}
+        {{- '<|eot|>' -}}
+    {%- elif role == 'user' -%}
+        {{- '<|start|>user<|message|>' -}}
+        {%- if message.get('render_timestamp') and message.get('timestamp') -%}
+            {%- set ts = message['timestamp'] -%}
+            {%- if ts is string -%}
+                {{- '[' ~ esc_header(ts) ~ '] ' -}}
+            {%- elif ts.strftime is defined -%}
+                {{- '[' ~ (ts.strftime('%a %Y-%m-%d %H:%M %Z') | trim) ~ '] ' -}}
+            {%- endif -%}
+        {%- endif -%}
+        {{- render_content(message['content']) -}}
+        {{- '<|eot|>' -}}
+    {%- elif role == 'tool' -%}
+        {%- set tname = message.get('name') -%}
+        {%- if not tname -%}
+            {%- set tcid = message.get('tool_call_id') -%}
+            {%- set tns = namespace(name=tcid if tcid else '') -%}
+            {%- for m in messages -%}
+                {%- if m.get('tool_calls') -%}
+                    {%- for tc in m['tool_calls'] -%}
+                        {%- if tcid is not none and tc.id is defined and tc.id == tcid -%}
+                            {%- set tns.name = tc.function.name -%}
+                        {%- endif -%}
+                    {%- endfor -%}
+                {%- endif -%}
+            {%- endfor -%}
+            {%- set tname = tns.name -%}
+        {%- endif -%}
+        {%- if not tname -%}
+            {{- raise_exception('tool message has no name and no matching tool_call_id') -}}
+        {%- endif -%}
+        {{- '<|start|>tool ' -}}{{- esc_header(tname) -}}{{- '<|message|><tool_output name="' -}}{{- esc_attr(tname) -}}{{- '">\n' -}}
+        {{- render_content(message['content']) -}}
+        {{- '\n</tool_output><|eot|>' -}}
+    {%- elif role == 'assistant' -%}
+        {%- generation %}
+        {%- set reasoning = message.get('reasoning_content') or message.get('reasoning') -%}
+        {%- if reasoning -%}
+            {{- '<|start|>assistant to=self<|message|>' -}}{{- esc(reasoning) -}}{{- '<|eom|>' -}}
+        {%- endif -%}
+        {%- set body = render_content(message.get('content')) -%}
+        {%- if message.get('tool_calls') -%}
+            {%- if body -%}
+                {{- '<|start|>assistant to=commentary<|message|>' + body + '<|eom|>' -}}
+            {%- endif -%}
+            {%- for tc in message['tool_calls'] -%}
+                {{- '<|start|>assistant to=' -}}{{- esc_header(tc.function.name) -}}{{- '<|message|>' -}}
+                {{- render_atem(tc) -}}
+                {{- (end_token if loop.last else '<|eom|>') -}}
+            {%- endfor -%}
+        {%- elif body or not reasoning -%}
+            {%- set recipient = message.get('recipient') or 'user' -%}
+            {%- if recipient == 'user' -%}
+                {%- set end_turn = true -%}
+            {%- elif recipient in ['self', 'commentary'] -%}
+                {%- set end_turn = false -%}
+            {%- elif message.get('end_turn') is not none -%}
+                {%- set end_turn = message.get('end_turn') -%}
+            {%- else -%}
+                {%- set end_turn = false -%}
+            {%- endif -%}
+            {{- '<|start|>assistant to=' -}}{{- esc_header(recipient) -}}{{- '<|message|>' -}}
+            {{- body -}}
+            {{- ('<|eot|>' if end_turn else '<|eom|>') -}}
+        {%- endif -%}
+        {%- endgeneration %}
+    {%- endif -%}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{- '<|start|>assistant' -}}
+{%- endif -%}

config.json

@@ -0,0 +1,174 @@
+{
+  "architectures": [
+    "MuseSparkForConditionalGeneration"
+  ],
+  "audio_config": {
+    "conv_kernel_size": 3,
+    "depths": [
+      1,
+      1,
+      1,
+      1
+    ],
+    "hidden_act": "gelu",
+    "hidden_sizes": [
+      16,
+      16,
+      32,
+      32
+    ],
+    "initializer_range": 0.02,
+    "mlp_ratio": 2.0,
+    "model_type": "musespark1x_omni_audio",
+    "num_hidden_layers": 4,
+    "out_hidden_size": 64,
+    "patch_sizes": [
+      16,
+      6,
+      5,
+      4
+    ],
+    "rms_norm_eps": 1e-06
+  },
+  "audio_token_id": 200038,
+  "dtype": "bfloat16",
+  "image_token_id": 200092,
+  "model_type": "musespark1x_omni",
+  "out_hidden_size": 128,
+  "projector_hidden_act": "gelu",
+  "projector_hidden_size": 32,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": 200000,
+    "eos_token_id": 200001,
+    "final_logit_softcapping": 20.0,
+    "head_dim": 16,
+    "hidden_act": "silu",
+    "hidden_size": 64,
+    "initializer_range": 0.02,
+    "intermediate_size": 32,
+    "layer_rope_theta": [
+      500000.0,
+      0,
+      500000.0,
+      500000.0,
+      500000.0,
+      0,
+      500000.0,
+      500000.0,
+      500000.0,
+      0,
+      500000.0,
+      500000.0,
+      500000.0,
+      0,
+      500000.0,
+      500000.0,
+      500000.0,
+      0,
+      500000.0,
+      500000.0,
+      500000.0,
+      0,
+      500000.0,
+      500000.0,
+      500000.0,
+      0,
+      500000.0,
+      500000.0,
+      500000.0,
+      0,
+      500000.0,
+      500000.0,
+      500000.0,
+      0,
+      500000.0,
+      500000.0,
+      500000.0,
+      0,
+      500000.0,
+      500000.0,
+      500000.0,
+      0,
+      500000.0,
+      500000.0,
+      500000.0,
+      0,
+      500000.0,
+      500000.0,
+      500000.0,
+      0,
+      500000.0,
+      500000.0,
+      500000.0,
+      0,
+      500000.0,
+      500000.0,
+      500000.0,
+      0,
+      500000.0,
+      500000.0,
+      500000.0,
+      0
+    ],
+    "layer_types": [
+      "full_attention",
+      "sliding_attention"
+    ],
+    "max_position_embeddings": 1048576,
+    "mlp_layer_types": [
+      "sparse",
+      "sparse"
+    ],
+    "model_type": "musespark1x_omni_text",
+    "moe_hidden_size": 16,
+    "moe_intermediate_size": 32,
+    "num_attention_heads": 4,
+    "num_experts_per_tok": 4,
+    "num_hidden_layers": 2,
+    "num_key_value_heads": 2,
+    "num_local_experts": 16,
+    "num_shared_experts": 0,
+    "output_multiplier": 0.17677669529663687,
+    "pad_token_id": 200018,
+    "post_norm_eps": 1e-08,
+    "qk_scale_factor": 5.4730064863875,
+    "residual_gate_temperature": 0.3,
+    "rms_norm_eps": 1e-05,
+    "rope_parameters": {
+      "rope_theta": 500000.0,
+      "rope_type": "default"
+    },
+    "routed_scaling_factor": 1.0,
+    "sliding_window": 32,
+    "tie_word_embeddings": false,
+    "use_cache": true,
+    "vocab_size": 202048
+  },
+  "transformers_version": "5.18.0.dev0",
+  "video_token_id": 200091,
+  "vision_config": {
+    "hidden_act": "gelu",
+    "hidden_size": 32,
+    "intermediate_size": 64,
+    "layer_norm_eps": 1e-05,
+    "layer_types": [
+      "window_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 1024,
+    "merge_size": 2,
+    "model_type": "musespark1x_omni_vision",
+    "num_attention_heads": 4,
+    "num_hidden_layers": 2,
+    "patch_size": 14,
+    "patch_temporal": 2,
+    "pos_emb_height": 32,
+    "pos_emb_width": 32,
+    "rope_parameters": {
+      "rope_theta": 10000,
+      "rope_type": "axial"
+    }
+  }
+}

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 200000,
+  "do_sample": true,
+  "eos_token_id": [
+    200001,
+    200008
+  ],
+  "max_length": 1048576,
+  "pad_token_id": 200018,
+  "temperature": 1.0,
+  "top_k": 64,
+  "top_p": 0.95,
+  "transformers_version": "5.18.0.dev0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80e75d8cd917fd5d886d934975bd1505c7f03c2bad7da6ef823d6e7f9923281d
+size 52150008

processor_config.json

@@ -0,0 +1,62 @@
+{
+  "feature_extractor": {
+    "feature_extractor_type": "MuseSparkFeatureExtractor",
+    "feature_size": 1,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "return_attention_mask": true,
+    "samples_per_token": 1920,
+    "sampling_rate": 24000
+  },
+  "image_processor": {
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_processor_type": "MuseGlimmerImageProcessor",
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "max_image_tokens": 4096,
+    "merge_size": 2,
+    "patch_size": 14,
+    "resample": 1,
+    "rescale_factor": 0.00392156862745098,
+    "temporal_patch_size": 2
+  },
+  "processor_class": "MuseSparkProcessor",
+  "video_processor": {
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "do_sample_frames": true,
+    "fps": 2.0,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "max_video_frame_tokens": 144,
+    "merge_size": 2,
+    "num_frames": 96,
+    "patch_size": 14,
+    "resample": 1,
+    "rescale_factor": 0.00392156862745098,
+    "return_metadata": true,
+    "temporal_patch_size": 2,
+    "video_processor_type": "MuseGlimmerVideoProcessor"
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67fd0540977f672f1c139c0b82b9342bec1e07349e6d1a89c4331496cbab7238
+size 28086030

tokenizer_config.json

@@ -0,0 +1,57 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|end_of_text|>",
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 1048576,
+  "pad_token": "<|finetune_right_pad|>",
+  "processor_class": "MuseSparkProcessor",
+  "response_template": {
+    "defaults": {
+      "role": "assistant"
+    },
+    "fields": {
+      "content": {
+        "close": [
+          "<|eot|>",
+          "<|eom|>"
+        ],
+        "content": "text",
+        "join": "",
+        "open_pattern": "to=(?:user|commentary)<\\|message\\|>",
+        "repeats": true
+      },
+      "reasoning_content": {
+        "close": "<|eom|>",
+        "content": "text",
+        "open_pattern": "to=self<\\|message\\|>"
+      },
+      "tool_calls": {
+        "close": "</atem:invoke>",
+        "content": "xml-inline",
+        "content_args": {
+          "tag_pattern": "<atem:parameter\\b[^>]*?\\bname=\"(?P<key>[^\"]+)\"[^>]*?>(?P<value>.*?)</atem:parameter>",
+          "value_parser": {
+            "args": {
+              "allow_non_json": true
+            },
+            "name": "json"
+          }
+        },
+        "open_pattern": "<atem:invoke\\b[^>]*?\\bname=\"(?P<name>[^\"]+)\">",
+        "repeats": true,
+        "transform": {
+          "function": {
+            "arguments": "{content}",
+            "name": "{name}"
+          },
+          "type": "function"
+        }
+      }
+    },
+    "start_anchor": "<|start|>assistant"
+  },
+  "tokenizer_class": "TokenizersBackend"
+}