Upload MuseSparkForConditionalGeneration

chat_template.jinja

@@ -1,9 +1,13 @@
 {%- set tools = tools | default(none) -%}
 {%- set add_generation_prompt = add_generation_prompt | default(false) -%}
-{%- set reasoning_effort = reasoning_effort | default('medium') -%}
+{%- set reasoning_effort = reasoning_effort | default('xhigh', true) -%}
 {%- set cutoff_date = cutoff_date | default('2026-01-04') -%}
 {%- set current_date = current_date | default(strftime_now('%Y-%m-%d') if strftime_now is defined else none) -%}
-{%- set reasoning_effort_map = {'minimal': 1, 'low': 16, 'medium': 64, 'high': 256, 'xhigh': 512} -%}
+{%- set reasoning_effort_map = {'low': 8, 'medium': 32, 'high': 256, 'xhigh': 512} -%}
+{%- if not (reasoning_effort is string and reasoning_effort in reasoning_effort_map) -%}
+    {{- raise_exception('Unsupported reasoning_effort: ' ~ reasoning_effort ~ '. Expected one of: low, medium, high, xhigh.') -}}
+{%- endif -%}
+{%- set deferred_render_mode = deferred_render_mode | default('tool_name_and_description') -%}
 
 {#- Neutralise control sequences in caller-supplied text so it cannot be
     tokenized as structure. "<|" is broken because every special token has the
@@ -76,20 +80,67 @@
     {{- '</atem:invoke>\n</atem:function_calls>' -}}
 {%- endmacro -%}
 
+{#- One function line, as the training renderer prints it. Deferred: name plus description and annotation
+    (tool_name_and_description) or annotation only (tool_name_only). Full: "additionalProperties": false is added
+    when the schema lacks it, and empty parameters become an empty object schema. -#}
+{%- macro fn_line(name, fn, deferred) -%}
+    {%- set desc = fn.description if fn.description is defined and fn.description else '' -%}
+    {%- set p = fn.parameters if fn.parameters is defined and fn.parameters else {} -%}
+    {%- if deferred -%}
+        {%- set desc = ('' if deferred_render_mode == 'tool_name_only' else desc ~ ' ') ~ '[deferred: use tool_search to load full schema]' -%}
+        {{- '\n{"name": ' ~ esc(name | tojson) ~ ', "description": ' ~ esc(desc | tojson) ~ ', "defer_loading": true}' -}}
+    {%- else -%}
+        {%- if p is mapping and p | length == 0 -%}
+            {%- set p = '{"type": "object", "properties": {}, "additionalProperties": false}' -%}
+        {%- elif p is mapping and 'additionalProperties' not in p -%}
+            {%- set p = (p | tojson)[:-1] ~ ', "additionalProperties": false}' -%}
+        {%- else -%}
+            {%- set p = p | tojson -%}
+        {%- endif -%}
+        {{- '\n{"name": ' ~ esc(name | tojson) ~ ', "description": ' ~ esc(desc | tojson) ~ ', "parameters": ' ~ esc(p) ~ '}' -}}
+    {%- endif -%}
+{%- endmacro -%}
+
+{#- tools: OpenAI function tools, plus namespace tools {"type": "namespace", "name", "description", "defer_loading",
+    "tools": [function, ...]}. In a deferred namespace, functions without their own flag inherit the deferral; only
+    when some are true and some explicitly false do the explicitly eager ones keep their full schema. -#}
 {%- macro render_tool_defs(tools) -%}
     {{- 'In this environment you have access to a set of tools you can use to answer the user\'s question.\n\n' -}}
     {{- 'You can invoke a function by writing a "<atem:function_calls>" block like the following as part of your reply to the user:\n' -}}
     {{- '<atem:function_calls>\n<atem:invoke name="$FUNCTION_NAME">\n<atem:parameter name="$PARAMETER_NAME">$PARAMETER_VALUE</atem:parameter>\n...\n</atem:invoke>\n</atem:function_calls>\n\n' -}}
     {{- 'String and scalar parameters should be specified as is, while lists and objects should use JSON format. Note that spaces for string values are not stripped. The output is not expected to be valid XML and is parsed with regular expressions.\n' -}}
     {{- 'Here are the functions available in JSONSchema format:\n' -}}
-    {{- '// Tool metadata\n{"name": "", "description": ""}\n' -}}
-    {{- '// Function schemas' -}}
+    {%- set td = namespace(meta='', fns='', loose=false) -%}
     {%- for tool in tools -%}
-        {%- set fn = tool.function if tool.function is defined else tool -%}
-        {{- '\n{"name": ' -}}{{- fn.name | tojson -}}
-        {{- ', "description": ' -}}{{- (fn.description if fn.description is defined and fn.description else '') | tojson -}}
-        {{- ', "parameters": ' -}}{{- (fn.parameters if fn.parameters is defined and fn.parameters else {}) | tojson -}}{{- '}' -}}
+        {%- if tool.type is defined and tool.type == 'namespace' -%}
+            {%- set ns_deferred = tool.defer_loading is defined and tool.defer_loading -%}
+            {%- set ns_desc = tool.description if tool.description is defined and tool.description else '' -%}
+            {%- set members = tool.tools if tool.tools is defined and tool.tools else [] -%}
+            {%- set flags = namespace(t=false, f=false) -%}
+            {%- for m in members -%}
+                {%- set d = (m.function if m.function is defined else m).defer_loading -%}
+                {%- set flags.t = flags.t or d is not sameas false -%}
+                {%- set flags.f = flags.f or d is sameas false -%}
+            {%- endfor -%}
+            {%- if ns_deferred -%}
+                {%- set ns_desc = '[deferred: use tool_search to load descriptions and schemas]' if deferred_render_mode == 'tool_name_only' else (ns_desc ~ ' ' if ns_desc else '') ~ '[deferred: use tool_search to discover functions]' -%}
+            {%- endif -%}
+            {%- set td.meta = td.meta ~ '\n{"name": ' ~ esc(tool.name | tojson) ~ ', "description": ' ~ esc(ns_desc | tojson) ~ (', "has_deferred_functions": true}' if ns_deferred else '}') -%}
+            {%- for m in members -%}
+                {%- set mfn = m.function if m.function is defined else m -%}
+                {%- set eager = flags.t and flags.f and mfn.defer_loading is defined and mfn.defer_loading is sameas false -%}
+                {%- if not ns_deferred or eager or deferred_render_mode == 'tool_name_only' -%}
+                    {%- set td.fns = td.fns ~ fn_line(tool.name ~ '.' ~ mfn.name, mfn, (not eager) if ns_deferred else (mfn.defer_loading is defined and mfn.defer_loading)) -%}
+                {%- endif -%}
+            {%- endfor -%}
+        {%- else -%}
+            {%- set fn = tool.function if tool.function is defined else tool -%}
+            {%- set td.meta = td.meta ~ ('' if td.loose else '\n{"name": "", "description": ""}') -%}
+            {%- set td.loose = true -%}
+            {%- set td.fns = td.fns ~ fn_line(fn.name, fn, fn.defer_loading is defined and fn.defer_loading) -%}
+        {%- endif -%}
     {%- endfor -%}
+    {{- '// Tool metadata' ~ td.meta ~ '\n// Function schemas' ~ td.fns -}}
     {{- '\n\n' -}}
     {{- 'Here are examples of how to call functions in the tool set. To make parallel tool calls, emit each call as its own message (one tool call per message) in consecutive messages, as shown below:\n' -}}
     {{- '(If the tool namespace is not specified, invoke the function directly as `example_function_name_1` rather than `example_tool_name_1.example_function_name_1`)\n\n' -}}
@@ -112,9 +163,15 @@
     {%- set rns.recipients = rns.recipients + ['"commentary"'] -%}
     {{- '- "commentary": user-visible intermediate updates while the assistant will continue working, including messages sent to the user before or between tool calls.\n' -}}
     {%- if tools -%}
-        {%- set first_tool = tools | first -%}
-        {%- set first_fn = (first_tool.function if first_tool.function is defined else first_tool).name -%}
-        {%- set rns.recipients = rns.recipients + ['"[func_name] (e.g., ' ~ esc_header(first_fn) ~ ')"'] -%}
+        {%- set rt = namespace(first=none, ns=[]) -%}
+        {%- for tool in tools -%}
+            {%- if tool.type is defined and tool.type == 'namespace' -%}
+                {%- set rt.ns = rt.ns + ['"' ~ esc_header(tool.name) ~ '.*"'] -%}
+            {%- elif rt.first is none -%}
+                {%- set rt.first = (tool.function if tool.function is defined else tool).name -%}
+            {%- endif -%}
+        {%- endfor -%}
+        {%- set rns.recipients = rns.recipients + (['"[func_name] (e.g., ' ~ esc_header(rt.first) ~ ')"'] if rt.first is not none else []) + rt.ns -%}
         {{- '- "tool" or "[tool name]": the name of the tool to call.\n' -}}
     {%- endif -%}
     {%- set rns.recipients = rns.recipients + ['"user"'] -%}
@@ -142,7 +199,7 @@
 {%- if tools -%}
     {%- set sys.parts = sys.parts + [render_tool_defs(tools) | trim] -%}
 {%- endif -%}
-{%- if reasoning_effort in reasoning_effort_map and 'reasoning strength:' not in ((ns.instructions if ns.instructions else '') | lower) -%}
+{%- if 'reasoning strength:' not in ((ns.instructions if ns.instructions else '') | lower) -%}
     {%- set sys.parts = sys.parts + ['Reasoning strength: ' ~ reasoning_effort_map[reasoning_effort] ~ '.'] -%}
 {%- endif -%}
 {%- set sys.parts = sys.parts + [render_valid_recipients(tools) | trim] -%}

config.json

@@ -19,7 +19,7 @@
     ],
     "initializer_range": 0.02,
     "mlp_ratio": 2.0,
-    "model_type": "musespark1x_omni_audio",
+    "model_type": "muse_spark_audio",
     "num_hidden_layers": 4,
     "out_hidden_size": 64,
     "patch_sizes": [
@@ -33,7 +33,7 @@
   "audio_token_id": 200038,
   "dtype": "bfloat16",
   "image_token_id": 200092,
-  "model_type": "musespark1x_omni",
+  "model_type": "muse_spark",
   "out_hidden_size": 128,
   "projector_hidden_act": "gelu",
   "projector_hidden_size": 32,
@@ -121,7 +121,7 @@
       "sparse",
       "sparse"
     ],
-    "model_type": "musespark1x_omni_text",
+    "model_type": "muse_spark_text",
     "moe_hidden_size": 16,
     "moe_intermediate_size": 32,
     "num_attention_heads": 4,
@@ -131,6 +131,7 @@
     "num_local_experts": 16,
     "num_shared_experts": 0,
     "output_multiplier": 0.17677669529663687,
+    "output_router_logits": false,
     "pad_token_id": 200018,
     "post_norm_eps": 1e-08,
     "qk_scale_factor": 5.4730064863875,
@@ -146,7 +147,7 @@
     "use_cache": true,
     "vocab_size": 202048
   },
-  "transformers_version": "5.18.0.dev0",
+  "transformers_version": "5.19.0.dev0",
   "video_token_id": 200091,
   "vision_config": {
     "hidden_act": "gelu",
@@ -159,7 +160,7 @@
     ],
     "max_position_embeddings": 1024,
     "merge_size": 2,
-    "model_type": "musespark1x_omni_vision",
+    "model_type": "muse_spark_vision",
     "num_attention_heads": 4,
     "num_hidden_layers": 2,
     "patch_size": 14,

generation_config.json

@@ -9,6 +9,6 @@
   "pad_token_id": 200018,
   "temperature": 1.0,
   "top_k": 64,
-  "top_p": 0.95,
-  "transformers_version": "5.18.0.dev0"
+  "top_p": 1,
+  "transformers_version": "5.19.0.dev0"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0eca1a0351ae70d60e5c7b1031984afbb33f61c1aef279b139b417a45194f6b0
+oid sha256:10f92659782580ffd54363ad540da89f3203a176879a15ec905d125c47f95356
 size 52154128

processor_config.json

@@ -39,6 +39,7 @@
     "do_resize": true,
     "do_sample_frames": true,
     "fps": 2.0,
+    "frame_timestamp_format": "Time: {timestamp:.2f}s",
     "image_mean": [
       0.5,
       0.5,
@@ -49,9 +50,9 @@
       0.5,
       0.5
     ],
-    "max_video_frame_tokens": 144,
+    "max_video_frame_tokens": 256,
     "merge_size": 2,
-    "num_frames": 96,
+    "num_frames": 600,
     "patch_size": 14,
     "resample": 1,
     "rescale_factor": 0.00392156862745098,

tokenizer_config.json

@@ -3,7 +3,7 @@
   "bos_token": "<|begin_of_text|>",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|end_of_text|>",
-  "is_local": true,
+  "is_local": false,
   "local_files_only": false,
   "model_max_length": 1048576,
   "pad_token": "<|finetune_right_pad|>",