| { |
| "type": "LanguageModelConfig", |
| "token_codec_config": { |
| "type": "ChatCodecConfig", |
| "prompt_template": "{{- bos_token -}}\n{%- set preserve_thinking = preserve_thinking | default(keep_past_thinking | default(false)) -%}\n\n{%- macro format_arg_value(arg_value) -%}\n {%- if arg_value is string -%}\n {{- \"'\" + (arg_value | replace(\"\\\\\", \"\\\\\\\\\") | replace(\"'\", \"\\\\'\") | replace(\"\\n\", \"\\\\n\") | replace(\"\\r\", \"\\\\r\")) + \"'\" -}}\n {%- elif arg_value is mapping or arg_value is iterable -%}\n {{- arg_value | tojson -}}\n {%- else -%}\n {{- arg_value | string -}}\n {%- endif -%}\n{%- endmacro -%}\n\n{%- macro parse_content(content) -%}\n {%- if content is string -%}\n {{- content -}}\n {%- elif content is mapping -%}\n {{- content | tojson -}}\n {%- elif content is iterable -%}\n {%- set _ns = namespace(result=\"\") -%}\n {%- for item in content -%}\n {%- if item is string -%}\n {%- set _ns.result = _ns.result + item -%}\n {%- elif item is mapping and item.get(\"type\") == \"image\" -%}\n {%- set _ns.result = _ns.result + \"<image>\" -%}\n {%- elif item is mapping and item.get(\"type\") == \"text\" -%}\n {%- set _ns.result = _ns.result + ((item.get(\"text\") or \"\") | string) -%}\n {%- else -%}\n {%- set _ns.result = _ns.result + (item | tojson) -%}\n {%- endif -%}\n {%- endfor -%}\n {{- _ns.result -}}\n {%- endif -%}\n{%- endmacro -%}\n\n{%- macro render_tool_calls(tool_calls) -%}\n {%- set tool_calls_ns = namespace(tool_calls=[]) -%}\n {%- for tool_call in tool_calls -%}\n {%- set func = tool_call[\"function\"] if \"function\" in tool_call else tool_call -%}\n {%- set func_name = func[\"name\"] -%}\n {%- set func_args = func.get(\"arguments\") -%}\n {%- set args_ns = namespace(arg_strings=[]) -%}\n {%- if func_args is mapping -%}\n {%- for arg_name, arg_value in func_args.items() -%}\n {%- set args_ns.arg_strings = args_ns.arg_strings + [arg_name + \"=\" + format_arg_value(arg_value)] -%}\n {%- endfor -%}\n {%- elif func_args is string and (func_args | trim) not in [\"\", \"{}\", \"null\"] -%}\n {{- raise_exception(\"Tool call arguments must be a mapping, got a JSON-encoded string: parse arguments with json.loads() before applying the chat template\") -}}\n {%- endif -%}\n {%- set tool_calls_ns.tool_calls = tool_calls_ns.tool_calls + [func_name + \"(\" + (args_ns.arg_strings | join(\", \")) + \")\"] -%}\n {%- endfor -%}\n {{- \"<|tool_call_start|>[\" + (tool_calls_ns.tool_calls | join(\", \")) + \"]<|tool_call_end|>\" -}}\n{%- endmacro -%}\n\n{%- set ns = namespace(system_prompt=\"\", last_assistant_index=-1) -%}\n{%- if messages and messages[0][\"role\"] == \"system\" -%}\n {%- if messages[0].get(\"content\") -%}\n {%- set ns.system_prompt = parse_content(messages[0][\"content\"]) -%}\n {%- endif -%}\n {%- set messages = messages[1:] -%}\n{%- endif -%}\n{%- if tools -%}\n {%- set ns.system_prompt = ns.system_prompt + (\"\\n\" if ns.system_prompt else \"\") + \"List of tools: [\" -%}\n {%- for tool in tools -%}\n {%- if tool is not string -%}\n {%- set tool = tool | tojson -%}\n {%- endif -%}\n {%- set ns.system_prompt = ns.system_prompt + tool -%}\n {%- if not loop.last -%}\n {%- set ns.system_prompt = ns.system_prompt + \", \" -%}\n {%- endif -%}\n {%- endfor -%}\n {%- set ns.system_prompt = ns.system_prompt + \"]\" -%}\n{%- endif -%}\n{%- if ns.system_prompt -%}\n {{- \"<|im_start|>system\\n\" + ns.system_prompt + \"<|im_end|>\\n\" -}}\n{%- endif -%}\n{%- for message in messages -%}\n {%- if message[\"role\"] == \"assistant\" -%}\n {%- set ns.last_assistant_index = loop.index0 -%}\n {%- endif -%}\n{%- endfor -%}\n{%- for message in messages -%}\n {{- \"<|im_start|>\" + message.role + \"\\n\" -}}\n {%- if message.role == \"assistant\" -%}\n \n {%- set keep_thinking = preserve_thinking or loop.index0 == ns.last_assistant_index -%}\n {%- set thinking = message.thinking or message.reasoning or message.reasoning_content -%}\n {%- set thinking = thinking if thinking is string else \"\" -%}\n {%- if thinking and keep_thinking -%}\n {{- \"<think>\" + thinking + \"</think>\" -}}\n {%- endif -%}\n {%- set _cfm_tag = \"CONTINUE_FINAL_MESSAGE_TAG \" -%}\n {%- set _has_cfm = false -%}\n {%- set content = \"\" -%}\n {%- if message.get(\"content\") -%}\n {%- set content = parse_content(message.content) -%}\n {%- endif -%}\n {%- if not keep_thinking and \"</think>\" in content -%}\n {%- set content = content.split(\"</think>\")[-1] | trim -%}\n {%- endif -%}\n {%- if content.endswith(_cfm_tag) -%}\n {%- set _has_cfm = true -%}\n {%- set _trunc_len = (content | length) - (_cfm_tag | length) -%}\n {%- set content = content[:_trunc_len] -%}\n {%- endif -%}\n {{- content -}}\n {%- if message.tool_calls -%}\n {{- render_tool_calls(message.tool_calls) -}}\n {%- endif -%}\n {%- if _has_cfm -%}\n {{- _cfm_tag -}}\n {%- endif -%}\n {{- \"<|im_end|>\\n\" -}}\n \n {%- else %}\n {%- if message.get(\"content\") -%}\n {{- parse_content(message[\"content\"]) -}}\n {%- endif -%}\n {{- \"<|im_end|>\\n\" -}}\n {%- endif %}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{- \"<|im_start|>assistant\\n\" -}}\n{%- endif -%}\n", |
| "output_parser_regex": null, |
| "system_role_name": "system", |
| "user_role_name": "user", |
| "assistant_role_name": "assistant", |
| "eos_token": "<|im_end|>", |
| "bos_token": "<|startoftext|>", |
| "end_of_thinking_tag": "\n</think>", |
| "default_system_prompt": null |
| }, |
| "decoder_config": { |
| "embedding_config": { |
| "type": "TiedEmbeddingConfig", |
| "input_scale": null, |
| "logit_soft_cap": null, |
| "logit_scale": null |
| }, |
| "transformer_config": { |
| "layer_configs": [ |
| { |
| "pre_mixer_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mixer_config": { |
| "type": "ShortConvConfig", |
| "in_projection_config": {}, |
| "conv_config": { |
| "has_biases": false |
| }, |
| "out_projection_config": {}, |
| "kernel_size": 3 |
| }, |
| "post_mixer_norm_config": null, |
| "pre_mlp_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mlp_config": { |
| "type": "DenseMLPConfig", |
| "linear_config": {}, |
| "activation": { |
| "type": "SiLU", |
| "alpha": 1.0 |
| }, |
| "has_up_biases": false, |
| "has_down_biases": false, |
| "gate_clipping": null, |
| "up_clipping": null |
| }, |
| "post_mlp_norm_config": null, |
| "hidden_dim": null, |
| "ple_config": null, |
| "has_post_layer_scalar": false, |
| "kv_source_layer_index": null, |
| "rope_config": null |
| }, |
| { |
| "pre_mixer_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mixer_config": { |
| "type": "ShortConvConfig", |
| "in_projection_config": {}, |
| "conv_config": { |
| "has_biases": false |
| }, |
| "out_projection_config": {}, |
| "kernel_size": 3 |
| }, |
| "post_mixer_norm_config": null, |
| "pre_mlp_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mlp_config": { |
| "type": "DenseMLPConfig", |
| "linear_config": {}, |
| "activation": { |
| "type": "SiLU", |
| "alpha": 1.0 |
| }, |
| "has_up_biases": false, |
| "has_down_biases": false, |
| "gate_clipping": null, |
| "up_clipping": null |
| }, |
| "post_mlp_norm_config": null, |
| "hidden_dim": null, |
| "ple_config": null, |
| "has_post_layer_scalar": false, |
| "kv_source_layer_index": null, |
| "rope_config": null |
| }, |
| { |
| "pre_mixer_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mixer_config": { |
| "type": "AttentionConfig", |
| "out_projection_config": {}, |
| "query_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "key_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "num_heads": 32, |
| "num_groups": 8, |
| "head_dim": 64, |
| "is_causal": true, |
| "scale": null, |
| "sliding_window_size": null, |
| "logit_soft_cap": null, |
| "has_sinks": false, |
| "has_out_biases": false, |
| "normalize_values": false, |
| "is_kv_sharing": false, |
| "qkv_projection_config": {}, |
| "has_qkv_biases": false, |
| "gate_projection_config": null |
| }, |
| "post_mixer_norm_config": null, |
| "pre_mlp_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mlp_config": { |
| "type": "DenseMLPConfig", |
| "linear_config": {}, |
| "activation": { |
| "type": "SiLU", |
| "alpha": 1.0 |
| }, |
| "has_up_biases": false, |
| "has_down_biases": false, |
| "gate_clipping": null, |
| "up_clipping": null |
| }, |
| "post_mlp_norm_config": null, |
| "hidden_dim": null, |
| "ple_config": null, |
| "has_post_layer_scalar": false, |
| "kv_source_layer_index": null, |
| "rope_config": { |
| "type": "UnscaledRoPEConfig", |
| "base": 1000000.0, |
| "max_sequence_length": 128000, |
| "head_dim": 64 |
| } |
| }, |
| { |
| "pre_mixer_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mixer_config": { |
| "type": "ShortConvConfig", |
| "in_projection_config": {}, |
| "conv_config": { |
| "has_biases": false |
| }, |
| "out_projection_config": {}, |
| "kernel_size": 3 |
| }, |
| "post_mixer_norm_config": null, |
| "pre_mlp_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mlp_config": { |
| "type": "DenseMLPConfig", |
| "linear_config": {}, |
| "activation": { |
| "type": "SiLU", |
| "alpha": 1.0 |
| }, |
| "has_up_biases": false, |
| "has_down_biases": false, |
| "gate_clipping": null, |
| "up_clipping": null |
| }, |
| "post_mlp_norm_config": null, |
| "hidden_dim": null, |
| "ple_config": null, |
| "has_post_layer_scalar": false, |
| "kv_source_layer_index": null, |
| "rope_config": null |
| }, |
| { |
| "pre_mixer_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mixer_config": { |
| "type": "ShortConvConfig", |
| "in_projection_config": {}, |
| "conv_config": { |
| "has_biases": false |
| }, |
| "out_projection_config": {}, |
| "kernel_size": 3 |
| }, |
| "post_mixer_norm_config": null, |
| "pre_mlp_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mlp_config": { |
| "type": "DenseMLPConfig", |
| "linear_config": {}, |
| "activation": { |
| "type": "SiLU", |
| "alpha": 1.0 |
| }, |
| "has_up_biases": false, |
| "has_down_biases": false, |
| "gate_clipping": null, |
| "up_clipping": null |
| }, |
| "post_mlp_norm_config": null, |
| "hidden_dim": null, |
| "ple_config": null, |
| "has_post_layer_scalar": false, |
| "kv_source_layer_index": null, |
| "rope_config": null |
| }, |
| { |
| "pre_mixer_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mixer_config": { |
| "type": "AttentionConfig", |
| "out_projection_config": {}, |
| "query_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "key_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "num_heads": 32, |
| "num_groups": 8, |
| "head_dim": 64, |
| "is_causal": true, |
| "scale": null, |
| "sliding_window_size": null, |
| "logit_soft_cap": null, |
| "has_sinks": false, |
| "has_out_biases": false, |
| "normalize_values": false, |
| "is_kv_sharing": false, |
| "qkv_projection_config": {}, |
| "has_qkv_biases": false, |
| "gate_projection_config": null |
| }, |
| "post_mixer_norm_config": null, |
| "pre_mlp_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mlp_config": { |
| "type": "DenseMLPConfig", |
| "linear_config": {}, |
| "activation": { |
| "type": "SiLU", |
| "alpha": 1.0 |
| }, |
| "has_up_biases": false, |
| "has_down_biases": false, |
| "gate_clipping": null, |
| "up_clipping": null |
| }, |
| "post_mlp_norm_config": null, |
| "hidden_dim": null, |
| "ple_config": null, |
| "has_post_layer_scalar": false, |
| "kv_source_layer_index": null, |
| "rope_config": { |
| "type": "UnscaledRoPEConfig", |
| "base": 1000000.0, |
| "max_sequence_length": 128000, |
| "head_dim": 64 |
| } |
| }, |
| { |
| "pre_mixer_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mixer_config": { |
| "type": "ShortConvConfig", |
| "in_projection_config": {}, |
| "conv_config": { |
| "has_biases": false |
| }, |
| "out_projection_config": {}, |
| "kernel_size": 3 |
| }, |
| "post_mixer_norm_config": null, |
| "pre_mlp_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mlp_config": { |
| "type": "DenseMLPConfig", |
| "linear_config": {}, |
| "activation": { |
| "type": "SiLU", |
| "alpha": 1.0 |
| }, |
| "has_up_biases": false, |
| "has_down_biases": false, |
| "gate_clipping": null, |
| "up_clipping": null |
| }, |
| "post_mlp_norm_config": null, |
| "hidden_dim": null, |
| "ple_config": null, |
| "has_post_layer_scalar": false, |
| "kv_source_layer_index": null, |
| "rope_config": null |
| }, |
| { |
| "pre_mixer_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mixer_config": { |
| "type": "ShortConvConfig", |
| "in_projection_config": {}, |
| "conv_config": { |
| "has_biases": false |
| }, |
| "out_projection_config": {}, |
| "kernel_size": 3 |
| }, |
| "post_mixer_norm_config": null, |
| "pre_mlp_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mlp_config": { |
| "type": "DenseMLPConfig", |
| "linear_config": {}, |
| "activation": { |
| "type": "SiLU", |
| "alpha": 1.0 |
| }, |
| "has_up_biases": false, |
| "has_down_biases": false, |
| "gate_clipping": null, |
| "up_clipping": null |
| }, |
| "post_mlp_norm_config": null, |
| "hidden_dim": null, |
| "ple_config": null, |
| "has_post_layer_scalar": false, |
| "kv_source_layer_index": null, |
| "rope_config": null |
| }, |
| { |
| "pre_mixer_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mixer_config": { |
| "type": "AttentionConfig", |
| "out_projection_config": {}, |
| "query_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "key_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "num_heads": 32, |
| "num_groups": 8, |
| "head_dim": 64, |
| "is_causal": true, |
| "scale": null, |
| "sliding_window_size": null, |
| "logit_soft_cap": null, |
| "has_sinks": false, |
| "has_out_biases": false, |
| "normalize_values": false, |
| "is_kv_sharing": false, |
| "qkv_projection_config": {}, |
| "has_qkv_biases": false, |
| "gate_projection_config": null |
| }, |
| "post_mixer_norm_config": null, |
| "pre_mlp_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mlp_config": { |
| "type": "DenseMLPConfig", |
| "linear_config": {}, |
| "activation": { |
| "type": "SiLU", |
| "alpha": 1.0 |
| }, |
| "has_up_biases": false, |
| "has_down_biases": false, |
| "gate_clipping": null, |
| "up_clipping": null |
| }, |
| "post_mlp_norm_config": null, |
| "hidden_dim": null, |
| "ple_config": null, |
| "has_post_layer_scalar": false, |
| "kv_source_layer_index": null, |
| "rope_config": { |
| "type": "UnscaledRoPEConfig", |
| "base": 1000000.0, |
| "max_sequence_length": 128000, |
| "head_dim": 64 |
| } |
| }, |
| { |
| "pre_mixer_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mixer_config": { |
| "type": "ShortConvConfig", |
| "in_projection_config": {}, |
| "conv_config": { |
| "has_biases": false |
| }, |
| "out_projection_config": {}, |
| "kernel_size": 3 |
| }, |
| "post_mixer_norm_config": null, |
| "pre_mlp_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mlp_config": { |
| "type": "DenseMLPConfig", |
| "linear_config": {}, |
| "activation": { |
| "type": "SiLU", |
| "alpha": 1.0 |
| }, |
| "has_up_biases": false, |
| "has_down_biases": false, |
| "gate_clipping": null, |
| "up_clipping": null |
| }, |
| "post_mlp_norm_config": null, |
| "hidden_dim": null, |
| "ple_config": null, |
| "has_post_layer_scalar": false, |
| "kv_source_layer_index": null, |
| "rope_config": null |
| }, |
| { |
| "pre_mixer_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mixer_config": { |
| "type": "AttentionConfig", |
| "out_projection_config": {}, |
| "query_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "key_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "num_heads": 32, |
| "num_groups": 8, |
| "head_dim": 64, |
| "is_causal": true, |
| "scale": null, |
| "sliding_window_size": null, |
| "logit_soft_cap": null, |
| "has_sinks": false, |
| "has_out_biases": false, |
| "normalize_values": false, |
| "is_kv_sharing": false, |
| "qkv_projection_config": {}, |
| "has_qkv_biases": false, |
| "gate_projection_config": null |
| }, |
| "post_mixer_norm_config": null, |
| "pre_mlp_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mlp_config": { |
| "type": "DenseMLPConfig", |
| "linear_config": {}, |
| "activation": { |
| "type": "SiLU", |
| "alpha": 1.0 |
| }, |
| "has_up_biases": false, |
| "has_down_biases": false, |
| "gate_clipping": null, |
| "up_clipping": null |
| }, |
| "post_mlp_norm_config": null, |
| "hidden_dim": null, |
| "ple_config": null, |
| "has_post_layer_scalar": false, |
| "kv_source_layer_index": null, |
| "rope_config": { |
| "type": "UnscaledRoPEConfig", |
| "base": 1000000.0, |
| "max_sequence_length": 128000, |
| "head_dim": 64 |
| } |
| }, |
| { |
| "pre_mixer_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mixer_config": { |
| "type": "ShortConvConfig", |
| "in_projection_config": {}, |
| "conv_config": { |
| "has_biases": false |
| }, |
| "out_projection_config": {}, |
| "kernel_size": 3 |
| }, |
| "post_mixer_norm_config": null, |
| "pre_mlp_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mlp_config": { |
| "type": "DenseMLPConfig", |
| "linear_config": {}, |
| "activation": { |
| "type": "SiLU", |
| "alpha": 1.0 |
| }, |
| "has_up_biases": false, |
| "has_down_biases": false, |
| "gate_clipping": null, |
| "up_clipping": null |
| }, |
| "post_mlp_norm_config": null, |
| "hidden_dim": null, |
| "ple_config": null, |
| "has_post_layer_scalar": false, |
| "kv_source_layer_index": null, |
| "rope_config": null |
| }, |
| { |
| "pre_mixer_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mixer_config": { |
| "type": "AttentionConfig", |
| "out_projection_config": {}, |
| "query_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "key_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "num_heads": 32, |
| "num_groups": 8, |
| "head_dim": 64, |
| "is_causal": true, |
| "scale": null, |
| "sliding_window_size": null, |
| "logit_soft_cap": null, |
| "has_sinks": false, |
| "has_out_biases": false, |
| "normalize_values": false, |
| "is_kv_sharing": false, |
| "qkv_projection_config": {}, |
| "has_qkv_biases": false, |
| "gate_projection_config": null |
| }, |
| "post_mixer_norm_config": null, |
| "pre_mlp_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mlp_config": { |
| "type": "DenseMLPConfig", |
| "linear_config": {}, |
| "activation": { |
| "type": "SiLU", |
| "alpha": 1.0 |
| }, |
| "has_up_biases": false, |
| "has_down_biases": false, |
| "gate_clipping": null, |
| "up_clipping": null |
| }, |
| "post_mlp_norm_config": null, |
| "hidden_dim": null, |
| "ple_config": null, |
| "has_post_layer_scalar": false, |
| "kv_source_layer_index": null, |
| "rope_config": { |
| "type": "UnscaledRoPEConfig", |
| "base": 1000000.0, |
| "max_sequence_length": 128000, |
| "head_dim": 64 |
| } |
| }, |
| { |
| "pre_mixer_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mixer_config": { |
| "type": "ShortConvConfig", |
| "in_projection_config": {}, |
| "conv_config": { |
| "has_biases": false |
| }, |
| "out_projection_config": {}, |
| "kernel_size": 3 |
| }, |
| "post_mixer_norm_config": null, |
| "pre_mlp_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mlp_config": { |
| "type": "DenseMLPConfig", |
| "linear_config": {}, |
| "activation": { |
| "type": "SiLU", |
| "alpha": 1.0 |
| }, |
| "has_up_biases": false, |
| "has_down_biases": false, |
| "gate_clipping": null, |
| "up_clipping": null |
| }, |
| "post_mlp_norm_config": null, |
| "hidden_dim": null, |
| "ple_config": null, |
| "has_post_layer_scalar": false, |
| "kv_source_layer_index": null, |
| "rope_config": null |
| }, |
| { |
| "pre_mixer_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mixer_config": { |
| "type": "AttentionConfig", |
| "out_projection_config": {}, |
| "query_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "key_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "num_heads": 32, |
| "num_groups": 8, |
| "head_dim": 64, |
| "is_causal": true, |
| "scale": null, |
| "sliding_window_size": null, |
| "logit_soft_cap": null, |
| "has_sinks": false, |
| "has_out_biases": false, |
| "normalize_values": false, |
| "is_kv_sharing": false, |
| "qkv_projection_config": {}, |
| "has_qkv_biases": false, |
| "gate_projection_config": null |
| }, |
| "post_mixer_norm_config": null, |
| "pre_mlp_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mlp_config": { |
| "type": "DenseMLPConfig", |
| "linear_config": {}, |
| "activation": { |
| "type": "SiLU", |
| "alpha": 1.0 |
| }, |
| "has_up_biases": false, |
| "has_down_biases": false, |
| "gate_clipping": null, |
| "up_clipping": null |
| }, |
| "post_mlp_norm_config": null, |
| "hidden_dim": null, |
| "ple_config": null, |
| "has_post_layer_scalar": false, |
| "kv_source_layer_index": null, |
| "rope_config": { |
| "type": "UnscaledRoPEConfig", |
| "base": 1000000.0, |
| "max_sequence_length": 128000, |
| "head_dim": 64 |
| } |
| }, |
| { |
| "pre_mixer_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mixer_config": { |
| "type": "ShortConvConfig", |
| "in_projection_config": {}, |
| "conv_config": { |
| "has_biases": false |
| }, |
| "out_projection_config": {}, |
| "kernel_size": 3 |
| }, |
| "post_mixer_norm_config": null, |
| "pre_mlp_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "mlp_config": { |
| "type": "DenseMLPConfig", |
| "linear_config": {}, |
| "activation": { |
| "type": "SiLU", |
| "alpha": 1.0 |
| }, |
| "has_up_biases": false, |
| "has_down_biases": false, |
| "gate_clipping": null, |
| "up_clipping": null |
| }, |
| "post_mlp_norm_config": null, |
| "hidden_dim": null, |
| "ple_config": null, |
| "has_post_layer_scalar": false, |
| "kv_source_layer_index": null, |
| "rope_config": null |
| } |
| ], |
| "output_norm_config": { |
| "epsilon": 1e-05, |
| "scale_offset": null, |
| "upcast_mode": "only_normalization", |
| "subtract_mean": false, |
| "has_scale": true, |
| "has_biases": false |
| }, |
| "model_dim": 2048, |
| "hidden_dim": 8192 |
| }, |
| "vocab_size": 65536, |
| "ple_model_config": null, |
| "embedding_norm_config": null |
| }, |
| "generation_config": { |
| "stop_token_ids": [ |
| 7 |
| ], |
| "temperature": 0.1, |
| "top_k": 50, |
| "top_p": 0.1, |
| "min_p": null, |
| "banned_tokens": null, |
| "repetition_penalty": null, |
| "presence_penalty": null, |
| "frequency_penalty": null, |
| "suffix_repetition_length": null |
| } |
| } |
