Hugging Face's logo

trymirai
/
LFM2.5-2.6B-L

Text Generation
Safetensors
uzu
mirai
8-bit precision
Model card Files
xet
 Community
knyazer commited on
Commit
cb0a24e
·
0 Parent(s):

Initial commit

Browse files
Files changed (7) hide show
  1. .gitattributes +36 -0
  2. .gitignore +2 -0
  3. README.md +71 -0
  4. assets/kl-vs-size.svg +2 -0
  5. config.json +1674 -0
  6. model.safetensors +3 -0
  7. tokenizer.json +3 -0
.gitattributes ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
.gitignore ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ /readme-generator.toml
2
+ /kl-results.csv
README.md ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ tags:
3
+ - uzu
4
+ - mirai
5
+ - 8-bit
6
+ library_name: uzu
7
+ license: other
8
+ license_name: lfm-open-license
9
+ license_link: https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/main/LICENSE
10
+ pipeline_tag: text-generation
11
+ base_model:
12
+ - LiquidAI/LFM2.5-2.6B
13
+ base_model_relation: quantized
14
+ ---
15
+
16
+ <div style="display:flex;align-items:center;justify-content:space-between;gap:20px;flex-wrap:wrap">
17
+ <div style="flex:1;min-width:260px">
18
+ <h1 style="margin:0 0 12px">Mirai's LFM2.5-2.6B Large Quantization</h1>
19
+ <p style="margin:0">A large, high-quality quantization of LFM2.5-2.6B prepared for efficient local inference on Apple silicon.</p>
20
+ </div>
21
+ <img src="https://assets.trymirai.com/images/logo/ml_small_logo.svg" alt="Mirai Labs" width="64" height="64" style="width:80px;height:80px;object-fit:contain;flex:0 0 auto">
22
+ </div>
23
+
24
+ ![KL divergence versus language-model checkpoint size](assets/kl-vs-size.svg)
25
+
26
+ KL divergence is shown on a logarithmic scale. Size is the decimal GB occupied by the language-model checkpoint; vision encoders, MTP heads, and speculative-decoding models are excluded.
27
+
28
+ ### Quickstart
29
+
30
+ If you are on macOS, the easiest way is to install the `mirai` Homebrew package and then run the CLI:
31
+
32
+ ```sh
33
+ brew install mirai
34
+ mirai --model trymirai/LFM2.5-2.6B-L
35
+ ```
36
+
37
+ Currently only Apple silicon inference is supported. If you want to build things from source, read this [overview](https://github.com/trymirai/uzu/blob/how-to/docs/how-to-run-uzu.md).
38
+
39
+ <div style="display:flex;flex-wrap:wrap;gap:12px;align-items:center;margin-top:32px">
40
+ <a href="https://docs.trymirai.com/" style="display:inline-flex;align-items:center;justify-content:space-between;width:220px;min-height:53px;padding:0 20px;box-sizing:border-box;border:1px solid #cccccc;border-radius:0;background:#ffffff;color:#3d3d3d;text-decoration:none;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Inter,sans-serif;font-size:15px;font-weight:500;line-height:1.5">
41
+ <span>API Docs</span>
42
+ <span aria-hidden="true" style="font-size:16px;line-height:1">&#8594;</span>
43
+ </a>
44
+ </div>
45
+
46
+ ---
47
+
48
+ ### Method
49
+
50
+ Mirai Large uses 8-bit symmetric integer quantization with bfloat16 scales and group size 64. Block-diagonal Random Hadamard Transforms with block size 32 are used to reduce activation and weight outliers. The checkpoint was prepared with post-training quantization.
51
+
52
+
53
+
54
+ ### Citation
55
+
56
+ If you find our work helpful, feel free to give us a cite.
57
+
58
+ ```bibtex
59
+ @misc{mirai-quant,
60
+ title = {{Mirai Quantization}: Redefining the speed-quality frontier for local LLMs on Apple silicon},
61
+ author = {Artur Chakhvadze and Ryan Mathieu and Roman Knyazhitskiy and Nikolai Voinilenko and Chen-Chen Yeh and Artur Mullakhmetov and Eugene Bokhan and others},
62
+ note = {In collaboration with others at Mirai Labs},
63
+ month = {June},
64
+ year = {2026},
65
+ url = {https://trymirai.com/blog/quantization}
66
+ }
67
+ ```
68
+
69
+ ### Original model
70
+
71
+ This is a quantized version of [LiquidAI/LFM2.5-2.6B](https://huggingface.co/LiquidAI/LFM2.5-2.6B). For architecture details, intended use, evaluations, and limitations, see the [original model card](https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/main/README.md).
assets/kl-vs-size.svg ADDED
config.json ADDED
@@ -0,0 +1,1674 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "type": "LanguageModelConfig",
3
+ "token_codec_config": {
4
+ "type": "ChatCodecConfig",
5
+ "prompt_template": "{{- bos_token -}}\n{%- set preserve_thinking = preserve_thinking | default(false) -%}\n\n{%- macro format_arg_value(arg_value) -%}\n {%- if arg_value is string -%}\n {{- \"'\" + (arg_value | replace(\"\\\\\", \"\\\\\\\\\") | replace(\"'\", \"\\\\'\") | replace(\"\\n\", \"\\\\n\") | replace(\"\\r\", \"\\\\r\")) + \"'\" -}}\n {%- elif arg_value is mapping or arg_value is iterable -%}\n {{- arg_value | tojson -}}\n {%- else -%}\n {{- arg_value | string -}}\n {%- endif -%}\n{%- endmacro -%}\n\n{%- macro parse_content(content) -%}\n {%- if content is string -%}\n {{- content -}}\n {%- elif content is mapping -%}\n {{- content | tojson -}}\n {%- elif content is iterable -%}\n {%- set _ns = namespace(result=\"\") -%}\n {%- for item in content -%}\n {%- if item is string -%}\n {%- set _ns.result = _ns.result + item -%}\n {%- elif item is mapping and item.get(\"type\") == \"image\" -%}\n {%- set _ns.result = _ns.result + \"<image>\" -%}\n {%- elif item is mapping and item.get(\"type\") == \"text\" -%}\n {%- set _ns.result = _ns.result + ((item.get(\"text\") or \"\") | string) -%}\n {%- else -%}\n {%- set _ns.result = _ns.result + (item | tojson) -%}\n {%- endif -%}\n {%- endfor -%}\n {{- _ns.result -}}\n {%- endif -%}\n{%- endmacro -%}\n\n{%- macro render_tool_calls(tool_calls) -%}\n {%- set tool_calls_ns = namespace(tool_calls=[]) -%}\n {%- for tool_call in tool_calls -%}\n {%- set func = tool_call[\"function\"] if \"function\" in tool_call else tool_call -%}\n {%- set func_name = func[\"name\"] -%}\n {%- set func_args = func.get(\"arguments\") -%}\n {%- set args_ns = namespace(arg_strings=[]) -%}\n {%- if func_args is mapping -%}\n {%- for arg_name, arg_value in func_args.items() -%}\n {%- set args_ns.arg_strings = args_ns.arg_strings + [arg_name + \"=\" + format_arg_value(arg_value)] -%}\n {%- endfor -%}\n {%- elif func_args is string and (func_args | trim) not in [\"\", \"{}\", \"null\"] -%}\n {{- raise_exception(\"Tool call arguments must be a mapping, got a JSON-encoded string: parse arguments with json.loads() before applying the chat template\") -}}\n {%- endif -%}\n {%- set tool_calls_ns.tool_calls = tool_calls_ns.tool_calls + [func_name + \"(\" + (args_ns.arg_strings | join(\", \")) + \")\"] -%}\n {%- endfor -%}\n {{- \"<|tool_call_start|>[\" + (tool_calls_ns.tool_calls | join(\", \")) + \"]<|tool_call_end|>\" -}}\n{%- endmacro -%}\n\n{%- set ns = namespace(system_prompt=\"\", last_user_index=-1) -%}\n{%- if messages and messages[0][\"role\"] == \"system\" -%}\n {%- if messages[0].get(\"content\") -%}\n {%- set ns.system_prompt = parse_content(messages[0][\"content\"]) -%}\n {%- endif -%}\n {%- set messages = messages[1:] -%}\n{%- endif -%}\n{%- if tools -%}\n {%- set ns.system_prompt = ns.system_prompt + (\"\\n\" if ns.system_prompt else \"\") + \"List of tools: [\" -%}\n {%- for tool in tools -%}\n {%- if tool is not string -%}\n {%- set tool = tool | tojson -%}\n {%- endif -%}\n {%- set ns.system_prompt = ns.system_prompt + tool -%}\n {%- if not loop.last -%}\n {%- set ns.system_prompt = ns.system_prompt + \", \" -%}\n {%- endif -%}\n {%- endfor -%}\n {%- set ns.system_prompt = ns.system_prompt + \"]\" -%}\n{%- endif -%}\n{%- if ns.system_prompt -%}\n {{- \"<|im_start|>system\\n\" + ns.system_prompt + \"<|im_end|>\\n\" -}}\n{%- endif -%}\n{%- for message in messages -%}\n {%- if message[\"role\"] == \"user\" -%}\n {%- set ns.last_user_index = loop.index0 -%}\n {%- endif -%}\n{%- endfor -%}\n{%- for message in messages -%}\n {{- \"<|im_start|>\" + message.role + \"\\n\" -}}\n {%- if message.role == \"assistant\" -%}\n \n {%- set keep_thinking = preserve_thinking or loop.index0 > ns.last_user_index -%}\n {%- set thinking = message.thinking or message.reasoning or message.reasoning_content -%}\n {%- set thinking = thinking if thinking is string else \"\" -%}\n {%- if thinking and keep_thinking -%}\n {{- \"<think>\" + thinking + \"</think>\" -}}\n {%- endif -%}\n {%- set _cfm_tag = \"CONTINUE_FINAL_MESSAGE_TAG \" -%}\n {%- set _has_cfm = false -%}\n {%- set content = \"\" -%}\n {%- if message.get(\"content\") -%}\n {%- set content = parse_content(message.content) -%}\n {%- endif -%}\n {%- if not keep_thinking and \"</think>\" in content -%}\n {%- set content = content.split(\"</think>\")[-1] | trim -%}\n {%- endif -%}\n {%- if content.endswith(_cfm_tag) -%}\n {%- set _has_cfm = true -%}\n {%- set _trunc_len = (content | length) - (_cfm_tag | length) -%}\n {%- set content = content[:_trunc_len] -%}\n {%- endif -%}\n {{- content -}}\n {%- if message.tool_calls -%}\n {{- render_tool_calls(message.tool_calls) -}}\n {%- endif -%}\n {%- if _has_cfm -%}\n {{- _cfm_tag -}}\n {%- endif -%}\n {{- \"<|im_end|>\\n\" -}}\n \n {%- else %}\n {%- if message.get(\"content\") -%}\n {{- parse_content(message[\"content\"]) -}}\n {%- endif -%}\n {{- \"<|im_end|>\\n\" -}}\n {%- endif %}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{- \"<|im_start|>assistant\\n<think>\" -}}\n{%- endif -%}\n",
6
+ "output_parser_regex": null,
7
+ "system_role_name": "system",
8
+ "user_role_name": "user",
9
+ "assistant_role_name": "assistant",
10
+ "eos_token": "<|im_end|>",
11
+ "bos_token": "<|startoftext|>",
12
+ "end_of_thinking_tag": "\n</think>",
13
+ "default_system_prompt": null
14
+ },
15
+ "decoder_config": {
16
+ "embedding_config": {
17
+ "type": "TiedEmbeddingConfig",
18
+ "input_scale": null,
19
+ "logit_soft_cap": null,
20
+ "logit_scale": null
21
+ },
22
+ "transformer_config": {
23
+ "layer_configs": [
24
+ {
25
+ "pre_mixer_norm_config": {
26
+ "epsilon": 1e-05,
27
+ "scale_offset": null,
28
+ "upcast_mode": "only_normalization",
29
+ "subtract_mean": false,
30
+ "has_scale": true,
31
+ "has_biases": false
32
+ },
33
+ "mixer_config": {
34
+ "type": "ShortConvConfig",
35
+ "in_projection_config": {},
36
+ "conv_config": {
37
+ "has_biases": false
38
+ },
39
+ "out_projection_config": {},
40
+ "kernel_size": 3
41
+ },
42
+ "post_mixer_norm_config": null,
43
+ "pre_mlp_norm_config": {
44
+ "epsilon": 1e-05,
45
+ "scale_offset": null,
46
+ "upcast_mode": "only_normalization",
47
+ "subtract_mean": false,
48
+ "has_scale": true,
49
+ "has_biases": false
50
+ },
51
+ "mlp_config": {
52
+ "type": "DenseMLPConfig",
53
+ "linear_config": {},
54
+ "activation": {
55
+ "type": "SiLU",
56
+ "alpha": 1.0
57
+ },
58
+ "has_up_biases": false,
59
+ "has_down_biases": false,
60
+ "gate_clipping": null,
61
+ "up_clipping": null
62
+ },
63
+ "post_mlp_norm_config": null,
64
+ "hidden_dim": null,
65
+ "ple_config": null,
66
+ "has_post_layer_scalar": false,
67
+ "kv_source_layer_index": null,
68
+ "rope_config": null
69
+ },
70
+ {
71
+ "pre_mixer_norm_config": {
72
+ "epsilon": 1e-05,
73
+ "scale_offset": null,
74
+ "upcast_mode": "only_normalization",
75
+ "subtract_mean": false,
76
+ "has_scale": true,
77
+ "has_biases": false
78
+ },
79
+ "mixer_config": {
80
+ "type": "ShortConvConfig",
81
+ "in_projection_config": {},
82
+ "conv_config": {
83
+ "has_biases": false
84
+ },
85
+ "out_projection_config": {},
86
+ "kernel_size": 3
87
+ },
88
+ "post_mixer_norm_config": null,
89
+ "pre_mlp_norm_config": {
90
+ "epsilon": 1e-05,
91
+ "scale_offset": null,
92
+ "upcast_mode": "only_normalization",
93
+ "subtract_mean": false,
94
+ "has_scale": true,
95
+ "has_biases": false
96
+ },
97
+ "mlp_config": {
98
+ "type": "DenseMLPConfig",
99
+ "linear_config": {},
100
+ "activation": {
101
+ "type": "SiLU",
102
+ "alpha": 1.0
103
+ },
104
+ "has_up_biases": false,
105
+ "has_down_biases": false,
106
+ "gate_clipping": null,
107
+ "up_clipping": null
108
+ },
109
+ "post_mlp_norm_config": null,
110
+ "hidden_dim": null,
111
+ "ple_config": null,
112
+ "has_post_layer_scalar": false,
113
+ "kv_source_layer_index": null,
114
+ "rope_config": null
115
+ },
116
+ {
117
+ "pre_mixer_norm_config": {
118
+ "epsilon": 1e-05,
119
+ "scale_offset": null,
120
+ "upcast_mode": "only_normalization",
121
+ "subtract_mean": false,
122
+ "has_scale": true,
123
+ "has_biases": false
124
+ },
125
+ "mixer_config": {
126
+ "type": "AttentionConfig",
127
+ "out_projection_config": {},
128
+ "query_norm_config": {
129
+ "epsilon": 1e-05,
130
+ "scale_offset": null,
131
+ "upcast_mode": "only_normalization",
132
+ "subtract_mean": false,
133
+ "has_scale": true,
134
+ "has_biases": false
135
+ },
136
+ "key_norm_config": {
137
+ "epsilon": 1e-05,
138
+ "scale_offset": null,
139
+ "upcast_mode": "only_normalization",
140
+ "subtract_mean": false,
141
+ "has_scale": true,
142
+ "has_biases": false
143
+ },
144
+ "num_heads": 32,
145
+ "num_groups": 8,
146
+ "head_dim": 64,
147
+ "is_causal": true,
148
+ "scale": null,
149
+ "sliding_window_size": null,
150
+ "logit_soft_cap": null,
151
+ "has_sinks": false,
152
+ "has_out_biases": false,
153
+ "normalize_values": false,
154
+ "is_kv_sharing": false,
155
+ "qkv_projection_config": {},
156
+ "has_qkv_biases": false,
157
+ "gate_projection_config": null
158
+ },
159
+ "post_mixer_norm_config": null,
160
+ "pre_mlp_norm_config": {
161
+ "epsilon": 1e-05,
162
+ "scale_offset": null,
163
+ "upcast_mode": "only_normalization",
164
+ "subtract_mean": false,
165
+ "has_scale": true,
166
+ "has_biases": false
167
+ },
168
+ "mlp_config": {
169
+ "type": "DenseMLPConfig",
170
+ "linear_config": {},
171
+ "activation": {
172
+ "type": "SiLU",
173
+ "alpha": 1.0
174
+ },
175
+ "has_up_biases": false,
176
+ "has_down_biases": false,
177
+ "gate_clipping": null,
178
+ "up_clipping": null
179
+ },
180
+ "post_mlp_norm_config": null,
181
+ "hidden_dim": null,
182
+ "ple_config": null,
183
+ "has_post_layer_scalar": false,
184
+ "kv_source_layer_index": null,
185
+ "rope_config": {
186
+ "type": "UnscaledRoPEConfig",
187
+ "base": 10000000.0,
188
+ "max_sequence_length": 131072,
189
+ "head_dim": 64
190
+ }
191
+ },
192
+ {
193
+ "pre_mixer_norm_config": {
194
+ "epsilon": 1e-05,
195
+ "scale_offset": null,
196
+ "upcast_mode": "only_normalization",
197
+ "subtract_mean": false,
198
+ "has_scale": true,
199
+ "has_biases": false
200
+ },
201
+ "mixer_config": {
202
+ "type": "ShortConvConfig",
203
+ "in_projection_config": {},
204
+ "conv_config": {
205
+ "has_biases": false
206
+ },
207
+ "out_projection_config": {},
208
+ "kernel_size": 3
209
+ },
210
+ "post_mixer_norm_config": null,
211
+ "pre_mlp_norm_config": {
212
+ "epsilon": 1e-05,
213
+ "scale_offset": null,
214
+ "upcast_mode": "only_normalization",
215
+ "subtract_mean": false,
216
+ "has_scale": true,
217
+ "has_biases": false
218
+ },
219
+ "mlp_config": {
220
+ "type": "DenseMLPConfig",
221
+ "linear_config": {},
222
+ "activation": {
223
+ "type": "SiLU",
224
+ "alpha": 1.0
225
+ },
226
+ "has_up_biases": false,
227
+ "has_down_biases": false,
228
+ "gate_clipping": null,
229
+ "up_clipping": null
230
+ },
231
+ "post_mlp_norm_config": null,
232
+ "hidden_dim": null,
233
+ "ple_config": null,
234
+ "has_post_layer_scalar": false,
235
+ "kv_source_layer_index": null,
236
+ "rope_config": null
237
+ },
238
+ {
239
+ "pre_mixer_norm_config": {
240
+ "epsilon": 1e-05,
241
+ "scale_offset": null,
242
+ "upcast_mode": "only_normalization",
243
+ "subtract_mean": false,
244
+ "has_scale": true,
245
+ "has_biases": false
246
+ },
247
+ "mixer_config": {
248
+ "type": "ShortConvConfig",
249
+ "in_projection_config": {},
250
+ "conv_config": {
251
+ "has_biases": false
252
+ },
253
+ "out_projection_config": {},
254
+ "kernel_size": 3
255
+ },
256
+ "post_mixer_norm_config": null,
257
+ "pre_mlp_norm_config": {
258
+ "epsilon": 1e-05,
259
+ "scale_offset": null,
260
+ "upcast_mode": "only_normalization",
261
+ "subtract_mean": false,
262
+ "has_scale": true,
263
+ "has_biases": false
264
+ },
265
+ "mlp_config": {
266
+ "type": "DenseMLPConfig",
267
+ "linear_config": {},
268
+ "activation": {
269
+ "type": "SiLU",
270
+ "alpha": 1.0
271
+ },
272
+ "has_up_biases": false,
273
+ "has_down_biases": false,
274
+ "gate_clipping": null,
275
+ "up_clipping": null
276
+ },
277
+ "post_mlp_norm_config": null,
278
+ "hidden_dim": null,
279
+ "ple_config": null,
280
+ "has_post_layer_scalar": false,
281
+ "kv_source_layer_index": null,
282
+ "rope_config": null
283
+ },
284
+ {
285
+ "pre_mixer_norm_config": {
286
+ "epsilon": 1e-05,
287
+ "scale_offset": null,
288
+ "upcast_mode": "only_normalization",
289
+ "subtract_mean": false,
290
+ "has_scale": true,
291
+ "has_biases": false
292
+ },
293
+ "mixer_config": {
294
+ "type": "AttentionConfig",
295
+ "out_projection_config": {},
296
+ "query_norm_config": {
297
+ "epsilon": 1e-05,
298
+ "scale_offset": null,
299
+ "upcast_mode": "only_normalization",
300
+ "subtract_mean": false,
301
+ "has_scale": true,
302
+ "has_biases": false
303
+ },
304
+ "key_norm_config": {
305
+ "epsilon": 1e-05,
306
+ "scale_offset": null,
307
+ "upcast_mode": "only_normalization",
308
+ "subtract_mean": false,
309
+ "has_scale": true,
310
+ "has_biases": false
311
+ },
312
+ "num_heads": 32,
313
+ "num_groups": 8,
314
+ "head_dim": 64,
315
+ "is_causal": true,
316
+ "scale": null,
317
+ "sliding_window_size": null,
318
+ "logit_soft_cap": null,
319
+ "has_sinks": false,
320
+ "has_out_biases": false,
321
+ "normalize_values": false,
322
+ "is_kv_sharing": false,
323
+ "qkv_projection_config": {},
324
+ "has_qkv_biases": false,
325
+ "gate_projection_config": null
326
+ },
327
+ "post_mixer_norm_config": null,
328
+ "pre_mlp_norm_config": {
329
+ "epsilon": 1e-05,
330
+ "scale_offset": null,
331
+ "upcast_mode": "only_normalization",
332
+ "subtract_mean": false,
333
+ "has_scale": true,
334
+ "has_biases": false
335
+ },
336
+ "mlp_config": {
337
+ "type": "DenseMLPConfig",
338
+ "linear_config": {},
339
+ "activation": {
340
+ "type": "SiLU",
341
+ "alpha": 1.0
342
+ },
343
+ "has_up_biases": false,
344
+ "has_down_biases": false,
345
+ "gate_clipping": null,
346
+ "up_clipping": null
347
+ },
348
+ "post_mlp_norm_config": null,
349
+ "hidden_dim": null,
350
+ "ple_config": null,
351
+ "has_post_layer_scalar": false,
352
+ "kv_source_layer_index": null,
353
+ "rope_config": {
354
+ "type": "UnscaledRoPEConfig",
355
+ "base": 10000000.0,
356
+ "max_sequence_length": 131072,
357
+ "head_dim": 64
358
+ }
359
+ },
360
+ {
361
+ "pre_mixer_norm_config": {
362
+ "epsilon": 1e-05,
363
+ "scale_offset": null,
364
+ "upcast_mode": "only_normalization",
365
+ "subtract_mean": false,
366
+ "has_scale": true,
367
+ "has_biases": false
368
+ },
369
+ "mixer_config": {
370
+ "type": "ShortConvConfig",
371
+ "in_projection_config": {},
372
+ "conv_config": {
373
+ "has_biases": false
374
+ },
375
+ "out_projection_config": {},
376
+ "kernel_size": 3
377
+ },
378
+ "post_mixer_norm_config": null,
379
+ "pre_mlp_norm_config": {
380
+ "epsilon": 1e-05,
381
+ "scale_offset": null,
382
+ "upcast_mode": "only_normalization",
383
+ "subtract_mean": false,
384
+ "has_scale": true,
385
+ "has_biases": false
386
+ },
387
+ "mlp_config": {
388
+ "type": "DenseMLPConfig",
389
+ "linear_config": {},
390
+ "activation": {
391
+ "type": "SiLU",
392
+ "alpha": 1.0
393
+ },
394
+ "has_up_biases": false,
395
+ "has_down_biases": false,
396
+ "gate_clipping": null,
397
+ "up_clipping": null
398
+ },
399
+ "post_mlp_norm_config": null,
400
+ "hidden_dim": null,
401
+ "ple_config": null,
402
+ "has_post_layer_scalar": false,
403
+ "kv_source_layer_index": null,
404
+ "rope_config": null
405
+ },
406
+ {
407
+ "pre_mixer_norm_config": {
408
+ "epsilon": 1e-05,
409
+ "scale_offset": null,
410
+ "upcast_mode": "only_normalization",
411
+ "subtract_mean": false,
412
+ "has_scale": true,
413
+ "has_biases": false
414
+ },
415
+ "mixer_config": {
416
+ "type": "ShortConvConfig",
417
+ "in_projection_config": {},
418
+ "conv_config": {
419
+ "has_biases": false
420
+ },
421
+ "out_projection_config": {},
422
+ "kernel_size": 3
423
+ },
424
+ "post_mixer_norm_config": null,
425
+ "pre_mlp_norm_config": {
426
+ "epsilon": 1e-05,
427
+ "scale_offset": null,
428
+ "upcast_mode": "only_normalization",
429
+ "subtract_mean": false,
430
+ "has_scale": true,
431
+ "has_biases": false
432
+ },
433
+ "mlp_config": {
434
+ "type": "DenseMLPConfig",
435
+ "linear_config": {},
436
+ "activation": {
437
+ "type": "SiLU",
438
+ "alpha": 1.0
439
+ },
440
+ "has_up_biases": false,
441
+ "has_down_biases": false,
442
+ "gate_clipping": null,
443
+ "up_clipping": null
444
+ },
445
+ "post_mlp_norm_config": null,
446
+ "hidden_dim": null,
447
+ "ple_config": null,
448
+ "has_post_layer_scalar": false,
449
+ "kv_source_layer_index": null,
450
+ "rope_config": null
451
+ },
452
+ {
453
+ "pre_mixer_norm_config": {
454
+ "epsilon": 1e-05,
455
+ "scale_offset": null,
456
+ "upcast_mode": "only_normalization",
457
+ "subtract_mean": false,
458
+ "has_scale": true,
459
+ "has_biases": false
460
+ },
461
+ "mixer_config": {
462
+ "type": "ShortConvConfig",
463
+ "in_projection_config": {},
464
+ "conv_config": {
465
+ "has_biases": false
466
+ },
467
+ "out_projection_config": {},
468
+ "kernel_size": 3
469
+ },
470
+ "post_mixer_norm_config": null,
471
+ "pre_mlp_norm_config": {
472
+ "epsilon": 1e-05,
473
+ "scale_offset": null,
474
+ "upcast_mode": "only_normalization",
475
+ "subtract_mean": false,
476
+ "has_scale": true,
477
+ "has_biases": false
478
+ },
479
+ "mlp_config": {
480
+ "type": "DenseMLPConfig",
481
+ "linear_config": {},
482
+ "activation": {
483
+ "type": "SiLU",
484
+ "alpha": 1.0
485
+ },
486
+ "has_up_biases": false,
487
+ "has_down_biases": false,
488
+ "gate_clipping": null,
489
+ "up_clipping": null
490
+ },
491
+ "post_mlp_norm_config": null,
492
+ "hidden_dim": null,
493
+ "ple_config": null,
494
+ "has_post_layer_scalar": false,
495
+ "kv_source_layer_index": null,
496
+ "rope_config": null
497
+ },
498
+ {
499
+ "pre_mixer_norm_config": {
500
+ "epsilon": 1e-05,
501
+ "scale_offset": null,
502
+ "upcast_mode": "only_normalization",
503
+ "subtract_mean": false,
504
+ "has_scale": true,
505
+ "has_biases": false
506
+ },
507
+ "mixer_config": {
508
+ "type": "AttentionConfig",
509
+ "out_projection_config": {},
510
+ "query_norm_config": {
511
+ "epsilon": 1e-05,
512
+ "scale_offset": null,
513
+ "upcast_mode": "only_normalization",
514
+ "subtract_mean": false,
515
+ "has_scale": true,
516
+ "has_biases": false
517
+ },
518
+ "key_norm_config": {
519
+ "epsilon": 1e-05,
520
+ "scale_offset": null,
521
+ "upcast_mode": "only_normalization",
522
+ "subtract_mean": false,
523
+ "has_scale": true,
524
+ "has_biases": false
525
+ },
526
+ "num_heads": 32,
527
+ "num_groups": 8,
528
+ "head_dim": 64,
529
+ "is_causal": true,
530
+ "scale": null,
531
+ "sliding_window_size": null,
532
+ "logit_soft_cap": null,
533
+ "has_sinks": false,
534
+ "has_out_biases": false,
535
+ "normalize_values": false,
536
+ "is_kv_sharing": false,
537
+ "qkv_projection_config": {},
538
+ "has_qkv_biases": false,
539
+ "gate_projection_config": null
540
+ },
541
+ "post_mixer_norm_config": null,
542
+ "pre_mlp_norm_config": {
543
+ "epsilon": 1e-05,
544
+ "scale_offset": null,
545
+ "upcast_mode": "only_normalization",
546
+ "subtract_mean": false,
547
+ "has_scale": true,
548
+ "has_biases": false
549
+ },
550
+ "mlp_config": {
551
+ "type": "DenseMLPConfig",
552
+ "linear_config": {},
553
+ "activation": {
554
+ "type": "SiLU",
555
+ "alpha": 1.0
556
+ },
557
+ "has_up_biases": false,
558
+ "has_down_biases": false,
559
+ "gate_clipping": null,
560
+ "up_clipping": null
561
+ },
562
+ "post_mlp_norm_config": null,
563
+ "hidden_dim": null,
564
+ "ple_config": null,
565
+ "has_post_layer_scalar": false,
566
+ "kv_source_layer_index": null,
567
+ "rope_config": {
568
+ "type": "UnscaledRoPEConfig",
569
+ "base": 10000000.0,
570
+ "max_sequence_length": 131072,
571
+ "head_dim": 64
572
+ }
573
+ },
574
+ {
575
+ "pre_mixer_norm_config": {
576
+ "epsilon": 1e-05,
577
+ "scale_offset": null,
578
+ "upcast_mode": "only_normalization",
579
+ "subtract_mean": false,
580
+ "has_scale": true,
581
+ "has_biases": false
582
+ },
583
+ "mixer_config": {
584
+ "type": "ShortConvConfig",
585
+ "in_projection_config": {},
586
+ "conv_config": {
587
+ "has_biases": false
588
+ },
589
+ "out_projection_config": {},
590
+ "kernel_size": 3
591
+ },
592
+ "post_mixer_norm_config": null,
593
+ "pre_mlp_norm_config": {
594
+ "epsilon": 1e-05,
595
+ "scale_offset": null,
596
+ "upcast_mode": "only_normalization",
597
+ "subtract_mean": false,
598
+ "has_scale": true,
599
+ "has_biases": false
600
+ },
601
+ "mlp_config": {
602
+ "type": "DenseMLPConfig",
603
+ "linear_config": {},
604
+ "activation": {
605
+ "type": "SiLU",
606
+ "alpha": 1.0
607
+ },
608
+ "has_up_biases": false,
609
+ "has_down_biases": false,
610
+ "gate_clipping": null,
611
+ "up_clipping": null
612
+ },
613
+ "post_mlp_norm_config": null,
614
+ "hidden_dim": null,
615
+ "ple_config": null,
616
+ "has_post_layer_scalar": false,
617
+ "kv_source_layer_index": null,
618
+ "rope_config": null
619
+ },
620
+ {
621
+ "pre_mixer_norm_config": {
622
+ "epsilon": 1e-05,
623
+ "scale_offset": null,
624
+ "upcast_mode": "only_normalization",
625
+ "subtract_mean": false,
626
+ "has_scale": true,
627
+ "has_biases": false
628
+ },
629
+ "mixer_config": {
630
+ "type": "ShortConvConfig",
631
+ "in_projection_config": {},
632
+ "conv_config": {
633
+ "has_biases": false
634
+ },
635
+ "out_projection_config": {},
636
+ "kernel_size": 3
637
+ },
638
+ "post_mixer_norm_config": null,
639
+ "pre_mlp_norm_config": {
640
+ "epsilon": 1e-05,
641
+ "scale_offset": null,
642
+ "upcast_mode": "only_normalization",
643
+ "subtract_mean": false,
644
+ "has_scale": true,
645
+ "has_biases": false
646
+ },
647
+ "mlp_config": {
648
+ "type": "DenseMLPConfig",
649
+ "linear_config": {},
650
+ "activation": {
651
+ "type": "SiLU",
652
+ "alpha": 1.0
653
+ },
654
+ "has_up_biases": false,
655
+ "has_down_biases": false,
656
+ "gate_clipping": null,
657
+ "up_clipping": null
658
+ },
659
+ "post_mlp_norm_config": null,
660
+ "hidden_dim": null,
661
+ "ple_config": null,
662
+ "has_post_layer_scalar": false,
663
+ "kv_source_layer_index": null,
664
+ "rope_config": null
665
+ },
666
+ {
667
+ "pre_mixer_norm_config": {
668
+ "epsilon": 1e-05,
669
+ "scale_offset": null,
670
+ "upcast_mode": "only_normalization",
671
+ "subtract_mean": false,
672
+ "has_scale": true,
673
+ "has_biases": false
674
+ },
675
+ "mixer_config": {
676
+ "type": "ShortConvConfig",
677
+ "in_projection_config": {},
678
+ "conv_config": {
679
+ "has_biases": false
680
+ },
681
+ "out_projection_config": {},
682
+ "kernel_size": 3
683
+ },
684
+ "post_mixer_norm_config": null,
685
+ "pre_mlp_norm_config": {
686
+ "epsilon": 1e-05,
687
+ "scale_offset": null,
688
+ "upcast_mode": "only_normalization",
689
+ "subtract_mean": false,
690
+ "has_scale": true,
691
+ "has_biases": false
692
+ },
693
+ "mlp_config": {
694
+ "type": "DenseMLPConfig",
695
+ "linear_config": {},
696
+ "activation": {
697
+ "type": "SiLU",
698
+ "alpha": 1.0
699
+ },
700
+ "has_up_biases": false,
701
+ "has_down_biases": false,
702
+ "gate_clipping": null,
703
+ "up_clipping": null
704
+ },
705
+ "post_mlp_norm_config": null,
706
+ "hidden_dim": null,
707
+ "ple_config": null,
708
+ "has_post_layer_scalar": false,
709
+ "kv_source_layer_index": null,
710
+ "rope_config": null
711
+ },
712
+ {
713
+ "pre_mixer_norm_config": {
714
+ "epsilon": 1e-05,
715
+ "scale_offset": null,
716
+ "upcast_mode": "only_normalization",
717
+ "subtract_mean": false,
718
+ "has_scale": true,
719
+ "has_biases": false
720
+ },
721
+ "mixer_config": {
722
+ "type": "AttentionConfig",
723
+ "out_projection_config": {},
724
+ "query_norm_config": {
725
+ "epsilon": 1e-05,
726
+ "scale_offset": null,
727
+ "upcast_mode": "only_normalization",
728
+ "subtract_mean": false,
729
+ "has_scale": true,
730
+ "has_biases": false
731
+ },
732
+ "key_norm_config": {
733
+ "epsilon": 1e-05,
734
+ "scale_offset": null,
735
+ "upcast_mode": "only_normalization",
736
+ "subtract_mean": false,
737
+ "has_scale": true,
738
+ "has_biases": false
739
+ },
740
+ "num_heads": 32,
741
+ "num_groups": 8,
742
+ "head_dim": 64,
743
+ "is_causal": true,
744
+ "scale": null,
745
+ "sliding_window_size": null,
746
+ "logit_soft_cap": null,
747
+ "has_sinks": false,
748
+ "has_out_biases": false,
749
+ "normalize_values": false,
750
+ "is_kv_sharing": false,
751
+ "qkv_projection_config": {},
752
+ "has_qkv_biases": false,
753
+ "gate_projection_config": null
754
+ },
755
+ "post_mixer_norm_config": null,
756
+ "pre_mlp_norm_config": {
757
+ "epsilon": 1e-05,
758
+ "scale_offset": null,
759
+ "upcast_mode": "only_normalization",
760
+ "subtract_mean": false,
761
+ "has_scale": true,
762
+ "has_biases": false
763
+ },
764
+ "mlp_config": {
765
+ "type": "DenseMLPConfig",
766
+ "linear_config": {},
767
+ "activation": {
768
+ "type": "SiLU",
769
+ "alpha": 1.0
770
+ },
771
+ "has_up_biases": false,
772
+ "has_down_biases": false,
773
+ "gate_clipping": null,
774
+ "up_clipping": null
775
+ },
776
+ "post_mlp_norm_config": null,
777
+ "hidden_dim": null,
778
+ "ple_config": null,
779
+ "has_post_layer_scalar": false,
780
+ "kv_source_layer_index": null,
781
+ "rope_config": {
782
+ "type": "UnscaledRoPEConfig",
783
+ "base": 10000000.0,
784
+ "max_sequence_length": 131072,
785
+ "head_dim": 64
786
+ }
787
+ },
788
+ {
789
+ "pre_mixer_norm_config": {
790
+ "epsilon": 1e-05,
791
+ "scale_offset": null,
792
+ "upcast_mode": "only_normalization",
793
+ "subtract_mean": false,
794
+ "has_scale": true,
795
+ "has_biases": false
796
+ },
797
+ "mixer_config": {
798
+ "type": "ShortConvConfig",
799
+ "in_projection_config": {},
800
+ "conv_config": {
801
+ "has_biases": false
802
+ },
803
+ "out_projection_config": {},
804
+ "kernel_size": 3
805
+ },
806
+ "post_mixer_norm_config": null,
807
+ "pre_mlp_norm_config": {
808
+ "epsilon": 1e-05,
809
+ "scale_offset": null,
810
+ "upcast_mode": "only_normalization",
811
+ "subtract_mean": false,
812
+ "has_scale": true,
813
+ "has_biases": false
814
+ },
815
+ "mlp_config": {
816
+ "type": "DenseMLPConfig",
817
+ "linear_config": {},
818
+ "activation": {
819
+ "type": "SiLU",
820
+ "alpha": 1.0
821
+ },
822
+ "has_up_biases": false,
823
+ "has_down_biases": false,
824
+ "gate_clipping": null,
825
+ "up_clipping": null
826
+ },
827
+ "post_mlp_norm_config": null,
828
+ "hidden_dim": null,
829
+ "ple_config": null,
830
+ "has_post_layer_scalar": false,
831
+ "kv_source_layer_index": null,
832
+ "rope_config": null
833
+ },
834
+ {
835
+ "pre_mixer_norm_config": {
836
+ "epsilon": 1e-05,
837
+ "scale_offset": null,
838
+ "upcast_mode": "only_normalization",
839
+ "subtract_mean": false,
840
+ "has_scale": true,
841
+ "has_biases": false
842
+ },
843
+ "mixer_config": {
844
+ "type": "ShortConvConfig",
845
+ "in_projection_config": {},
846
+ "conv_config": {
847
+ "has_biases": false
848
+ },
849
+ "out_projection_config": {},
850
+ "kernel_size": 3
851
+ },
852
+ "post_mixer_norm_config": null,
853
+ "pre_mlp_norm_config": {
854
+ "epsilon": 1e-05,
855
+ "scale_offset": null,
856
+ "upcast_mode": "only_normalization",
857
+ "subtract_mean": false,
858
+ "has_scale": true,
859
+ "has_biases": false
860
+ },
861
+ "mlp_config": {
862
+ "type": "DenseMLPConfig",
863
+ "linear_config": {},
864
+ "activation": {
865
+ "type": "SiLU",
866
+ "alpha": 1.0
867
+ },
868
+ "has_up_biases": false,
869
+ "has_down_biases": false,
870
+ "gate_clipping": null,
871
+ "up_clipping": null
872
+ },
873
+ "post_mlp_norm_config": null,
874
+ "hidden_dim": null,
875
+ "ple_config": null,
876
+ "has_post_layer_scalar": false,
877
+ "kv_source_layer_index": null,
878
+ "rope_config": null
879
+ },
880
+ {
881
+ "pre_mixer_norm_config": {
882
+ "epsilon": 1e-05,
883
+ "scale_offset": null,
884
+ "upcast_mode": "only_normalization",
885
+ "subtract_mean": false,
886
+ "has_scale": true,
887
+ "has_biases": false
888
+ },
889
+ "mixer_config": {
890
+ "type": "ShortConvConfig",
891
+ "in_projection_config": {},
892
+ "conv_config": {
893
+ "has_biases": false
894
+ },
895
+ "out_projection_config": {},
896
+ "kernel_size": 3
897
+ },
898
+ "post_mixer_norm_config": null,
899
+ "pre_mlp_norm_config": {
900
+ "epsilon": 1e-05,
901
+ "scale_offset": null,
902
+ "upcast_mode": "only_normalization",
903
+ "subtract_mean": false,
904
+ "has_scale": true,
905
+ "has_biases": false
906
+ },
907
+ "mlp_config": {
908
+ "type": "DenseMLPConfig",
909
+ "linear_config": {},
910
+ "activation": {
911
+ "type": "SiLU",
912
+ "alpha": 1.0
913
+ },
914
+ "has_up_biases": false,
915
+ "has_down_biases": false,
916
+ "gate_clipping": null,
917
+ "up_clipping": null
918
+ },
919
+ "post_mlp_norm_config": null,
920
+ "hidden_dim": null,
921
+ "ple_config": null,
922
+ "has_post_layer_scalar": false,
923
+ "kv_source_layer_index": null,
924
+ "rope_config": null
925
+ },
926
+ {
927
+ "pre_mixer_norm_config": {
928
+ "epsilon": 1e-05,
929
+ "scale_offset": null,
930
+ "upcast_mode": "only_normalization",
931
+ "subtract_mean": false,
932
+ "has_scale": true,
933
+ "has_biases": false
934
+ },
935
+ "mixer_config": {
936
+ "type": "AttentionConfig",
937
+ "out_projection_config": {},
938
+ "query_norm_config": {
939
+ "epsilon": 1e-05,
940
+ "scale_offset": null,
941
+ "upcast_mode": "only_normalization",
942
+ "subtract_mean": false,
943
+ "has_scale": true,
944
+ "has_biases": false
945
+ },
946
+ "key_norm_config": {
947
+ "epsilon": 1e-05,
948
+ "scale_offset": null,
949
+ "upcast_mode": "only_normalization",
950
+ "subtract_mean": false,
951
+ "has_scale": true,
952
+ "has_biases": false
953
+ },
954
+ "num_heads": 32,
955
+ "num_groups": 8,
956
+ "head_dim": 64,
957
+ "is_causal": true,
958
+ "scale": null,
959
+ "sliding_window_size": null,
960
+ "logit_soft_cap": null,
961
+ "has_sinks": false,
962
+ "has_out_biases": false,
963
+ "normalize_values": false,
964
+ "is_kv_sharing": false,
965
+ "qkv_projection_config": {},
966
+ "has_qkv_biases": false,
967
+ "gate_projection_config": null
968
+ },
969
+ "post_mixer_norm_config": null,
970
+ "pre_mlp_norm_config": {
971
+ "epsilon": 1e-05,
972
+ "scale_offset": null,
973
+ "upcast_mode": "only_normalization",
974
+ "subtract_mean": false,
975
+ "has_scale": true,
976
+ "has_biases": false
977
+ },
978
+ "mlp_config": {
979
+ "type": "DenseMLPConfig",
980
+ "linear_config": {},
981
+ "activation": {
982
+ "type": "SiLU",
983
+ "alpha": 1.0
984
+ },
985
+ "has_up_biases": false,
986
+ "has_down_biases": false,
987
+ "gate_clipping": null,
988
+ "up_clipping": null
989
+ },
990
+ "post_mlp_norm_config": null,
991
+ "hidden_dim": null,
992
+ "ple_config": null,
993
+ "has_post_layer_scalar": false,
994
+ "kv_source_layer_index": null,
995
+ "rope_config": {
996
+ "type": "UnscaledRoPEConfig",
997
+ "base": 10000000.0,
998
+ "max_sequence_length": 131072,
999
+ "head_dim": 64
1000
+ }
1001
+ },
1002
+ {
1003
+ "pre_mixer_norm_config": {
1004
+ "epsilon": 1e-05,
1005
+ "scale_offset": null,
1006
+ "upcast_mode": "only_normalization",
1007
+ "subtract_mean": false,
1008
+ "has_scale": true,
1009
+ "has_biases": false
1010
+ },
1011
+ "mixer_config": {
1012
+ "type": "ShortConvConfig",
1013
+ "in_projection_config": {},
1014
+ "conv_config": {
1015
+ "has_biases": false
1016
+ },
1017
+ "out_projection_config": {},
1018
+ "kernel_size": 3
1019
+ },
1020
+ "post_mixer_norm_config": null,
1021
+ "pre_mlp_norm_config": {
1022
+ "epsilon": 1e-05,
1023
+ "scale_offset": null,
1024
+ "upcast_mode": "only_normalization",
1025
+ "subtract_mean": false,
1026
+ "has_scale": true,
1027
+ "has_biases": false
1028
+ },
1029
+ "mlp_config": {
1030
+ "type": "DenseMLPConfig",
1031
+ "linear_config": {},
1032
+ "activation": {
1033
+ "type": "SiLU",
1034
+ "alpha": 1.0
1035
+ },
1036
+ "has_up_biases": false,
1037
+ "has_down_biases": false,
1038
+ "gate_clipping": null,
1039
+ "up_clipping": null
1040
+ },
1041
+ "post_mlp_norm_config": null,
1042
+ "hidden_dim": null,
1043
+ "ple_config": null,
1044
+ "has_post_layer_scalar": false,
1045
+ "kv_source_layer_index": null,
1046
+ "rope_config": null
1047
+ },
1048
+ {
1049
+ "pre_mixer_norm_config": {
1050
+ "epsilon": 1e-05,
1051
+ "scale_offset": null,
1052
+ "upcast_mode": "only_normalization",
1053
+ "subtract_mean": false,
1054
+ "has_scale": true,
1055
+ "has_biases": false
1056
+ },
1057
+ "mixer_config": {
1058
+ "type": "ShortConvConfig",
1059
+ "in_projection_config": {},
1060
+ "conv_config": {
1061
+ "has_biases": false
1062
+ },
1063
+ "out_projection_config": {},
1064
+ "kernel_size": 3
1065
+ },
1066
+ "post_mixer_norm_config": null,
1067
+ "pre_mlp_norm_config": {
1068
+ "epsilon": 1e-05,
1069
+ "scale_offset": null,
1070
+ "upcast_mode": "only_normalization",
1071
+ "subtract_mean": false,
1072
+ "has_scale": true,
1073
+ "has_biases": false
1074
+ },
1075
+ "mlp_config": {
1076
+ "type": "DenseMLPConfig",
1077
+ "linear_config": {},
1078
+ "activation": {
1079
+ "type": "SiLU",
1080
+ "alpha": 1.0
1081
+ },
1082
+ "has_up_biases": false,
1083
+ "has_down_biases": false,
1084
+ "gate_clipping": null,
1085
+ "up_clipping": null
1086
+ },
1087
+ "post_mlp_norm_config": null,
1088
+ "hidden_dim": null,
1089
+ "ple_config": null,
1090
+ "has_post_layer_scalar": false,
1091
+ "kv_source_layer_index": null,
1092
+ "rope_config": null
1093
+ },
1094
+ {
1095
+ "pre_mixer_norm_config": {
1096
+ "epsilon": 1e-05,
1097
+ "scale_offset": null,
1098
+ "upcast_mode": "only_normalization",
1099
+ "subtract_mean": false,
1100
+ "has_scale": true,
1101
+ "has_biases": false
1102
+ },
1103
+ "mixer_config": {
1104
+ "type": "ShortConvConfig",
1105
+ "in_projection_config": {},
1106
+ "conv_config": {
1107
+ "has_biases": false
1108
+ },
1109
+ "out_projection_config": {},
1110
+ "kernel_size": 3
1111
+ },
1112
+ "post_mixer_norm_config": null,
1113
+ "pre_mlp_norm_config": {
1114
+ "epsilon": 1e-05,
1115
+ "scale_offset": null,
1116
+ "upcast_mode": "only_normalization",
1117
+ "subtract_mean": false,
1118
+ "has_scale": true,
1119
+ "has_biases": false
1120
+ },
1121
+ "mlp_config": {
1122
+ "type": "DenseMLPConfig",
1123
+ "linear_config": {},
1124
+ "activation": {
1125
+ "type": "SiLU",
1126
+ "alpha": 1.0
1127
+ },
1128
+ "has_up_biases": false,
1129
+ "has_down_biases": false,
1130
+ "gate_clipping": null,
1131
+ "up_clipping": null
1132
+ },
1133
+ "post_mlp_norm_config": null,
1134
+ "hidden_dim": null,
1135
+ "ple_config": null,
1136
+ "has_post_layer_scalar": false,
1137
+ "kv_source_layer_index": null,
1138
+ "rope_config": null
1139
+ },
1140
+ {
1141
+ "pre_mixer_norm_config": {
1142
+ "epsilon": 1e-05,
1143
+ "scale_offset": null,
1144
+ "upcast_mode": "only_normalization",
1145
+ "subtract_mean": false,
1146
+ "has_scale": true,
1147
+ "has_biases": false
1148
+ },
1149
+ "mixer_config": {
1150
+ "type": "AttentionConfig",
1151
+ "out_projection_config": {},
1152
+ "query_norm_config": {
1153
+ "epsilon": 1e-05,
1154
+ "scale_offset": null,
1155
+ "upcast_mode": "only_normalization",
1156
+ "subtract_mean": false,
1157
+ "has_scale": true,
1158
+ "has_biases": false
1159
+ },
1160
+ "key_norm_config": {
1161
+ "epsilon": 1e-05,
1162
+ "scale_offset": null,
1163
+ "upcast_mode": "only_normalization",
1164
+ "subtract_mean": false,
1165
+ "has_scale": true,
1166
+ "has_biases": false
1167
+ },
1168
+ "num_heads": 32,
1169
+ "num_groups": 8,
1170
+ "head_dim": 64,
1171
+ "is_causal": true,
1172
+ "scale": null,
1173
+ "sliding_window_size": null,
1174
+ "logit_soft_cap": null,
1175
+ "has_sinks": false,
1176
+ "has_out_biases": false,
1177
+ "normalize_values": false,
1178
+ "is_kv_sharing": false,
1179
+ "qkv_projection_config": {},
1180
+ "has_qkv_biases": false,
1181
+ "gate_projection_config": null
1182
+ },
1183
+ "post_mixer_norm_config": null,
1184
+ "pre_mlp_norm_config": {
1185
+ "epsilon": 1e-05,
1186
+ "scale_offset": null,
1187
+ "upcast_mode": "only_normalization",
1188
+ "subtract_mean": false,
1189
+ "has_scale": true,
1190
+ "has_biases": false
1191
+ },
1192
+ "mlp_config": {
1193
+ "type": "DenseMLPConfig",
1194
+ "linear_config": {},
1195
+ "activation": {
1196
+ "type": "SiLU",
1197
+ "alpha": 1.0
1198
+ },
1199
+ "has_up_biases": false,
1200
+ "has_down_biases": false,
1201
+ "gate_clipping": null,
1202
+ "up_clipping": null
1203
+ },
1204
+ "post_mlp_norm_config": null,
1205
+ "hidden_dim": null,
1206
+ "ple_config": null,
1207
+ "has_post_layer_scalar": false,
1208
+ "kv_source_layer_index": null,
1209
+ "rope_config": {
1210
+ "type": "UnscaledRoPEConfig",
1211
+ "base": 10000000.0,
1212
+ "max_sequence_length": 131072,
1213
+ "head_dim": 64
1214
+ }
1215
+ },
1216
+ {
1217
+ "pre_mixer_norm_config": {
1218
+ "epsilon": 1e-05,
1219
+ "scale_offset": null,
1220
+ "upcast_mode": "only_normalization",
1221
+ "subtract_mean": false,
1222
+ "has_scale": true,
1223
+ "has_biases": false
1224
+ },
1225
+ "mixer_config": {
1226
+ "type": "ShortConvConfig",
1227
+ "in_projection_config": {},
1228
+ "conv_config": {
1229
+ "has_biases": false
1230
+ },
1231
+ "out_projection_config": {},
1232
+ "kernel_size": 3
1233
+ },
1234
+ "post_mixer_norm_config": null,
1235
+ "pre_mlp_norm_config": {
1236
+ "epsilon": 1e-05,
1237
+ "scale_offset": null,
1238
+ "upcast_mode": "only_normalization",
1239
+ "subtract_mean": false,
1240
+ "has_scale": true,
1241
+ "has_biases": false
1242
+ },
1243
+ "mlp_config": {
1244
+ "type": "DenseMLPConfig",
1245
+ "linear_config": {},
1246
+ "activation": {
1247
+ "type": "SiLU",
1248
+ "alpha": 1.0
1249
+ },
1250
+ "has_up_biases": false,
1251
+ "has_down_biases": false,
1252
+ "gate_clipping": null,
1253
+ "up_clipping": null
1254
+ },
1255
+ "post_mlp_norm_config": null,
1256
+ "hidden_dim": null,
1257
+ "ple_config": null,
1258
+ "has_post_layer_scalar": false,
1259
+ "kv_source_layer_index": null,
1260
+ "rope_config": null
1261
+ },
1262
+ {
1263
+ "pre_mixer_norm_config": {
1264
+ "epsilon": 1e-05,
1265
+ "scale_offset": null,
1266
+ "upcast_mode": "only_normalization",
1267
+ "subtract_mean": false,
1268
+ "has_scale": true,
1269
+ "has_biases": false
1270
+ },
1271
+ "mixer_config": {
1272
+ "type": "ShortConvConfig",
1273
+ "in_projection_config": {},
1274
+ "conv_config": {
1275
+ "has_biases": false
1276
+ },
1277
+ "out_projection_config": {},
1278
+ "kernel_size": 3
1279
+ },
1280
+ "post_mixer_norm_config": null,
1281
+ "pre_mlp_norm_config": {
1282
+ "epsilon": 1e-05,
1283
+ "scale_offset": null,
1284
+ "upcast_mode": "only_normalization",
1285
+ "subtract_mean": false,
1286
+ "has_scale": true,
1287
+ "has_biases": false
1288
+ },
1289
+ "mlp_config": {
1290
+ "type": "DenseMLPConfig",
1291
+ "linear_config": {},
1292
+ "activation": {
1293
+ "type": "SiLU",
1294
+ "alpha": 1.0
1295
+ },
1296
+ "has_up_biases": false,
1297
+ "has_down_biases": false,
1298
+ "gate_clipping": null,
1299
+ "up_clipping": null
1300
+ },
1301
+ "post_mlp_norm_config": null,
1302
+ "hidden_dim": null,
1303
+ "ple_config": null,
1304
+ "has_post_layer_scalar": false,
1305
+ "kv_source_layer_index": null,
1306
+ "rope_config": null
1307
+ },
1308
+ {
1309
+ "pre_mixer_norm_config": {
1310
+ "epsilon": 1e-05,
1311
+ "scale_offset": null,
1312
+ "upcast_mode": "only_normalization",
1313
+ "subtract_mean": false,
1314
+ "has_scale": true,
1315
+ "has_biases": false
1316
+ },
1317
+ "mixer_config": {
1318
+ "type": "AttentionConfig",
1319
+ "out_projection_config": {},
1320
+ "query_norm_config": {
1321
+ "epsilon": 1e-05,
1322
+ "scale_offset": null,
1323
+ "upcast_mode": "only_normalization",
1324
+ "subtract_mean": false,
1325
+ "has_scale": true,
1326
+ "has_biases": false
1327
+ },
1328
+ "key_norm_config": {
1329
+ "epsilon": 1e-05,
1330
+ "scale_offset": null,
1331
+ "upcast_mode": "only_normalization",
1332
+ "subtract_mean": false,
1333
+ "has_scale": true,
1334
+ "has_biases": false
1335
+ },
1336
+ "num_heads": 32,
1337
+ "num_groups": 8,
1338
+ "head_dim": 64,
1339
+ "is_causal": true,
1340
+ "scale": null,
1341
+ "sliding_window_size": null,
1342
+ "logit_soft_cap": null,
1343
+ "has_sinks": false,
1344
+ "has_out_biases": false,
1345
+ "normalize_values": false,
1346
+ "is_kv_sharing": false,
1347
+ "qkv_projection_config": {},
1348
+ "has_qkv_biases": false,
1349
+ "gate_projection_config": null
1350
+ },
1351
+ "post_mixer_norm_config": null,
1352
+ "pre_mlp_norm_config": {
1353
+ "epsilon": 1e-05,
1354
+ "scale_offset": null,
1355
+ "upcast_mode": "only_normalization",
1356
+ "subtract_mean": false,
1357
+ "has_scale": true,
1358
+ "has_biases": false
1359
+ },
1360
+ "mlp_config": {
1361
+ "type": "DenseMLPConfig",
1362
+ "linear_config": {},
1363
+ "activation": {
1364
+ "type": "SiLU",
1365
+ "alpha": 1.0
1366
+ },
1367
+ "has_up_biases": false,
1368
+ "has_down_biases": false,
1369
+ "gate_clipping": null,
1370
+ "up_clipping": null
1371
+ },
1372
+ "post_mlp_norm_config": null,
1373
+ "hidden_dim": null,
1374
+ "ple_config": null,
1375
+ "has_post_layer_scalar": false,
1376
+ "kv_source_layer_index": null,
1377
+ "rope_config": {
1378
+ "type": "UnscaledRoPEConfig",
1379
+ "base": 10000000.0,
1380
+ "max_sequence_length": 131072,
1381
+ "head_dim": 64
1382
+ }
1383
+ },
1384
+ {
1385
+ "pre_mixer_norm_config": {
1386
+ "epsilon": 1e-05,
1387
+ "scale_offset": null,
1388
+ "upcast_mode": "only_normalization",
1389
+ "subtract_mean": false,
1390
+ "has_scale": true,
1391
+ "has_biases": false
1392
+ },
1393
+ "mixer_config": {
1394
+ "type": "ShortConvConfig",
1395
+ "in_projection_config": {},
1396
+ "conv_config": {
1397
+ "has_biases": false
1398
+ },
1399
+ "out_projection_config": {},
1400
+ "kernel_size": 3
1401
+ },
1402
+ "post_mixer_norm_config": null,
1403
+ "pre_mlp_norm_config": {
1404
+ "epsilon": 1e-05,
1405
+ "scale_offset": null,
1406
+ "upcast_mode": "only_normalization",
1407
+ "subtract_mean": false,
1408
+ "has_scale": true,
1409
+ "has_biases": false
1410
+ },
1411
+ "mlp_config": {
1412
+ "type": "DenseMLPConfig",
1413
+ "linear_config": {},
1414
+ "activation": {
1415
+ "type": "SiLU",
1416
+ "alpha": 1.0
1417
+ },
1418
+ "has_up_biases": false,
1419
+ "has_down_biases": false,
1420
+ "gate_clipping": null,
1421
+ "up_clipping": null
1422
+ },
1423
+ "post_mlp_norm_config": null,
1424
+ "hidden_dim": null,
1425
+ "ple_config": null,
1426
+ "has_post_layer_scalar": false,
1427
+ "kv_source_layer_index": null,
1428
+ "rope_config": null
1429
+ },
1430
+ {
1431
+ "pre_mixer_norm_config": {
1432
+ "epsilon": 1e-05,
1433
+ "scale_offset": null,
1434
+ "upcast_mode": "only_normalization",
1435
+ "subtract_mean": false,
1436
+ "has_scale": true,
1437
+ "has_biases": false
1438
+ },
1439
+ "mixer_config": {
1440
+ "type": "ShortConvConfig",
1441
+ "in_projection_config": {},
1442
+ "conv_config": {
1443
+ "has_biases": false
1444
+ },
1445
+ "out_projection_config": {},
1446
+ "kernel_size": 3
1447
+ },
1448
+ "post_mixer_norm_config": null,
1449
+ "pre_mlp_norm_config": {
1450
+ "epsilon": 1e-05,
1451
+ "scale_offset": null,
1452
+ "upcast_mode": "only_normalization",
1453
+ "subtract_mean": false,
1454
+ "has_scale": true,
1455
+ "has_biases": false
1456
+ },
1457
+ "mlp_config": {
1458
+ "type": "DenseMLPConfig",
1459
+ "linear_config": {},
1460
+ "activation": {
1461
+ "type": "SiLU",
1462
+ "alpha": 1.0
1463
+ },
1464
+ "has_up_biases": false,
1465
+ "has_down_biases": false,
1466
+ "gate_clipping": null,
1467
+ "up_clipping": null
1468
+ },
1469
+ "post_mlp_norm_config": null,
1470
+ "hidden_dim": null,
1471
+ "ple_config": null,
1472
+ "has_post_layer_scalar": false,
1473
+ "kv_source_layer_index": null,
1474
+ "rope_config": null
1475
+ },
1476
+ {
1477
+ "pre_mixer_norm_config": {
1478
+ "epsilon": 1e-05,
1479
+ "scale_offset": null,
1480
+ "upcast_mode": "only_normalization",
1481
+ "subtract_mean": false,
1482
+ "has_scale": true,
1483
+ "has_biases": false
1484
+ },
1485
+ "mixer_config": {
1486
+ "type": "AttentionConfig",
1487
+ "out_projection_config": {},
1488
+ "query_norm_config": {
1489
+ "epsilon": 1e-05,
1490
+ "scale_offset": null,
1491
+ "upcast_mode": "only_normalization",
1492
+ "subtract_mean": false,
1493
+ "has_scale": true,
1494
+ "has_biases": false
1495
+ },
1496
+ "key_norm_config": {
1497
+ "epsilon": 1e-05,
1498
+ "scale_offset": null,
1499
+ "upcast_mode": "only_normalization",
1500
+ "subtract_mean": false,
1501
+ "has_scale": true,
1502
+ "has_biases": false
1503
+ },
1504
+ "num_heads": 32,
1505
+ "num_groups": 8,
1506
+ "head_dim": 64,
1507
+ "is_causal": true,
1508
+ "scale": null,
1509
+ "sliding_window_size": null,
1510
+ "logit_soft_cap": null,
1511
+ "has_sinks": false,
1512
+ "has_out_biases": false,
1513
+ "normalize_values": false,
1514
+ "is_kv_sharing": false,
1515
+ "qkv_projection_config": {},
1516
+ "has_qkv_biases": false,
1517
+ "gate_projection_config": null
1518
+ },
1519
+ "post_mixer_norm_config": null,
1520
+ "pre_mlp_norm_config": {
1521
+ "epsilon": 1e-05,
1522
+ "scale_offset": null,
1523
+ "upcast_mode": "only_normalization",
1524
+ "subtract_mean": false,
1525
+ "has_scale": true,
1526
+ "has_biases": false
1527
+ },
1528
+ "mlp_config": {
1529
+ "type": "DenseMLPConfig",
1530
+ "linear_config": {},
1531
+ "activation": {
1532
+ "type": "SiLU",
1533
+ "alpha": 1.0
1534
+ },
1535
+ "has_up_biases": false,
1536
+ "has_down_biases": false,
1537
+ "gate_clipping": null,
1538
+ "up_clipping": null
1539
+ },
1540
+ "post_mlp_norm_config": null,
1541
+ "hidden_dim": null,
1542
+ "ple_config": null,
1543
+ "has_post_layer_scalar": false,
1544
+ "kv_source_layer_index": null,
1545
+ "rope_config": {
1546
+ "type": "UnscaledRoPEConfig",
1547
+ "base": 10000000.0,
1548
+ "max_sequence_length": 131072,
1549
+ "head_dim": 64
1550
+ }
1551
+ },
1552
+ {
1553
+ "pre_mixer_norm_config": {
1554
+ "epsilon": 1e-05,
1555
+ "scale_offset": null,
1556
+ "upcast_mode": "only_normalization",
1557
+ "subtract_mean": false,
1558
+ "has_scale": true,
1559
+ "has_biases": false
1560
+ },
1561
+ "mixer_config": {
1562
+ "type": "ShortConvConfig",
1563
+ "in_projection_config": {},
1564
+ "conv_config": {
1565
+ "has_biases": false
1566
+ },
1567
+ "out_projection_config": {},
1568
+ "kernel_size": 3
1569
+ },
1570
+ "post_mixer_norm_config": null,
1571
+ "pre_mlp_norm_config": {
1572
+ "epsilon": 1e-05,
1573
+ "scale_offset": null,
1574
+ "upcast_mode": "only_normalization",
1575
+ "subtract_mean": false,
1576
+ "has_scale": true,
1577
+ "has_biases": false
1578
+ },
1579
+ "mlp_config": {
1580
+ "type": "DenseMLPConfig",
1581
+ "linear_config": {},
1582
+ "activation": {
1583
+ "type": "SiLU",
1584
+ "alpha": 1.0
1585
+ },
1586
+ "has_up_biases": false,
1587
+ "has_down_biases": false,
1588
+ "gate_clipping": null,
1589
+ "up_clipping": null
1590
+ },
1591
+ "post_mlp_norm_config": null,
1592
+ "hidden_dim": null,
1593
+ "ple_config": null,
1594
+ "has_post_layer_scalar": false,
1595
+ "kv_source_layer_index": null,
1596
+ "rope_config": null
1597
+ },
1598
+ {
1599
+ "pre_mixer_norm_config": {
1600
+ "epsilon": 1e-05,
1601
+ "scale_offset": null,
1602
+ "upcast_mode": "only_normalization",
1603
+ "subtract_mean": false,
1604
+ "has_scale": true,
1605
+ "has_biases": false
1606
+ },
1607
+ "mixer_config": {
1608
+ "type": "ShortConvConfig",
1609
+ "in_projection_config": {},
1610
+ "conv_config": {
1611
+ "has_biases": false
1612
+ },
1613
+ "out_projection_config": {},
1614
+ "kernel_size": 3
1615
+ },
1616
+ "post_mixer_norm_config": null,
1617
+ "pre_mlp_norm_config": {
1618
+ "epsilon": 1e-05,
1619
+ "scale_offset": null,
1620
+ "upcast_mode": "only_normalization",
1621
+ "subtract_mean": false,
1622
+ "has_scale": true,
1623
+ "has_biases": false
1624
+ },
1625
+ "mlp_config": {
1626
+ "type": "DenseMLPConfig",
1627
+ "linear_config": {},
1628
+ "activation": {
1629
+ "type": "SiLU",
1630
+ "alpha": 1.0
1631
+ },
1632
+ "has_up_biases": false,
1633
+ "has_down_biases": false,
1634
+ "gate_clipping": null,
1635
+ "up_clipping": null
1636
+ },
1637
+ "post_mlp_norm_config": null,
1638
+ "hidden_dim": null,
1639
+ "ple_config": null,
1640
+ "has_post_layer_scalar": false,
1641
+ "kv_source_layer_index": null,
1642
+ "rope_config": null
1643
+ }
1644
+ ],
1645
+ "output_norm_config": {
1646
+ "epsilon": 1e-05,
1647
+ "scale_offset": null,
1648
+ "upcast_mode": "only_normalization",
1649
+ "subtract_mean": false,
1650
+ "has_scale": true,
1651
+ "has_biases": false
1652
+ },
1653
+ "model_dim": 2048,
1654
+ "hidden_dim": 10752
1655
+ },
1656
+ "vocab_size": 128000,
1657
+ "ple_model_config": null,
1658
+ "embedding_norm_config": null
1659
+ },
1660
+ "generation_config": {
1661
+ "stop_token_ids": [
1662
+ 124900
1663
+ ],
1664
+ "temperature": 0.1,
1665
+ "top_k": 50,
1666
+ "top_p": 0.1,
1667
+ "min_p": null,
1668
+ "banned_tokens": null,
1669
+ "repetition_penalty": null,
1670
+ "presence_penalty": null,
1671
+ "frequency_penalty": null,
1672
+ "suffix_repetition_length": null
1673
+ }
1674
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:623d40e583c60edb8b89f0cbf2cb66ca15598dc5e3345686da4ff52c19f7c4e2
3
+ size 2788155800
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:695be7802a0e4b8a81048f0ff5ebb7fc811a0ba5a6be63dbb24deb5a81096f41
3
+ size 17905598