| { | |
| "format": "tensorstore", | |
| "version": "0.0.84", | |
| "prefixes": { | |
| "model": [ | |
| { | |
| "path": "model/lm_head/kernel", | |
| "shape": [ | |
| 2880, | |
| 201088 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/embed_tokens/embedding", | |
| "shape": [ | |
| 201088, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/0/input_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/0/mlp/experts/down_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/0/mlp/experts/down_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/0/mlp/experts/gate_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/0/mlp/experts/gate_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/0/mlp/experts/up_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/0/mlp/experts/up_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/0/mlp/router/bias", | |
| "shape": [ | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/0/mlp/router/kernel", | |
| "shape": [ | |
| 2880, | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/0/post_attention_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/0/self_attn/k_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/0/self_attn/k_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/0/self_attn/o_proj/bias", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/0/self_attn/o_proj/kernel", | |
| "shape": [ | |
| 4096, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/0/self_attn/q_proj/bias", | |
| "shape": [ | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/0/self_attn/q_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/0/self_attn/sinks", | |
| "shape": [ | |
| 64 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/0/self_attn/v_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/0/self_attn/v_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/1/input_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/1/mlp/experts/down_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/1/mlp/experts/down_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/1/mlp/experts/gate_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/1/mlp/experts/gate_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/1/mlp/experts/up_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/1/mlp/experts/up_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/1/mlp/router/bias", | |
| "shape": [ | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/1/mlp/router/kernel", | |
| "shape": [ | |
| 2880, | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/1/post_attention_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/1/self_attn/k_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/1/self_attn/k_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/1/self_attn/o_proj/bias", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/1/self_attn/o_proj/kernel", | |
| "shape": [ | |
| 4096, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/1/self_attn/q_proj/bias", | |
| "shape": [ | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/1/self_attn/q_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/1/self_attn/sinks", | |
| "shape": [ | |
| 64 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/1/self_attn/v_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/1/self_attn/v_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/2/input_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/2/mlp/experts/down_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/2/mlp/experts/down_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/2/mlp/experts/gate_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/2/mlp/experts/gate_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/2/mlp/experts/up_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/2/mlp/experts/up_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/2/mlp/router/bias", | |
| "shape": [ | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/2/mlp/router/kernel", | |
| "shape": [ | |
| 2880, | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/2/post_attention_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/2/self_attn/k_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/2/self_attn/k_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/2/self_attn/o_proj/bias", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/2/self_attn/o_proj/kernel", | |
| "shape": [ | |
| 4096, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/2/self_attn/q_proj/bias", | |
| "shape": [ | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/2/self_attn/q_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/2/self_attn/sinks", | |
| "shape": [ | |
| 64 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/2/self_attn/v_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/2/self_attn/v_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/3/input_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/3/mlp/experts/down_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/3/mlp/experts/down_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/3/mlp/experts/gate_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/3/mlp/experts/gate_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/3/mlp/experts/up_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/3/mlp/experts/up_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/3/mlp/router/bias", | |
| "shape": [ | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/3/mlp/router/kernel", | |
| "shape": [ | |
| 2880, | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/3/post_attention_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/3/self_attn/k_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/3/self_attn/k_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/3/self_attn/o_proj/bias", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/3/self_attn/o_proj/kernel", | |
| "shape": [ | |
| 4096, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/3/self_attn/q_proj/bias", | |
| "shape": [ | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/3/self_attn/q_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/3/self_attn/sinks", | |
| "shape": [ | |
| 64 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/3/self_attn/v_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/3/self_attn/v_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/4/input_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/4/mlp/experts/down_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/4/mlp/experts/down_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/4/mlp/experts/gate_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/4/mlp/experts/gate_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/4/mlp/experts/up_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/4/mlp/experts/up_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/4/mlp/router/bias", | |
| "shape": [ | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/4/mlp/router/kernel", | |
| "shape": [ | |
| 2880, | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/4/post_attention_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/4/self_attn/k_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/4/self_attn/k_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/4/self_attn/o_proj/bias", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/4/self_attn/o_proj/kernel", | |
| "shape": [ | |
| 4096, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/4/self_attn/q_proj/bias", | |
| "shape": [ | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/4/self_attn/q_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/4/self_attn/sinks", | |
| "shape": [ | |
| 64 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/4/self_attn/v_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/4/self_attn/v_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/5/input_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/5/mlp/experts/down_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/5/mlp/experts/down_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/5/mlp/experts/gate_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/5/mlp/experts/gate_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/5/mlp/experts/up_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/5/mlp/experts/up_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/5/mlp/router/bias", | |
| "shape": [ | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/5/mlp/router/kernel", | |
| "shape": [ | |
| 2880, | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/5/post_attention_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/5/self_attn/k_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/5/self_attn/k_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/5/self_attn/o_proj/bias", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/5/self_attn/o_proj/kernel", | |
| "shape": [ | |
| 4096, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/5/self_attn/q_proj/bias", | |
| "shape": [ | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/5/self_attn/q_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/5/self_attn/sinks", | |
| "shape": [ | |
| 64 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/5/self_attn/v_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/5/self_attn/v_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/6/input_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/6/mlp/experts/down_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/6/mlp/experts/down_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/6/mlp/experts/gate_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/6/mlp/experts/gate_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/6/mlp/experts/up_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/6/mlp/experts/up_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/6/mlp/router/bias", | |
| "shape": [ | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/6/mlp/router/kernel", | |
| "shape": [ | |
| 2880, | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/6/post_attention_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/6/self_attn/k_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/6/self_attn/k_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/6/self_attn/o_proj/bias", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/6/self_attn/o_proj/kernel", | |
| "shape": [ | |
| 4096, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/6/self_attn/q_proj/bias", | |
| "shape": [ | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/6/self_attn/q_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/6/self_attn/sinks", | |
| "shape": [ | |
| 64 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/6/self_attn/v_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/6/self_attn/v_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/7/input_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/7/mlp/experts/down_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/7/mlp/experts/down_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/7/mlp/experts/gate_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/7/mlp/experts/gate_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/7/mlp/experts/up_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/7/mlp/experts/up_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/7/mlp/router/bias", | |
| "shape": [ | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/7/mlp/router/kernel", | |
| "shape": [ | |
| 2880, | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/7/post_attention_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/7/self_attn/k_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/7/self_attn/k_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/7/self_attn/o_proj/bias", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/7/self_attn/o_proj/kernel", | |
| "shape": [ | |
| 4096, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/7/self_attn/q_proj/bias", | |
| "shape": [ | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/7/self_attn/q_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/7/self_attn/sinks", | |
| "shape": [ | |
| 64 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/7/self_attn/v_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/7/self_attn/v_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/8/input_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/8/mlp/experts/down_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/8/mlp/experts/down_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/8/mlp/experts/gate_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/8/mlp/experts/gate_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/8/mlp/experts/up_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/8/mlp/experts/up_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/8/mlp/router/bias", | |
| "shape": [ | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/8/mlp/router/kernel", | |
| "shape": [ | |
| 2880, | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/8/post_attention_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/8/self_attn/k_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/8/self_attn/k_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/8/self_attn/o_proj/bias", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/8/self_attn/o_proj/kernel", | |
| "shape": [ | |
| 4096, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/8/self_attn/q_proj/bias", | |
| "shape": [ | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/8/self_attn/q_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/8/self_attn/sinks", | |
| "shape": [ | |
| 64 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/8/self_attn/v_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/8/self_attn/v_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/9/input_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/9/mlp/experts/down_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/9/mlp/experts/down_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/9/mlp/experts/gate_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/9/mlp/experts/gate_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/9/mlp/experts/up_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/9/mlp/experts/up_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/9/mlp/router/bias", | |
| "shape": [ | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/9/mlp/router/kernel", | |
| "shape": [ | |
| 2880, | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/9/post_attention_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/9/self_attn/k_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/9/self_attn/k_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/9/self_attn/o_proj/bias", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/9/self_attn/o_proj/kernel", | |
| "shape": [ | |
| 4096, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/9/self_attn/q_proj/bias", | |
| "shape": [ | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/9/self_attn/q_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/9/self_attn/sinks", | |
| "shape": [ | |
| 64 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/9/self_attn/v_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/9/self_attn/v_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/10/input_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/10/mlp/experts/down_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/10/mlp/experts/down_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/10/mlp/experts/gate_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/10/mlp/experts/gate_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/10/mlp/experts/up_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/10/mlp/experts/up_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/10/mlp/router/bias", | |
| "shape": [ | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/10/mlp/router/kernel", | |
| "shape": [ | |
| 2880, | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/10/post_attention_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/10/self_attn/k_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/10/self_attn/k_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/10/self_attn/o_proj/bias", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/10/self_attn/o_proj/kernel", | |
| "shape": [ | |
| 4096, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/10/self_attn/q_proj/bias", | |
| "shape": [ | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/10/self_attn/q_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/10/self_attn/sinks", | |
| "shape": [ | |
| 64 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/10/self_attn/v_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/10/self_attn/v_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/11/input_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/11/mlp/experts/down_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/11/mlp/experts/down_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/11/mlp/experts/gate_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/11/mlp/experts/gate_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/11/mlp/experts/up_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/11/mlp/experts/up_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/11/mlp/router/bias", | |
| "shape": [ | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/11/mlp/router/kernel", | |
| "shape": [ | |
| 2880, | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/11/post_attention_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/11/self_attn/k_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/11/self_attn/k_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/11/self_attn/o_proj/bias", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/11/self_attn/o_proj/kernel", | |
| "shape": [ | |
| 4096, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/11/self_attn/q_proj/bias", | |
| "shape": [ | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/11/self_attn/q_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/11/self_attn/sinks", | |
| "shape": [ | |
| 64 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/11/self_attn/v_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/11/self_attn/v_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/12/input_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/12/mlp/experts/down_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/12/mlp/experts/down_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/12/mlp/experts/gate_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/12/mlp/experts/gate_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/12/mlp/experts/up_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/12/mlp/experts/up_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/12/mlp/router/bias", | |
| "shape": [ | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/12/mlp/router/kernel", | |
| "shape": [ | |
| 2880, | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/12/post_attention_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/12/self_attn/k_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/12/self_attn/k_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/12/self_attn/o_proj/bias", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/12/self_attn/o_proj/kernel", | |
| "shape": [ | |
| 4096, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/12/self_attn/q_proj/bias", | |
| "shape": [ | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/12/self_attn/q_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/12/self_attn/sinks", | |
| "shape": [ | |
| 64 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/12/self_attn/v_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/12/self_attn/v_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/13/input_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/13/mlp/experts/down_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/13/mlp/experts/down_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/13/mlp/experts/gate_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/13/mlp/experts/gate_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/13/mlp/experts/up_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/13/mlp/experts/up_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/13/mlp/router/bias", | |
| "shape": [ | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/13/mlp/router/kernel", | |
| "shape": [ | |
| 2880, | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/13/post_attention_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/13/self_attn/k_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/13/self_attn/k_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/13/self_attn/o_proj/bias", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/13/self_attn/o_proj/kernel", | |
| "shape": [ | |
| 4096, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/13/self_attn/q_proj/bias", | |
| "shape": [ | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/13/self_attn/q_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/13/self_attn/sinks", | |
| "shape": [ | |
| 64 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/13/self_attn/v_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/13/self_attn/v_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/14/input_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/14/mlp/experts/down_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/14/mlp/experts/down_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/14/mlp/experts/gate_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/14/mlp/experts/gate_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/14/mlp/experts/up_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/14/mlp/experts/up_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/14/mlp/router/bias", | |
| "shape": [ | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/14/mlp/router/kernel", | |
| "shape": [ | |
| 2880, | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/14/post_attention_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/14/self_attn/k_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/14/self_attn/k_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/14/self_attn/o_proj/bias", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/14/self_attn/o_proj/kernel", | |
| "shape": [ | |
| 4096, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/14/self_attn/q_proj/bias", | |
| "shape": [ | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/14/self_attn/q_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/14/self_attn/sinks", | |
| "shape": [ | |
| 64 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/14/self_attn/v_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/14/self_attn/v_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/15/input_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/15/mlp/experts/down_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/15/mlp/experts/down_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/15/mlp/experts/gate_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/15/mlp/experts/gate_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/15/mlp/experts/up_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/15/mlp/experts/up_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/15/mlp/router/bias", | |
| "shape": [ | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/15/mlp/router/kernel", | |
| "shape": [ | |
| 2880, | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/15/post_attention_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/15/self_attn/k_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/15/self_attn/k_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/15/self_attn/o_proj/bias", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/15/self_attn/o_proj/kernel", | |
| "shape": [ | |
| 4096, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/15/self_attn/q_proj/bias", | |
| "shape": [ | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/15/self_attn/q_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/15/self_attn/sinks", | |
| "shape": [ | |
| 64 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/15/self_attn/v_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/15/self_attn/v_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/16/input_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/16/mlp/experts/down_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/16/mlp/experts/down_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/16/mlp/experts/gate_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/16/mlp/experts/gate_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/16/mlp/experts/up_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/16/mlp/experts/up_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/16/mlp/router/bias", | |
| "shape": [ | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/16/mlp/router/kernel", | |
| "shape": [ | |
| 2880, | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/16/post_attention_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/16/self_attn/k_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/16/self_attn/k_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/16/self_attn/o_proj/bias", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/16/self_attn/o_proj/kernel", | |
| "shape": [ | |
| 4096, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/16/self_attn/q_proj/bias", | |
| "shape": [ | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/16/self_attn/q_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/16/self_attn/sinks", | |
| "shape": [ | |
| 64 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/16/self_attn/v_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/16/self_attn/v_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/17/input_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/17/mlp/experts/down_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/17/mlp/experts/down_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/17/mlp/experts/gate_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/17/mlp/experts/gate_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/17/mlp/experts/up_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/17/mlp/experts/up_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/17/mlp/router/bias", | |
| "shape": [ | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/17/mlp/router/kernel", | |
| "shape": [ | |
| 2880, | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/17/post_attention_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/17/self_attn/k_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/17/self_attn/k_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/17/self_attn/o_proj/bias", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/17/self_attn/o_proj/kernel", | |
| "shape": [ | |
| 4096, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/17/self_attn/q_proj/bias", | |
| "shape": [ | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/17/self_attn/q_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/17/self_attn/sinks", | |
| "shape": [ | |
| 64 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/17/self_attn/v_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/17/self_attn/v_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/18/input_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/18/mlp/experts/down_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/18/mlp/experts/down_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/18/mlp/experts/gate_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/18/mlp/experts/gate_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/18/mlp/experts/up_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/18/mlp/experts/up_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/18/mlp/router/bias", | |
| "shape": [ | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/18/mlp/router/kernel", | |
| "shape": [ | |
| 2880, | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/18/post_attention_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/18/self_attn/k_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/18/self_attn/k_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/18/self_attn/o_proj/bias", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/18/self_attn/o_proj/kernel", | |
| "shape": [ | |
| 4096, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/18/self_attn/q_proj/bias", | |
| "shape": [ | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/18/self_attn/q_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/18/self_attn/sinks", | |
| "shape": [ | |
| 64 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/18/self_attn/v_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/18/self_attn/v_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/19/input_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/19/mlp/experts/down_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/19/mlp/experts/down_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/19/mlp/experts/gate_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/19/mlp/experts/gate_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/19/mlp/experts/up_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/19/mlp/experts/up_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/19/mlp/router/bias", | |
| "shape": [ | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/19/mlp/router/kernel", | |
| "shape": [ | |
| 2880, | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/19/post_attention_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/19/self_attn/k_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/19/self_attn/k_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/19/self_attn/o_proj/bias", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/19/self_attn/o_proj/kernel", | |
| "shape": [ | |
| 4096, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/19/self_attn/q_proj/bias", | |
| "shape": [ | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/19/self_attn/q_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/19/self_attn/sinks", | |
| "shape": [ | |
| 64 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/19/self_attn/v_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/19/self_attn/v_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/20/input_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/20/mlp/experts/down_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/20/mlp/experts/down_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/20/mlp/experts/gate_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/20/mlp/experts/gate_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/20/mlp/experts/up_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/20/mlp/experts/up_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/20/mlp/router/bias", | |
| "shape": [ | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/20/mlp/router/kernel", | |
| "shape": [ | |
| 2880, | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/20/post_attention_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/20/self_attn/k_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/20/self_attn/k_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/20/self_attn/o_proj/bias", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/20/self_attn/o_proj/kernel", | |
| "shape": [ | |
| 4096, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/20/self_attn/q_proj/bias", | |
| "shape": [ | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/20/self_attn/q_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/20/self_attn/sinks", | |
| "shape": [ | |
| 64 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/20/self_attn/v_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/20/self_attn/v_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/21/input_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/21/mlp/experts/down_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/21/mlp/experts/down_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/21/mlp/experts/gate_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/21/mlp/experts/gate_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/21/mlp/experts/up_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/21/mlp/experts/up_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/21/mlp/router/bias", | |
| "shape": [ | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/21/mlp/router/kernel", | |
| "shape": [ | |
| 2880, | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/21/post_attention_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/21/self_attn/k_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/21/self_attn/k_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/21/self_attn/o_proj/bias", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/21/self_attn/o_proj/kernel", | |
| "shape": [ | |
| 4096, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/21/self_attn/q_proj/bias", | |
| "shape": [ | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/21/self_attn/q_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/21/self_attn/sinks", | |
| "shape": [ | |
| 64 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/21/self_attn/v_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/21/self_attn/v_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/22/input_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/22/mlp/experts/down_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/22/mlp/experts/down_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/22/mlp/experts/gate_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/22/mlp/experts/gate_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/22/mlp/experts/up_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/22/mlp/experts/up_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/22/mlp/router/bias", | |
| "shape": [ | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/22/mlp/router/kernel", | |
| "shape": [ | |
| 2880, | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/22/post_attention_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/22/self_attn/k_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/22/self_attn/k_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/22/self_attn/o_proj/bias", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/22/self_attn/o_proj/kernel", | |
| "shape": [ | |
| 4096, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/22/self_attn/q_proj/bias", | |
| "shape": [ | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/22/self_attn/q_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/22/self_attn/sinks", | |
| "shape": [ | |
| 64 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/22/self_attn/v_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/22/self_attn/v_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/23/input_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/23/mlp/experts/down_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/23/mlp/experts/down_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/23/mlp/experts/gate_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/23/mlp/experts/gate_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/23/mlp/experts/up_proj/bias", | |
| "shape": [ | |
| 32, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/23/mlp/experts/up_proj/kernel", | |
| "shape": [ | |
| 32, | |
| 2880, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/23/mlp/router/bias", | |
| "shape": [ | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/23/mlp/router/kernel", | |
| "shape": [ | |
| 2880, | |
| 32 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/23/post_attention_layernorm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/23/self_attn/k_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/23/self_attn/k_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/23/self_attn/o_proj/bias", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/23/self_attn/o_proj/kernel", | |
| "shape": [ | |
| 4096, | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/23/self_attn/q_proj/bias", | |
| "shape": [ | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/23/self_attn/q_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 4096 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/23/self_attn/sinks", | |
| "shape": [ | |
| 64 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/23/self_attn/v_proj/bias", | |
| "shape": [ | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/layers/23/self_attn/v_proj/kernel", | |
| "shape": [ | |
| 2880, | |
| 512 | |
| ], | |
| "dtype": "bfloat16" | |
| }, | |
| { | |
| "path": "model/model/norm/kernel", | |
| "shape": [ | |
| 2880 | |
| ], | |
| "dtype": "bfloat16" | |
| } | |
| ] | |
| } | |
| } |