| { |
| "algorithm": "dspark", |
| "base_model": "CohereLabs/North-Mini-Code-1.0-w4a16", |
| "checkpoint_epoch": 4, |
| "default_speculative_tokens": 4, |
| "largest_tensors": [ |
| { |
| "dtype": "BF16", |
| "name": "embed_tokens.weight", |
| "parameters": 536870912, |
| "shape": [ |
| 262144, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "markov_head.markov_w1.weight", |
| "parameters": 67108864, |
| "shape": [ |
| 262144, |
| 256 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "lm_head.weight", |
| "parameters": 65536000, |
| "shape": [ |
| 32000, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "fc.weight", |
| "parameters": 12582912, |
| "shape": [ |
| 2048, |
| 6144 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.self_attn.o_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 2048, |
| 4096 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.self_attn.q_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 4096, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.self_attn.o_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 2048, |
| 4096 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.self_attn.q_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 4096, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.self_attn.o_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 2048, |
| 4096 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.self_attn.q_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 4096, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.self_attn.o_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 2048, |
| 4096 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.self_attn.q_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 4096, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.self_attn.o_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 2048, |
| 4096 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.self_attn.q_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 4096, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "markov_head.markov_w2.weight", |
| "parameters": 8192000, |
| "shape": [ |
| 32000, |
| 256 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.mlp.down_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 2048, |
| 768 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.mlp.gate_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.mlp.up_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.mlp.down_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 2048, |
| 768 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.mlp.gate_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| } |
| ], |
| "model_family": "North Mini Code speculative draft", |
| "model_file": { |
| "name": "model.safetensors", |
| "safetensors_metadata": { |
| "format": "pt" |
| }, |
| "sha256": "64fd7d60c1c5a91d030dae0b7300bf8593b536b0a5569f53db7209e4a5f5b7f6", |
| "size_bytes": 1617092434 |
| }, |
| "parameter_count": 808577793, |
| "parameters_by_dtype": { |
| "BF16": 808283649, |
| "BOOL": 262144, |
| "I64": 32000 |
| }, |
| "schema_version": 1, |
| "tensor_count": 66, |
| "tensors": [ |
| { |
| "dtype": "BF16", |
| "name": "confidence_head.proj.bias", |
| "parameters": 1, |
| "shape": [ |
| 1 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "confidence_head.proj.weight", |
| "parameters": 2304, |
| "shape": [ |
| 1, |
| 2304 |
| ] |
| }, |
| { |
| "dtype": "I64", |
| "name": "d2t", |
| "parameters": 32000, |
| "shape": [ |
| 32000 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "embed_tokens.weight", |
| "parameters": 536870912, |
| "shape": [ |
| 262144, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "fc.weight", |
| "parameters": 12582912, |
| "shape": [ |
| 2048, |
| 6144 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "hidden_norm.weight", |
| "parameters": 2048, |
| "shape": [ |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.input_layernorm.weight", |
| "parameters": 2048, |
| "shape": [ |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.mlp.down_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 2048, |
| 768 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.mlp.gate_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.mlp.up_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.post_attention_layernorm.weight", |
| "parameters": 2048, |
| "shape": [ |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.self_attn.k_norm.weight", |
| "parameters": 128, |
| "shape": [ |
| 128 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.self_attn.k_proj.weight", |
| "parameters": 1048576, |
| "shape": [ |
| 512, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.self_attn.o_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 2048, |
| 4096 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.self_attn.q_norm.weight", |
| "parameters": 128, |
| "shape": [ |
| 128 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.self_attn.q_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 4096, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.0.self_attn.v_proj.weight", |
| "parameters": 1048576, |
| "shape": [ |
| 512, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.input_layernorm.weight", |
| "parameters": 2048, |
| "shape": [ |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.mlp.down_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 2048, |
| 768 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.mlp.gate_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.mlp.up_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.post_attention_layernorm.weight", |
| "parameters": 2048, |
| "shape": [ |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.self_attn.k_norm.weight", |
| "parameters": 128, |
| "shape": [ |
| 128 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.self_attn.k_proj.weight", |
| "parameters": 1048576, |
| "shape": [ |
| 512, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.self_attn.o_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 2048, |
| 4096 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.self_attn.q_norm.weight", |
| "parameters": 128, |
| "shape": [ |
| 128 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.self_attn.q_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 4096, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.1.self_attn.v_proj.weight", |
| "parameters": 1048576, |
| "shape": [ |
| 512, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.input_layernorm.weight", |
| "parameters": 2048, |
| "shape": [ |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.mlp.down_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 2048, |
| 768 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.mlp.gate_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.mlp.up_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.post_attention_layernorm.weight", |
| "parameters": 2048, |
| "shape": [ |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.self_attn.k_norm.weight", |
| "parameters": 128, |
| "shape": [ |
| 128 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.self_attn.k_proj.weight", |
| "parameters": 1048576, |
| "shape": [ |
| 512, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.self_attn.o_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 2048, |
| 4096 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.self_attn.q_norm.weight", |
| "parameters": 128, |
| "shape": [ |
| 128 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.self_attn.q_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 4096, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.2.self_attn.v_proj.weight", |
| "parameters": 1048576, |
| "shape": [ |
| 512, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.input_layernorm.weight", |
| "parameters": 2048, |
| "shape": [ |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.mlp.down_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 2048, |
| 768 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.mlp.gate_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.mlp.up_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.post_attention_layernorm.weight", |
| "parameters": 2048, |
| "shape": [ |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.self_attn.k_norm.weight", |
| "parameters": 128, |
| "shape": [ |
| 128 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.self_attn.k_proj.weight", |
| "parameters": 1048576, |
| "shape": [ |
| 512, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.self_attn.o_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 2048, |
| 4096 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.self_attn.q_norm.weight", |
| "parameters": 128, |
| "shape": [ |
| 128 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.self_attn.q_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 4096, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.3.self_attn.v_proj.weight", |
| "parameters": 1048576, |
| "shape": [ |
| 512, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.input_layernorm.weight", |
| "parameters": 2048, |
| "shape": [ |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.mlp.down_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 2048, |
| 768 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.mlp.gate_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.mlp.up_proj.weight", |
| "parameters": 1572864, |
| "shape": [ |
| 768, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.post_attention_layernorm.weight", |
| "parameters": 2048, |
| "shape": [ |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.self_attn.k_norm.weight", |
| "parameters": 128, |
| "shape": [ |
| 128 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.self_attn.k_proj.weight", |
| "parameters": 1048576, |
| "shape": [ |
| 512, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.self_attn.o_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 2048, |
| 4096 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.self_attn.q_norm.weight", |
| "parameters": 128, |
| "shape": [ |
| 128 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.self_attn.q_proj.weight", |
| "parameters": 8388608, |
| "shape": [ |
| 4096, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "layers.4.self_attn.v_proj.weight", |
| "parameters": 1048576, |
| "shape": [ |
| 512, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "lm_head.weight", |
| "parameters": 65536000, |
| "shape": [ |
| 32000, |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "markov_head.markov_w1.weight", |
| "parameters": 67108864, |
| "shape": [ |
| 262144, |
| 256 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "markov_head.markov_w2.weight", |
| "parameters": 8192000, |
| "shape": [ |
| 32000, |
| 256 |
| ] |
| }, |
| { |
| "dtype": "BF16", |
| "name": "norm.weight", |
| "parameters": 2048, |
| "shape": [ |
| 2048 |
| ] |
| }, |
| { |
| "dtype": "BOOL", |
| "name": "t2d", |
| "parameters": 262144, |
| "shape": [ |
| 262144 |
| ] |
| } |
| ] |
| } |
|
|
