{
  "algorithm": "dspark",
  "base_model": "CohereLabs/North-Mini-Code-1.0-w4a16",
  "checkpoint_epoch": 4,
  "default_speculative_tokens": 4,
  "largest_tensors": [
    {
      "dtype": "BF16",
      "name": "embed_tokens.weight",
      "parameters": 536870912,
      "shape": [
        262144,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "markov_head.markov_w1.weight",
      "parameters": 67108864,
      "shape": [
        262144,
        256
      ]
    },
    {
      "dtype": "BF16",
      "name": "lm_head.weight",
      "parameters": 65536000,
      "shape": [
        32000,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "fc.weight",
      "parameters": 12582912,
      "shape": [
        2048,
        6144
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.0.self_attn.o_proj.weight",
      "parameters": 8388608,
      "shape": [
        2048,
        4096
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.0.self_attn.q_proj.weight",
      "parameters": 8388608,
      "shape": [
        4096,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.1.self_attn.o_proj.weight",
      "parameters": 8388608,
      "shape": [
        2048,
        4096
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.1.self_attn.q_proj.weight",
      "parameters": 8388608,
      "shape": [
        4096,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.2.self_attn.o_proj.weight",
      "parameters": 8388608,
      "shape": [
        2048,
        4096
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.2.self_attn.q_proj.weight",
      "parameters": 8388608,
      "shape": [
        4096,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.3.self_attn.o_proj.weight",
      "parameters": 8388608,
      "shape": [
        2048,
        4096
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.3.self_attn.q_proj.weight",
      "parameters": 8388608,
      "shape": [
        4096,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.4.self_attn.o_proj.weight",
      "parameters": 8388608,
      "shape": [
        2048,
        4096
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.4.self_attn.q_proj.weight",
      "parameters": 8388608,
      "shape": [
        4096,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "markov_head.markov_w2.weight",
      "parameters": 8192000,
      "shape": [
        32000,
        256
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.0.mlp.down_proj.weight",
      "parameters": 1572864,
      "shape": [
        2048,
        768
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.0.mlp.gate_proj.weight",
      "parameters": 1572864,
      "shape": [
        768,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.0.mlp.up_proj.weight",
      "parameters": 1572864,
      "shape": [
        768,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.1.mlp.down_proj.weight",
      "parameters": 1572864,
      "shape": [
        2048,
        768
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.1.mlp.gate_proj.weight",
      "parameters": 1572864,
      "shape": [
        768,
        2048
      ]
    }
  ],
  "model_family": "North Mini Code speculative draft",
  "model_file": {
    "name": "model.safetensors",
    "safetensors_metadata": {
      "format": "pt"
    },
    "sha256": "64fd7d60c1c5a91d030dae0b7300bf8593b536b0a5569f53db7209e4a5f5b7f6",
    "size_bytes": 1617092434
  },
  "parameter_count": 808577793,
  "parameters_by_dtype": {
    "BF16": 808283649,
    "BOOL": 262144,
    "I64": 32000
  },
  "schema_version": 1,
  "tensor_count": 66,
  "tensors": [
    {
      "dtype": "BF16",
      "name": "confidence_head.proj.bias",
      "parameters": 1,
      "shape": [
        1
      ]
    },
    {
      "dtype": "BF16",
      "name": "confidence_head.proj.weight",
      "parameters": 2304,
      "shape": [
        1,
        2304
      ]
    },
    {
      "dtype": "I64",
      "name": "d2t",
      "parameters": 32000,
      "shape": [
        32000
      ]
    },
    {
      "dtype": "BF16",
      "name": "embed_tokens.weight",
      "parameters": 536870912,
      "shape": [
        262144,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "fc.weight",
      "parameters": 12582912,
      "shape": [
        2048,
        6144
      ]
    },
    {
      "dtype": "BF16",
      "name": "hidden_norm.weight",
      "parameters": 2048,
      "shape": [
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.0.input_layernorm.weight",
      "parameters": 2048,
      "shape": [
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.0.mlp.down_proj.weight",
      "parameters": 1572864,
      "shape": [
        2048,
        768
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.0.mlp.gate_proj.weight",
      "parameters": 1572864,
      "shape": [
        768,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.0.mlp.up_proj.weight",
      "parameters": 1572864,
      "shape": [
        768,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.0.post_attention_layernorm.weight",
      "parameters": 2048,
      "shape": [
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.0.self_attn.k_norm.weight",
      "parameters": 128,
      "shape": [
        128
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.0.self_attn.k_proj.weight",
      "parameters": 1048576,
      "shape": [
        512,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.0.self_attn.o_proj.weight",
      "parameters": 8388608,
      "shape": [
        2048,
        4096
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.0.self_attn.q_norm.weight",
      "parameters": 128,
      "shape": [
        128
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.0.self_attn.q_proj.weight",
      "parameters": 8388608,
      "shape": [
        4096,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.0.self_attn.v_proj.weight",
      "parameters": 1048576,
      "shape": [
        512,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.1.input_layernorm.weight",
      "parameters": 2048,
      "shape": [
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.1.mlp.down_proj.weight",
      "parameters": 1572864,
      "shape": [
        2048,
        768
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.1.mlp.gate_proj.weight",
      "parameters": 1572864,
      "shape": [
        768,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.1.mlp.up_proj.weight",
      "parameters": 1572864,
      "shape": [
        768,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.1.post_attention_layernorm.weight",
      "parameters": 2048,
      "shape": [
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.1.self_attn.k_norm.weight",
      "parameters": 128,
      "shape": [
        128
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.1.self_attn.k_proj.weight",
      "parameters": 1048576,
      "shape": [
        512,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.1.self_attn.o_proj.weight",
      "parameters": 8388608,
      "shape": [
        2048,
        4096
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.1.self_attn.q_norm.weight",
      "parameters": 128,
      "shape": [
        128
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.1.self_attn.q_proj.weight",
      "parameters": 8388608,
      "shape": [
        4096,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.1.self_attn.v_proj.weight",
      "parameters": 1048576,
      "shape": [
        512,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.2.input_layernorm.weight",
      "parameters": 2048,
      "shape": [
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.2.mlp.down_proj.weight",
      "parameters": 1572864,
      "shape": [
        2048,
        768
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.2.mlp.gate_proj.weight",
      "parameters": 1572864,
      "shape": [
        768,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.2.mlp.up_proj.weight",
      "parameters": 1572864,
      "shape": [
        768,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.2.post_attention_layernorm.weight",
      "parameters": 2048,
      "shape": [
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.2.self_attn.k_norm.weight",
      "parameters": 128,
      "shape": [
        128
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.2.self_attn.k_proj.weight",
      "parameters": 1048576,
      "shape": [
        512,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.2.self_attn.o_proj.weight",
      "parameters": 8388608,
      "shape": [
        2048,
        4096
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.2.self_attn.q_norm.weight",
      "parameters": 128,
      "shape": [
        128
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.2.self_attn.q_proj.weight",
      "parameters": 8388608,
      "shape": [
        4096,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.2.self_attn.v_proj.weight",
      "parameters": 1048576,
      "shape": [
        512,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.3.input_layernorm.weight",
      "parameters": 2048,
      "shape": [
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.3.mlp.down_proj.weight",
      "parameters": 1572864,
      "shape": [
        2048,
        768
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.3.mlp.gate_proj.weight",
      "parameters": 1572864,
      "shape": [
        768,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.3.mlp.up_proj.weight",
      "parameters": 1572864,
      "shape": [
        768,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.3.post_attention_layernorm.weight",
      "parameters": 2048,
      "shape": [
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.3.self_attn.k_norm.weight",
      "parameters": 128,
      "shape": [
        128
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.3.self_attn.k_proj.weight",
      "parameters": 1048576,
      "shape": [
        512,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.3.self_attn.o_proj.weight",
      "parameters": 8388608,
      "shape": [
        2048,
        4096
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.3.self_attn.q_norm.weight",
      "parameters": 128,
      "shape": [
        128
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.3.self_attn.q_proj.weight",
      "parameters": 8388608,
      "shape": [
        4096,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.3.self_attn.v_proj.weight",
      "parameters": 1048576,
      "shape": [
        512,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.4.input_layernorm.weight",
      "parameters": 2048,
      "shape": [
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.4.mlp.down_proj.weight",
      "parameters": 1572864,
      "shape": [
        2048,
        768
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.4.mlp.gate_proj.weight",
      "parameters": 1572864,
      "shape": [
        768,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.4.mlp.up_proj.weight",
      "parameters": 1572864,
      "shape": [
        768,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.4.post_attention_layernorm.weight",
      "parameters": 2048,
      "shape": [
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.4.self_attn.k_norm.weight",
      "parameters": 128,
      "shape": [
        128
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.4.self_attn.k_proj.weight",
      "parameters": 1048576,
      "shape": [
        512,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.4.self_attn.o_proj.weight",
      "parameters": 8388608,
      "shape": [
        2048,
        4096
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.4.self_attn.q_norm.weight",
      "parameters": 128,
      "shape": [
        128
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.4.self_attn.q_proj.weight",
      "parameters": 8388608,
      "shape": [
        4096,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "layers.4.self_attn.v_proj.weight",
      "parameters": 1048576,
      "shape": [
        512,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "lm_head.weight",
      "parameters": 65536000,
      "shape": [
        32000,
        2048
      ]
    },
    {
      "dtype": "BF16",
      "name": "markov_head.markov_w1.weight",
      "parameters": 67108864,
      "shape": [
        262144,
        256
      ]
    },
    {
      "dtype": "BF16",
      "name": "markov_head.markov_w2.weight",
      "parameters": 8192000,
      "shape": [
        32000,
        256
      ]
    },
    {
      "dtype": "BF16",
      "name": "norm.weight",
      "parameters": 2048,
      "shape": [
        2048
      ]
    },
    {
      "dtype": "BOOL",
      "name": "t2d",
      "parameters": 262144,
      "shape": [
        262144
      ]
    }
  ]
}