{
  "average_bits": 3.4999670749374423,
  "embed_bits": 8,
  "embed_uses_tied_head": false,
  "embed_reason": "The decision readout is last_hidden @ readout.T and does not use the tied lm_head. The embedding is input-only and is stored as 8-bit affine, group 64.",
  "embed_sum_kl": {
    "4": 0.055970916987464966,
    "8": 0.010904520401237319
  },
  "embed_bytes": {
    "4": 143032320,
    "8": 270172160
  },
  "joint_refine": {
    "calibration_n": 0,
    "calibration_ids": [],
    "joint_kl_before": null,
    "joint_kl_after": null,
    "swaps": 0
  },
  "assignment": [
    {
      "path": "language_model.model.layers.0.linear_attn.in_proj_qkv",
      "layer": 0,
      "layer_type": "linear_attention",
      "nparams": 6291456,
      "bits": 4,
      "kind": "4",
      "kl": 0.04734761182665103
    },
    {
      "path": "language_model.model.layers.0.linear_attn.in_proj_z",
      "layer": 0,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 3,
      "kind": "3",
      "kl": 0.1388501915724426
    },
    {
      "path": "language_model.model.layers.0.linear_attn.in_proj_b",
      "layer": 0,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 4,
      "kind": "4",
      "kl": 0.01794408019266606
    },
    {
      "path": "language_model.model.layers.0.linear_attn.in_proj_a",
      "layer": 0,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 4,
      "kind": "4",
      "kl": 0.011710990232290872
    },
    {
      "path": "language_model.model.layers.0.linear_attn.out_proj",
      "layer": 0,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.4239300562384638
    },
    {
      "path": "language_model.model.layers.0.mlp.gate_proj",
      "layer": 0,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.06092327188142901
    },
    {
      "path": "language_model.model.layers.0.mlp.down_proj",
      "layer": 0,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.02344921252365501
    },
    {
      "path": "language_model.model.layers.0.mlp.up_proj",
      "layer": 0,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.027068575012547745
    },
    {
      "path": "language_model.model.layers.1.linear_attn.in_proj_qkv",
      "layer": 1,
      "layer_type": "linear_attention",
      "nparams": 6291456,
      "bits": 4,
      "kind": "4",
      "kl": 0.023629120959874177
    },
    {
      "path": "language_model.model.layers.1.linear_attn.in_proj_z",
      "layer": 1,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.025880121687977955
    },
    {
      "path": "language_model.model.layers.1.linear_attn.in_proj_b",
      "layer": 1,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 4,
      "kind": "4",
      "kl": 0.01083456115500153
    },
    {
      "path": "language_model.model.layers.1.linear_attn.in_proj_a",
      "layer": 1,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 4,
      "kind": "4",
      "kl": 0.01273320237789333
    },
    {
      "path": "language_model.model.layers.1.linear_attn.out_proj",
      "layer": 1,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.0927767372959222
    },
    {
      "path": "language_model.model.layers.1.mlp.gate_proj",
      "layer": 1,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.021329556389151568
    },
    {
      "path": "language_model.model.layers.1.mlp.down_proj",
      "layer": 1,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.02737624581982894
    },
    {
      "path": "language_model.model.layers.1.mlp.up_proj",
      "layer": 1,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.037710730514948596
    },
    {
      "path": "language_model.model.layers.2.linear_attn.in_proj_qkv",
      "layer": 2,
      "layer_type": "linear_attention",
      "nparams": 6291456,
      "bits": 4,
      "kind": "4",
      "kl": 0.05043157109883073
    },
    {
      "path": "language_model.model.layers.2.linear_attn.in_proj_z",
      "layer": 2,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.03697225452164565
    },
    {
      "path": "language_model.model.layers.2.linear_attn.in_proj_b",
      "layer": 2,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 4,
      "kind": "4",
      "kl": 0.013773699583866855
    },
    {
      "path": "language_model.model.layers.2.linear_attn.in_proj_a",
      "layer": 2,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 3,
      "kind": "3",
      "kl": 0.011592477724978621
    },
    {
      "path": "language_model.model.layers.2.linear_attn.out_proj",
      "layer": 2,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.09072199036986346
    },
    {
      "path": "language_model.model.layers.2.mlp.gate_proj",
      "layer": 2,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.04075199032161302
    },
    {
      "path": "language_model.model.layers.2.mlp.down_proj",
      "layer": 2,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.057830560223890416
    },
    {
      "path": "language_model.model.layers.2.mlp.up_proj",
      "layer": 2,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.03367332028470472
    },
    {
      "path": "language_model.model.layers.3.self_attn.q_proj",
      "layer": 3,
      "layer_type": "full_attention",
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.0363731313499352
    },
    {
      "path": "language_model.model.layers.3.self_attn.k_proj",
      "layer": 3,
      "layer_type": "full_attention",
      "nparams": 524288,
      "bits": 4,
      "kind": "4",
      "kl": 0.017387628331155
    },
    {
      "path": "language_model.model.layers.3.self_attn.v_proj",
      "layer": 3,
      "layer_type": "full_attention",
      "nparams": 524288,
      "bits": 4,
      "kind": "4",
      "kl": 0.03509479391362726
    },
    {
      "path": "language_model.model.layers.3.self_attn.o_proj",
      "layer": 3,
      "layer_type": "full_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.047125915910451774
    },
    {
      "path": "language_model.model.layers.3.mlp.gate_proj",
      "layer": 3,
      "layer_type": "full_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.019515000915455945
    },
    {
      "path": "language_model.model.layers.3.mlp.down_proj",
      "layer": 3,
      "layer_type": "full_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.061700401195738386
    },
    {
      "path": "language_model.model.layers.3.mlp.up_proj",
      "layer": 3,
      "layer_type": "full_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.042194126852922675
    },
    {
      "path": "language_model.model.layers.4.linear_attn.in_proj_qkv",
      "layer": 4,
      "layer_type": "linear_attention",
      "nparams": 6291456,
      "bits": 4,
      "kind": "4",
      "kl": 0.02397921018909746
    },
    {
      "path": "language_model.model.layers.4.linear_attn.in_proj_z",
      "layer": 4,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.023035051569588204
    },
    {
      "path": "language_model.model.layers.4.linear_attn.in_proj_b",
      "layer": 4,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 4,
      "kind": "4",
      "kl": 0.01564397704726879
    },
    {
      "path": "language_model.model.layers.4.linear_attn.in_proj_a",
      "layer": 4,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 2,
      "kind": "ternary",
      "kl": 0.013407841441155961
    },
    {
      "path": "language_model.model.layers.4.linear_attn.out_proj",
      "layer": 4,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.023875757661911464
    },
    {
      "path": "language_model.model.layers.4.mlp.gate_proj",
      "layer": 4,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.031587638719582915
    },
    {
      "path": "language_model.model.layers.4.mlp.down_proj",
      "layer": 4,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.136789628508474
    },
    {
      "path": "language_model.model.layers.4.mlp.up_proj",
      "layer": 4,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.03427242462143647
    },
    {
      "path": "language_model.model.layers.5.linear_attn.in_proj_qkv",
      "layer": 5,
      "layer_type": "linear_attention",
      "nparams": 6291456,
      "bits": 4,
      "kind": "4",
      "kl": 0.03755131517139272
    },
    {
      "path": "language_model.model.layers.5.linear_attn.in_proj_z",
      "layer": 5,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.02201721690262345
    },
    {
      "path": "language_model.model.layers.5.linear_attn.in_proj_b",
      "layer": 5,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 3,
      "kind": "3",
      "kl": 0.014129301163061457
    },
    {
      "path": "language_model.model.layers.5.linear_attn.in_proj_a",
      "layer": 5,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 4,
      "kind": "4",
      "kl": 0.00921678707939411
    },
    {
      "path": "language_model.model.layers.5.linear_attn.out_proj",
      "layer": 5,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.05138253773226259
    },
    {
      "path": "language_model.model.layers.5.mlp.gate_proj",
      "layer": 5,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.032154125558556325
    },
    {
      "path": "language_model.model.layers.5.mlp.down_proj",
      "layer": 5,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.043968335455040024
    },
    {
      "path": "language_model.model.layers.5.mlp.up_proj",
      "layer": 5,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.062463296864341974
    },
    {
      "path": "language_model.model.layers.6.linear_attn.in_proj_qkv",
      "layer": 6,
      "layer_type": "linear_attention",
      "nparams": 6291456,
      "bits": 4,
      "kind": "4",
      "kl": 0.051413092635799004
    },
    {
      "path": "language_model.model.layers.6.linear_attn.in_proj_z",
      "layer": 6,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.02528388093525213
    },
    {
      "path": "language_model.model.layers.6.linear_attn.in_proj_b",
      "layer": 6,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 3,
      "kind": "3",
      "kl": 0.010495090312282684
    },
    {
      "path": "language_model.model.layers.6.linear_attn.in_proj_a",
      "layer": 6,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 3,
      "kind": "3",
      "kl": 0.010463668713146171
    },
    {
      "path": "language_model.model.layers.6.linear_attn.out_proj",
      "layer": 6,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.03687874271655201
    },
    {
      "path": "language_model.model.layers.6.mlp.gate_proj",
      "layer": 6,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.041433529224225885
    },
    {
      "path": "language_model.model.layers.6.mlp.down_proj",
      "layer": 6,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.04199239106005209
    },
    {
      "path": "language_model.model.layers.6.mlp.up_proj",
      "layer": 6,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.09370311534083604
    },
    {
      "path": "language_model.model.layers.7.self_attn.q_proj",
      "layer": 7,
      "layer_type": "full_attention",
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.0639473471482219
    },
    {
      "path": "language_model.model.layers.7.self_attn.k_proj",
      "layer": 7,
      "layer_type": "full_attention",
      "nparams": 524288,
      "bits": 4,
      "kind": "4",
      "kl": 0.01995626164111499
    },
    {
      "path": "language_model.model.layers.7.self_attn.v_proj",
      "layer": 7,
      "layer_type": "full_attention",
      "nparams": 524288,
      "bits": 4,
      "kind": "4",
      "kl": 0.03879161901214778
    },
    {
      "path": "language_model.model.layers.7.self_attn.o_proj",
      "layer": 7,
      "layer_type": "full_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.07775133380157301
    },
    {
      "path": "language_model.model.layers.7.mlp.gate_proj",
      "layer": 7,
      "layer_type": "full_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.01902822349416639
    },
    {
      "path": "language_model.model.layers.7.mlp.down_proj",
      "layer": 7,
      "layer_type": "full_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.02074294201563859
    },
    {
      "path": "language_model.model.layers.7.mlp.up_proj",
      "layer": 7,
      "layer_type": "full_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.03005311131768014
    },
    {
      "path": "language_model.model.layers.8.linear_attn.in_proj_qkv",
      "layer": 8,
      "layer_type": "linear_attention",
      "nparams": 6291456,
      "bits": 4,
      "kind": "4",
      "kl": 0.026959423507237353
    },
    {
      "path": "language_model.model.layers.8.linear_attn.in_proj_z",
      "layer": 8,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.010542849449285302
    },
    {
      "path": "language_model.model.layers.8.linear_attn.in_proj_b",
      "layer": 8,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 4,
      "kind": "4",
      "kl": 0.006259536017413378
    },
    {
      "path": "language_model.model.layers.8.linear_attn.in_proj_a",
      "layer": 8,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 4,
      "kind": "4",
      "kl": 0.005066103889982274
    },
    {
      "path": "language_model.model.layers.8.linear_attn.out_proj",
      "layer": 8,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.04619141676349868
    },
    {
      "path": "language_model.model.layers.8.mlp.gate_proj",
      "layer": 8,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.019728178176647466
    },
    {
      "path": "language_model.model.layers.8.mlp.down_proj",
      "layer": 8,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.026803769680060083
    },
    {
      "path": "language_model.model.layers.8.mlp.up_proj",
      "layer": 8,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.022680045334707168
    },
    {
      "path": "language_model.model.layers.9.linear_attn.in_proj_qkv",
      "layer": 9,
      "layer_type": "linear_attention",
      "nparams": 6291456,
      "bits": 4,
      "kind": "4",
      "kl": 0.03976326973341923
    },
    {
      "path": "language_model.model.layers.9.linear_attn.in_proj_z",
      "layer": 9,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.018086428763740822
    },
    {
      "path": "language_model.model.layers.9.linear_attn.in_proj_b",
      "layer": 9,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 4,
      "kind": "4",
      "kl": 0.008665403679117296
    },
    {
      "path": "language_model.model.layers.9.linear_attn.in_proj_a",
      "layer": 9,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 4,
      "kind": "4",
      "kl": 0.005111378159239233
    },
    {
      "path": "language_model.model.layers.9.linear_attn.out_proj",
      "layer": 9,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.024027441586703807
    },
    {
      "path": "language_model.model.layers.9.mlp.gate_proj",
      "layer": 9,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.017517147359535766
    },
    {
      "path": "language_model.model.layers.9.mlp.down_proj",
      "layer": 9,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.018829855820785585
    },
    {
      "path": "language_model.model.layers.9.mlp.up_proj",
      "layer": 9,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.021916257691088493
    },
    {
      "path": "language_model.model.layers.10.linear_attn.in_proj_qkv",
      "layer": 10,
      "layer_type": "linear_attention",
      "nparams": 6291456,
      "bits": 4,
      "kind": "4",
      "kl": 0.04024679641347246
    },
    {
      "path": "language_model.model.layers.10.linear_attn.in_proj_z",
      "layer": 10,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.011116799937381346
    },
    {
      "path": "language_model.model.layers.10.linear_attn.in_proj_b",
      "layer": 10,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 4,
      "kind": "4",
      "kl": 0.0043433013822391404
    },
    {
      "path": "language_model.model.layers.10.linear_attn.in_proj_a",
      "layer": 10,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 3,
      "kind": "3",
      "kl": 0.004665172839471169
    },
    {
      "path": "language_model.model.layers.10.linear_attn.out_proj",
      "layer": 10,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.02778384028510572
    },
    {
      "path": "language_model.model.layers.10.mlp.gate_proj",
      "layer": 10,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.007522035321253345
    },
    {
      "path": "language_model.model.layers.10.mlp.down_proj",
      "layer": 10,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.015915166493531592
    },
    {
      "path": "language_model.model.layers.10.mlp.up_proj",
      "layer": 10,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.017182175372423135
    },
    {
      "path": "language_model.model.layers.11.self_attn.q_proj",
      "layer": 11,
      "layer_type": "full_attention",
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.01918938407635554
    },
    {
      "path": "language_model.model.layers.11.self_attn.k_proj",
      "layer": 11,
      "layer_type": "full_attention",
      "nparams": 524288,
      "bits": 4,
      "kind": "4",
      "kl": 0.02101637260234458
    },
    {
      "path": "language_model.model.layers.11.self_attn.v_proj",
      "layer": 11,
      "layer_type": "full_attention",
      "nparams": 524288,
      "bits": 4,
      "kind": "4",
      "kl": 0.03523801972554967
    },
    {
      "path": "language_model.model.layers.11.self_attn.o_proj",
      "layer": 11,
      "layer_type": "full_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.044128938397831746
    },
    {
      "path": "language_model.model.layers.11.mlp.gate_proj",
      "layer": 11,
      "layer_type": "full_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.006564071190068745
    },
    {
      "path": "language_model.model.layers.11.mlp.down_proj",
      "layer": 11,
      "layer_type": "full_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.009954753161174175
    },
    {
      "path": "language_model.model.layers.11.mlp.up_proj",
      "layer": 11,
      "layer_type": "full_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.016408592796186942
    },
    {
      "path": "language_model.model.layers.12.linear_attn.in_proj_qkv",
      "layer": 12,
      "layer_type": "linear_attention",
      "nparams": 6291456,
      "bits": 4,
      "kind": "4",
      "kl": 0.012744699638467517
    },
    {
      "path": "language_model.model.layers.12.linear_attn.in_proj_z",
      "layer": 12,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.004262488847540524
    },
    {
      "path": "language_model.model.layers.12.linear_attn.in_proj_b",
      "layer": 12,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 4,
      "kind": "4",
      "kl": 0.003491112939298953
    },
    {
      "path": "language_model.model.layers.12.linear_attn.in_proj_a",
      "layer": 12,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 4,
      "kind": "4",
      "kl": 0.0023025657529113326
    },
    {
      "path": "language_model.model.layers.12.linear_attn.out_proj",
      "layer": 12,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.006398780878283918
    },
    {
      "path": "language_model.model.layers.12.mlp.gate_proj",
      "layer": 12,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.005122177783788138
    },
    {
      "path": "language_model.model.layers.12.mlp.down_proj",
      "layer": 12,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.00533927442938409
    },
    {
      "path": "language_model.model.layers.12.mlp.up_proj",
      "layer": 12,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.0061679559347479485
    },
    {
      "path": "language_model.model.layers.13.linear_attn.in_proj_qkv",
      "layer": 13,
      "layer_type": "linear_attention",
      "nparams": 6291456,
      "bits": 4,
      "kind": "4",
      "kl": 0.011533852265464987
    },
    {
      "path": "language_model.model.layers.13.linear_attn.in_proj_z",
      "layer": 13,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.0028721824605509763
    },
    {
      "path": "language_model.model.layers.13.linear_attn.in_proj_b",
      "layer": 13,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 4,
      "kind": "4",
      "kl": 0.0023588539987408805
    },
    {
      "path": "language_model.model.layers.13.linear_attn.in_proj_a",
      "layer": 13,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 3,
      "kind": "3",
      "kl": 0.00199120487849583
    },
    {
      "path": "language_model.model.layers.13.linear_attn.out_proj",
      "layer": 13,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.004224189863711925
    },
    {
      "path": "language_model.model.layers.13.mlp.gate_proj",
      "layer": 13,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 2,
      "kind": "2",
      "kl": 0.012466730321177164
    },
    {
      "path": "language_model.model.layers.13.mlp.down_proj",
      "layer": 13,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.002558384258649209
    },
    {
      "path": "language_model.model.layers.13.mlp.up_proj",
      "layer": 13,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.0029687005528531204
    },
    {
      "path": "language_model.model.layers.14.linear_attn.in_proj_qkv",
      "layer": 14,
      "layer_type": "linear_attention",
      "nparams": 6291456,
      "bits": 4,
      "kind": "4",
      "kl": 0.007997483313712273
    },
    {
      "path": "language_model.model.layers.14.linear_attn.in_proj_z",
      "layer": 14,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 3,
      "kind": "3",
      "kl": 0.0052743370907232245
    },
    {
      "path": "language_model.model.layers.14.linear_attn.in_proj_b",
      "layer": 14,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 4,
      "kind": "4",
      "kl": 0.0024342861764566695
    },
    {
      "path": "language_model.model.layers.14.linear_attn.in_proj_a",
      "layer": 14,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 4,
      "kind": "4",
      "kl": 0.0016676004025067079
    },
    {
      "path": "language_model.model.layers.14.linear_attn.out_proj",
      "layer": 14,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.004083662991277806
    },
    {
      "path": "language_model.model.layers.14.mlp.gate_proj",
      "layer": 14,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 3,
      "kind": "3",
      "kl": 0.008108828858961267
    },
    {
      "path": "language_model.model.layers.14.mlp.down_proj",
      "layer": 14,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.004123490271222968
    },
    {
      "path": "language_model.model.layers.14.mlp.up_proj",
      "layer": 14,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 3,
      "kind": "3",
      "kl": 0.007036047053022412
    },
    {
      "path": "language_model.model.layers.15.self_attn.q_proj",
      "layer": 15,
      "layer_type": "full_attention",
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.004153576767074394
    },
    {
      "path": "language_model.model.layers.15.self_attn.k_proj",
      "layer": 15,
      "layer_type": "full_attention",
      "nparams": 524288,
      "bits": 4,
      "kind": "4",
      "kl": 0.004112759303064329
    },
    {
      "path": "language_model.model.layers.15.self_attn.v_proj",
      "layer": 15,
      "layer_type": "full_attention",
      "nparams": 524288,
      "bits": 4,
      "kind": "4",
      "kl": 0.0045544497693278275
    },
    {
      "path": "language_model.model.layers.15.self_attn.o_proj",
      "layer": 15,
      "layer_type": "full_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.003484462206984148
    },
    {
      "path": "language_model.model.layers.15.mlp.gate_proj",
      "layer": 15,
      "layer_type": "full_attention",
      "nparams": 3670016,
      "bits": 3,
      "kind": "3",
      "kl": 0.002868431546808957
    },
    {
      "path": "language_model.model.layers.15.mlp.down_proj",
      "layer": 15,
      "layer_type": "full_attention",
      "nparams": 3670016,
      "bits": 3,
      "kind": "3",
      "kl": 0.006095650806291535
    },
    {
      "path": "language_model.model.layers.15.mlp.up_proj",
      "layer": 15,
      "layer_type": "full_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.004815552077497851
    },
    {
      "path": "language_model.model.layers.16.linear_attn.in_proj_qkv",
      "layer": 16,
      "layer_type": "linear_attention",
      "nparams": 6291456,
      "bits": 3,
      "kind": "3",
      "kl": 0.00292960581312596
    },
    {
      "path": "language_model.model.layers.16.linear_attn.in_proj_z",
      "layer": 16,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 2,
      "kind": "2",
      "kl": 0.0034485573524671727
    },
    {
      "path": "language_model.model.layers.16.linear_attn.in_proj_b",
      "layer": 16,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 3,
      "kind": "3",
      "kl": 0.0012654082102243791
    },
    {
      "path": "language_model.model.layers.16.linear_attn.in_proj_a",
      "layer": 16,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 2,
      "kind": "2",
      "kl": 0.0012813463656705713
    },
    {
      "path": "language_model.model.layers.16.linear_attn.out_proj",
      "layer": 16,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 3,
      "kind": "3",
      "kl": 0.005193882758604845
    },
    {
      "path": "language_model.model.layers.16.mlp.gate_proj",
      "layer": 16,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 3,
      "kind": "3",
      "kl": 0.002351040174577789
    },
    {
      "path": "language_model.model.layers.16.mlp.down_proj",
      "layer": 16,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 3,
      "kind": "3",
      "kl": 0.00440584063789979
    },
    {
      "path": "language_model.model.layers.16.mlp.up_proj",
      "layer": 16,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 2,
      "kind": "2",
      "kl": 0.007104415338209336
    },
    {
      "path": "language_model.model.layers.17.linear_attn.in_proj_qkv",
      "layer": 17,
      "layer_type": "linear_attention",
      "nparams": 6291456,
      "bits": 2,
      "kind": "ternary",
      "kl": 0.0021934894088310236
    },
    {
      "path": "language_model.model.layers.17.linear_attn.in_proj_z",
      "layer": 17,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 2,
      "kind": "2",
      "kl": 0.0032236417576331686
    },
    {
      "path": "language_model.model.layers.17.linear_attn.in_proj_b",
      "layer": 17,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 3,
      "kind": "3",
      "kl": 0.000886947803901282
    },
    {
      "path": "language_model.model.layers.17.linear_attn.in_proj_a",
      "layer": 17,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 3,
      "kind": "3",
      "kl": 0.000976995525569164
    },
    {
      "path": "language_model.model.layers.17.linear_attn.out_proj",
      "layer": 17,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 3,
      "kind": "3",
      "kl": 0.002114814895788902
    },
    {
      "path": "language_model.model.layers.17.mlp.gate_proj",
      "layer": 17,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 2,
      "kind": "2",
      "kl": 0.004665009833481794
    },
    {
      "path": "language_model.model.layers.17.mlp.down_proj",
      "layer": 17,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 3,
      "kind": "3",
      "kl": 0.004484325279917158
    },
    {
      "path": "language_model.model.layers.17.mlp.up_proj",
      "layer": 17,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 2,
      "kind": "ternary",
      "kl": 0.005121177837445393
    },
    {
      "path": "language_model.model.layers.18.linear_attn.in_proj_qkv",
      "layer": 18,
      "layer_type": "linear_attention",
      "nparams": 6291456,
      "bits": 2,
      "kind": "2",
      "kl": 0.006290895747047132
    },
    {
      "path": "language_model.model.layers.18.linear_attn.in_proj_z",
      "layer": 18,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 3,
      "kind": "3",
      "kl": 0.002068792589064126
    },
    {
      "path": "language_model.model.layers.18.linear_attn.in_proj_b",
      "layer": 18,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 3,
      "kind": "3",
      "kl": 0.0008810069546544934
    },
    {
      "path": "language_model.model.layers.18.linear_attn.in_proj_a",
      "layer": 18,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 4,
      "kind": "4",
      "kl": 0.000987608700835434
    },
    {
      "path": "language_model.model.layers.18.linear_attn.out_proj",
      "layer": 18,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 3,
      "kind": "3",
      "kl": 0.002887347314595934
    },
    {
      "path": "language_model.model.layers.18.mlp.gate_proj",
      "layer": 18,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 3,
      "kind": "3",
      "kl": 0.0018628583013199987
    },
    {
      "path": "language_model.model.layers.18.mlp.down_proj",
      "layer": 18,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.0023940537857527703
    },
    {
      "path": "language_model.model.layers.18.mlp.up_proj",
      "layer": 18,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 2,
      "kind": "2",
      "kl": 0.006746079837461991
    },
    {
      "path": "language_model.model.layers.19.self_attn.q_proj",
      "layer": 19,
      "layer_type": "full_attention",
      "nparams": 4194304,
      "bits": 3,
      "kind": "3",
      "kl": 0.006032338261324889
    },
    {
      "path": "language_model.model.layers.19.self_attn.k_proj",
      "layer": 19,
      "layer_type": "full_attention",
      "nparams": 524288,
      "bits": 4,
      "kind": "4",
      "kl": 0.0021031284974654106
    },
    {
      "path": "language_model.model.layers.19.self_attn.v_proj",
      "layer": 19,
      "layer_type": "full_attention",
      "nparams": 524288,
      "bits": 4,
      "kind": "4",
      "kl": 0.004348253480209944
    },
    {
      "path": "language_model.model.layers.19.self_attn.o_proj",
      "layer": 19,
      "layer_type": "full_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.004557360657928573
    },
    {
      "path": "language_model.model.layers.19.mlp.gate_proj",
      "layer": 19,
      "layer_type": "full_attention",
      "nparams": 3670016,
      "bits": 2,
      "kind": "2",
      "kl": 0.0029232948535193653
    },
    {
      "path": "language_model.model.layers.19.mlp.down_proj",
      "layer": 19,
      "layer_type": "full_attention",
      "nparams": 3670016,
      "bits": 2,
      "kind": "2",
      "kl": 0.0058071099995981815
    },
    {
      "path": "language_model.model.layers.19.mlp.up_proj",
      "layer": 19,
      "layer_type": "full_attention",
      "nparams": 3670016,
      "bits": 3,
      "kind": "3",
      "kl": 0.0023828522802972463
    },
    {
      "path": "language_model.model.layers.20.linear_attn.in_proj_qkv",
      "layer": 20,
      "layer_type": "linear_attention",
      "nparams": 6291456,
      "bits": 2,
      "kind": "2",
      "kl": 0.008125929608638258
    },
    {
      "path": "language_model.model.layers.20.linear_attn.in_proj_z",
      "layer": 20,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 2,
      "kind": "2",
      "kl": 0.0017258157022901578
    },
    {
      "path": "language_model.model.layers.20.linear_attn.in_proj_b",
      "layer": 20,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 4,
      "kind": "4",
      "kl": 0.0005154021203862869
    },
    {
      "path": "language_model.model.layers.20.linear_attn.in_proj_a",
      "layer": 20,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 2,
      "kind": "ternary",
      "kl": 0.0005305295310921597
    },
    {
      "path": "language_model.model.layers.20.linear_attn.out_proj",
      "layer": 20,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 3,
      "kind": "3",
      "kl": 0.0030152998807947977
    },
    {
      "path": "language_model.model.layers.20.mlp.gate_proj",
      "layer": 20,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 2,
      "kind": "2",
      "kl": 0.0022819108116763583
    },
    {
      "path": "language_model.model.layers.20.mlp.down_proj",
      "layer": 20,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 2,
      "kind": "2",
      "kl": 0.0015101953736824964
    },
    {
      "path": "language_model.model.layers.20.mlp.up_proj",
      "layer": 20,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 2,
      "kind": "ternary",
      "kl": 0.004383521037158472
    },
    {
      "path": "language_model.model.layers.21.linear_attn.in_proj_qkv",
      "layer": 21,
      "layer_type": "linear_attention",
      "nparams": 6291456,
      "bits": 2,
      "kind": "2",
      "kl": 0.0035359140742138107
    },
    {
      "path": "language_model.model.layers.21.linear_attn.in_proj_z",
      "layer": 21,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 2,
      "kind": "2",
      "kl": 0.0033209959668630907
    },
    {
      "path": "language_model.model.layers.21.linear_attn.in_proj_b",
      "layer": 21,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 2,
      "kind": "2",
      "kl": 0.0005307516921509054
    },
    {
      "path": "language_model.model.layers.21.linear_attn.in_proj_a",
      "layer": 21,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 4,
      "kind": "4",
      "kl": 0.0002769620310376814
    },
    {
      "path": "language_model.model.layers.21.linear_attn.out_proj",
      "layer": 21,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 3,
      "kind": "3",
      "kl": 0.0017829395230716451
    },
    {
      "path": "language_model.model.layers.21.mlp.gate_proj",
      "layer": 21,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 2,
      "kind": "ternary",
      "kl": 0.004034291358501529
    },
    {
      "path": "language_model.model.layers.21.mlp.down_proj",
      "layer": 21,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 3,
      "kind": "3",
      "kl": 0.0008362982179293819
    },
    {
      "path": "language_model.model.layers.21.mlp.up_proj",
      "layer": 21,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 2,
      "kind": "2",
      "kl": 0.003467660124694521
    },
    {
      "path": "language_model.model.layers.22.linear_attn.in_proj_qkv",
      "layer": 22,
      "layer_type": "linear_attention",
      "nparams": 6291456,
      "bits": 3,
      "kind": "3",
      "kl": 0.006269973941838318
    },
    {
      "path": "language_model.model.layers.22.linear_attn.in_proj_z",
      "layer": 22,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 3,
      "kind": "3",
      "kl": 0.0034496159643663885
    },
    {
      "path": "language_model.model.layers.22.linear_attn.in_proj_b",
      "layer": 22,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 4,
      "kind": "4",
      "kl": 0.0005182467336934553
    },
    {
      "path": "language_model.model.layers.22.linear_attn.in_proj_a",
      "layer": 22,
      "layer_type": "linear_attention",
      "nparams": 16384,
      "bits": 3,
      "kind": "3",
      "kl": 0.0003643164638947492
    },
    {
      "path": "language_model.model.layers.22.linear_attn.out_proj",
      "layer": 22,
      "layer_type": "linear_attention",
      "nparams": 2097152,
      "bits": 3,
      "kind": "3",
      "kl": 0.003477276947365615
    },
    {
      "path": "language_model.model.layers.22.mlp.gate_proj",
      "layer": 22,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 2,
      "kind": "2",
      "kl": 0.005100013411379256
    },
    {
      "path": "language_model.model.layers.22.mlp.down_proj",
      "layer": 22,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 3,
      "kind": "3",
      "kl": 0.0033566558855619606
    },
    {
      "path": "language_model.model.layers.22.mlp.up_proj",
      "layer": 22,
      "layer_type": "linear_attention",
      "nparams": 3670016,
      "bits": 3,
      "kind": "3",
      "kl": 0.00313410047627708
    },
    {
      "path": "language_model.model.layers.23.self_attn.q_proj",
      "layer": 23,
      "layer_type": "full_attention",
      "nparams": 4194304,
      "bits": 3,
      "kind": "3",
      "kl": 0.004782156469623039
    },
    {
      "path": "language_model.model.layers.23.self_attn.k_proj",
      "layer": 23,
      "layer_type": "full_attention",
      "nparams": 524288,
      "bits": 4,
      "kind": "4",
      "kl": 0.006514890193217124
    },
    {
      "path": "language_model.model.layers.23.self_attn.v_proj",
      "layer": 23,
      "layer_type": "full_attention",
      "nparams": 524288,
      "bits": 4,
      "kind": "4",
      "kl": 0.002427561433381982
    },
    {
      "path": "language_model.model.layers.23.self_attn.o_proj",
      "layer": 23,
      "layer_type": "full_attention",
      "nparams": 2097152,
      "bits": 4,
      "kind": "4",
      "kl": 0.002634655396783468
    },
    {
      "path": "language_model.model.layers.23.mlp.gate_proj",
      "layer": 23,
      "layer_type": "full_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.001242476476743588
    },
    {
      "path": "language_model.model.layers.23.mlp.down_proj",
      "layer": 23,
      "layer_type": "full_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.029779743010173505
    },
    {
      "path": "language_model.model.layers.23.mlp.up_proj",
      "layer": 23,
      "layer_type": "full_attention",
      "nparams": 3670016,
      "bits": 4,
      "kind": "4",
      "kl": 0.006386583194459594
    }
  ]
}