{
  "average_bits": 3.0498071979434447,
  "embed_bits": 8,
  "embed_uses_tied_head": false,
  "embed_reason": "The decision readout is last_hidden @ readout.T and does not use the tied lm_head. The embedding is input-only and is stored as 8-bit affine, group 64.",
  "embed_sum_kl": {
    "4": 0.011286197474019719,
    "8": 0.004745219702161558
  },
  "embed_bytes": {
    "4": 357580800,
    "8": 675430400
  },
  "joint_refine": {
    "calibration_n": 20,
    "calibration_ids": [
      "mnli/validation_matched/3117/relation",
      "banking77/test/1454/intent",
      "boolq/validation/2354/answer",
      "banking77/test/1793/permuted/intent",
      "composition/v3-composition-dev/conjunction/3/relevant/a/decision",
      "banking77/test/2120/none_absent/intent",
      "amazon/test/605/stars",
      "mnli/validation_matched/79/relation",
      "dbpedia14/test/40231/category",
      "dbpedia14/test/48873/category",
      "mnli/validation_matched/1139/relation",
      "composition/v3-composition-dev/nested_and/3/relevant/a/decision",
      "contrastive/age_eligibility/v3-control-dev-age_eligibility-0012/b/decision",
      "yelp/test/10368/rating",
      "agnews/test/1493/topic",
      "composition/v3-composition-dev/conditional/2/irrelevant/b/decision",
      "boolq/validation/2399/answer",
      "agnews/test/2095/topic",
      "mnli/validation_matched/8140/relation",
      "contrastive/age_eligibility/v3-control-dev-age_eligibility-0013/b/decision"
    ],
    "joint_kl_before": 0.27327187773454925,
    "joint_kl_after": 0.2351137807324731,
    "swaps": 4,
    "accepted": [
      {
        "up": "language_model.model.layers.5.linear_attn.in_proj_z",
        "down": "language_model.model.layers.16.linear_attn.in_proj_z",
        "up_to": "4",
        "down_to": "3",
        "joint_kl": 0.24233961590783762,
        "predicted": 5.7052450264859396e-05
      },
      {
        "up": "language_model.model.layers.24.linear_attn.in_proj_a",
        "down": "language_model.model.layers.13.linear_attn.in_proj_b",
        "up_to": "3",
        "down_to": "3",
        "joint_kl": 0.23988770723326833,
        "predicted": 0.0001434830382605536
      },
      {
        "up": "language_model.model.layers.5.linear_attn.in_proj_b",
        "down": "language_model.model.layers.24.linear_attn.in_proj_a",
        "up_to": "4",
        "down_to": "ternary",
        "joint_kl": 0.23789553741053193,
        "predicted": 1.5698582511071034e-05
      },
      {
        "up": "language_model.model.layers.25.linear_attn.in_proj_a",
        "down": "language_model.model.layers.5.linear_attn.in_proj_b",
        "up_to": "3",
        "down_to": "3",
        "joint_kl": 0.2351137807324731,
        "predicted": 1.9772355902315797e-05
      }
    ]
  },
  "assignment": [
    {
      "path": "language_model.model.layers.0.linear_attn.in_proj_qkv",
      "layer": 0,
      "layer_type": "linear_attention",
      "nparams": 20971520,
      "bits": 3,
      "kind": "3",
      "kl": 0.010448446673969113
    },
    {
      "path": "language_model.model.layers.0.linear_attn.in_proj_z",
      "layer": 0,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.007454903243540781
    },
    {
      "path": "language_model.model.layers.0.linear_attn.in_proj_b",
      "layer": 0,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 3,
      "kind": "3",
      "kl": 0.0033223642585997228
    },
    {
      "path": "language_model.model.layers.0.linear_attn.in_proj_a",
      "layer": 0,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 2,
      "kind": "2",
      "kl": 0.0035474445607511793
    },
    {
      "path": "language_model.model.layers.0.linear_attn.out_proj",
      "layer": 0,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.06154597570566438
    },
    {
      "path": "language_model.model.layers.0.mlp.gate_proj",
      "layer": 0,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.012915829515700692
    },
    {
      "path": "language_model.model.layers.0.mlp.down_proj",
      "layer": 0,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.006924833638929763
    },
    {
      "path": "language_model.model.layers.0.mlp.up_proj",
      "layer": 0,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.010924317433800512
    },
    {
      "path": "language_model.model.layers.1.linear_attn.in_proj_qkv",
      "layer": 1,
      "layer_type": "linear_attention",
      "nparams": 20971520,
      "bits": 4,
      "kind": "4",
      "kl": 0.010014838784017786
    },
    {
      "path": "language_model.model.layers.1.linear_attn.in_proj_z",
      "layer": 1,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.006952277388950299
    },
    {
      "path": "language_model.model.layers.1.linear_attn.in_proj_b",
      "layer": 1,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.0032649088695483855
    },
    {
      "path": "language_model.model.layers.1.linear_attn.in_proj_a",
      "layer": 1,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.0040076704179294684
    },
    {
      "path": "language_model.model.layers.1.linear_attn.out_proj",
      "layer": 1,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.005802257928629042
    },
    {
      "path": "language_model.model.layers.1.mlp.gate_proj",
      "layer": 1,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.01244909772269
    },
    {
      "path": "language_model.model.layers.1.mlp.down_proj",
      "layer": 1,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.0065094802589678565
    },
    {
      "path": "language_model.model.layers.1.mlp.up_proj",
      "layer": 1,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.014826100961388559
    },
    {
      "path": "language_model.model.layers.2.linear_attn.in_proj_qkv",
      "layer": 2,
      "layer_type": "linear_attention",
      "nparams": 20971520,
      "bits": 4,
      "kind": "4",
      "kl": 0.012259290078230753
    },
    {
      "path": "language_model.model.layers.2.linear_attn.in_proj_z",
      "layer": 2,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.006292201548291481
    },
    {
      "path": "language_model.model.layers.2.linear_attn.in_proj_b",
      "layer": 2,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 3,
      "kind": "3",
      "kl": 0.004659571511338003
    },
    {
      "path": "language_model.model.layers.2.linear_attn.in_proj_a",
      "layer": 2,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.00400962032856176
    },
    {
      "path": "language_model.model.layers.2.linear_attn.out_proj",
      "layer": 2,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.008425798009871279
    },
    {
      "path": "language_model.model.layers.2.mlp.gate_proj",
      "layer": 2,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.008391449294443439
    },
    {
      "path": "language_model.model.layers.2.mlp.down_proj",
      "layer": 2,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.005654083152917417
    },
    {
      "path": "language_model.model.layers.2.mlp.up_proj",
      "layer": 2,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.0231763729959581
    },
    {
      "path": "language_model.model.layers.3.self_attn.q_proj",
      "layer": 3,
      "layer_type": "full_attention",
      "nparams": 20971520,
      "bits": 4,
      "kind": "4",
      "kl": 0.0060903110340457785
    },
    {
      "path": "language_model.model.layers.3.self_attn.k_proj",
      "layer": 3,
      "layer_type": "full_attention",
      "nparams": 2621440,
      "bits": 4,
      "kind": "4",
      "kl": 0.0033800215613122805
    },
    {
      "path": "language_model.model.layers.3.self_attn.v_proj",
      "layer": 3,
      "layer_type": "full_attention",
      "nparams": 2621440,
      "bits": 4,
      "kind": "4",
      "kl": 0.01283825919462436
    },
    {
      "path": "language_model.model.layers.3.self_attn.o_proj",
      "layer": 3,
      "layer_type": "full_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.009046701374450225
    },
    {
      "path": "language_model.model.layers.3.mlp.gate_proj",
      "layer": 3,
      "layer_type": "full_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.01050212096987629
    },
    {
      "path": "language_model.model.layers.3.mlp.down_proj",
      "layer": 3,
      "layer_type": "full_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.0061818342546688335
    },
    {
      "path": "language_model.model.layers.3.mlp.up_proj",
      "layer": 3,
      "layer_type": "full_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.01611825974729804
    },
    {
      "path": "language_model.model.layers.4.linear_attn.in_proj_qkv",
      "layer": 4,
      "layer_type": "linear_attention",
      "nparams": 20971520,
      "bits": 3,
      "kind": "3",
      "kl": 0.013024711441356987
    },
    {
      "path": "language_model.model.layers.4.linear_attn.in_proj_z",
      "layer": 4,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.00429976138543272
    },
    {
      "path": "language_model.model.layers.4.linear_attn.in_proj_b",
      "layer": 4,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.0034435942018261646
    },
    {
      "path": "language_model.model.layers.4.linear_attn.in_proj_a",
      "layer": 4,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.00230237893619504
    },
    {
      "path": "language_model.model.layers.4.linear_attn.out_proj",
      "layer": 4,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.005720972922117528
    },
    {
      "path": "language_model.model.layers.4.mlp.gate_proj",
      "layer": 4,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.008319276305372148
    },
    {
      "path": "language_model.model.layers.4.mlp.down_proj",
      "layer": 4,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.00833979089090999
    },
    {
      "path": "language_model.model.layers.4.mlp.up_proj",
      "layer": 4,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.008235880725585083
    },
    {
      "path": "language_model.model.layers.5.linear_attn.in_proj_qkv",
      "layer": 5,
      "layer_type": "linear_attention",
      "nparams": 20971520,
      "bits": 4,
      "kind": "4",
      "kl": 0.008891306822033977
    },
    {
      "path": "language_model.model.layers.5.linear_attn.in_proj_z",
      "layer": 5,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.0049944552043391915
    },
    {
      "path": "language_model.model.layers.5.linear_attn.in_proj_b",
      "layer": 5,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 3,
      "kind": "3",
      "kl": 0.004080156970837005
    },
    {
      "path": "language_model.model.layers.5.linear_attn.in_proj_a",
      "layer": 5,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.0026296631091524943
    },
    {
      "path": "language_model.model.layers.5.linear_attn.out_proj",
      "layer": 5,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.005161598731319505
    },
    {
      "path": "language_model.model.layers.5.mlp.gate_proj",
      "layer": 5,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.0062223512029718495
    },
    {
      "path": "language_model.model.layers.5.mlp.down_proj",
      "layer": 5,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.004617157800457818
    },
    {
      "path": "language_model.model.layers.5.mlp.up_proj",
      "layer": 5,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.01450103365186557
    },
    {
      "path": "language_model.model.layers.6.linear_attn.in_proj_qkv",
      "layer": 6,
      "layer_type": "linear_attention",
      "nparams": 20971520,
      "bits": 4,
      "kind": "4",
      "kl": 0.010250001698275295
    },
    {
      "path": "language_model.model.layers.6.linear_attn.in_proj_z",
      "layer": 6,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.005800948212441958
    },
    {
      "path": "language_model.model.layers.6.linear_attn.in_proj_b",
      "layer": 6,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.0033723909629199142
    },
    {
      "path": "language_model.model.layers.6.linear_attn.in_proj_a",
      "layer": 6,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.0035418093499744747
    },
    {
      "path": "language_model.model.layers.6.linear_attn.out_proj",
      "layer": 6,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.00722899541177021
    },
    {
      "path": "language_model.model.layers.6.mlp.gate_proj",
      "layer": 6,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.00866014900850607
    },
    {
      "path": "language_model.model.layers.6.mlp.down_proj",
      "layer": 6,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.010980755737561482
    },
    {
      "path": "language_model.model.layers.6.mlp.up_proj",
      "layer": 6,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.01381060262887852
    },
    {
      "path": "language_model.model.layers.7.self_attn.q_proj",
      "layer": 7,
      "layer_type": "full_attention",
      "nparams": 20971520,
      "bits": 4,
      "kind": "4",
      "kl": 0.009426559410376102
    },
    {
      "path": "language_model.model.layers.7.self_attn.k_proj",
      "layer": 7,
      "layer_type": "full_attention",
      "nparams": 2621440,
      "bits": 4,
      "kind": "4",
      "kl": 0.004889550787478134
    },
    {
      "path": "language_model.model.layers.7.self_attn.v_proj",
      "layer": 7,
      "layer_type": "full_attention",
      "nparams": 2621440,
      "bits": 4,
      "kind": "4",
      "kl": 0.01626780704191527
    },
    {
      "path": "language_model.model.layers.7.self_attn.o_proj",
      "layer": 7,
      "layer_type": "full_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.02200452754786024
    },
    {
      "path": "language_model.model.layers.7.mlp.gate_proj",
      "layer": 7,
      "layer_type": "full_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.01603272086664161
    },
    {
      "path": "language_model.model.layers.7.mlp.down_proj",
      "layer": 7,
      "layer_type": "full_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.008829930829181317
    },
    {
      "path": "language_model.model.layers.7.mlp.up_proj",
      "layer": 7,
      "layer_type": "full_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.007551976471688681
    },
    {
      "path": "language_model.model.layers.8.linear_attn.in_proj_qkv",
      "layer": 8,
      "layer_type": "linear_attention",
      "nparams": 20971520,
      "bits": 4,
      "kind": "4",
      "kl": 0.00636630976733739
    },
    {
      "path": "language_model.model.layers.8.linear_attn.in_proj_z",
      "layer": 8,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.008684421003327794
    },
    {
      "path": "language_model.model.layers.8.linear_attn.in_proj_b",
      "layer": 8,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.0031855265832704638
    },
    {
      "path": "language_model.model.layers.8.linear_attn.in_proj_a",
      "layer": 8,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.002266341492353467
    },
    {
      "path": "language_model.model.layers.8.linear_attn.out_proj",
      "layer": 8,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.007247695609163844
    },
    {
      "path": "language_model.model.layers.8.mlp.gate_proj",
      "layer": 8,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.009257387943368597
    },
    {
      "path": "language_model.model.layers.8.mlp.down_proj",
      "layer": 8,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.011886287054785323
    },
    {
      "path": "language_model.model.layers.8.mlp.up_proj",
      "layer": 8,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.006176910843308165
    },
    {
      "path": "language_model.model.layers.9.linear_attn.in_proj_qkv",
      "layer": 9,
      "layer_type": "linear_attention",
      "nparams": 20971520,
      "bits": 4,
      "kind": "4",
      "kl": 0.007901331850660961
    },
    {
      "path": "language_model.model.layers.9.linear_attn.in_proj_z",
      "layer": 9,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.006090545368305263
    },
    {
      "path": "language_model.model.layers.9.linear_attn.in_proj_b",
      "layer": 9,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.0021647317109205087
    },
    {
      "path": "language_model.model.layers.9.linear_attn.in_proj_a",
      "layer": 9,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.0034912846735995606
    },
    {
      "path": "language_model.model.layers.9.linear_attn.out_proj",
      "layer": 9,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.006964942701371921
    },
    {
      "path": "language_model.model.layers.9.mlp.gate_proj",
      "layer": 9,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.010906623393109898
    },
    {
      "path": "language_model.model.layers.9.mlp.down_proj",
      "layer": 9,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.011869402951866571
    },
    {
      "path": "language_model.model.layers.9.mlp.up_proj",
      "layer": 9,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.008047571413088556
    },
    {
      "path": "language_model.model.layers.10.linear_attn.in_proj_qkv",
      "layer": 10,
      "layer_type": "linear_attention",
      "nparams": 20971520,
      "bits": 4,
      "kind": "4",
      "kl": 0.009507280075560192
    },
    {
      "path": "language_model.model.layers.10.linear_attn.in_proj_z",
      "layer": 10,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.006265319777060277
    },
    {
      "path": "language_model.model.layers.10.linear_attn.in_proj_b",
      "layer": 10,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.0038279725069975206
    },
    {
      "path": "language_model.model.layers.10.linear_attn.in_proj_a",
      "layer": 10,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 3,
      "kind": "3",
      "kl": 0.003955201643937205
    },
    {
      "path": "language_model.model.layers.10.linear_attn.out_proj",
      "layer": 10,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.0074089394346225565
    },
    {
      "path": "language_model.model.layers.10.mlp.gate_proj",
      "layer": 10,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.00640157690977524
    },
    {
      "path": "language_model.model.layers.10.mlp.down_proj",
      "layer": 10,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.008578138881508952
    },
    {
      "path": "language_model.model.layers.10.mlp.up_proj",
      "layer": 10,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.010964694488962036
    },
    {
      "path": "language_model.model.layers.11.self_attn.q_proj",
      "layer": 11,
      "layer_type": "full_attention",
      "nparams": 20971520,
      "bits": 4,
      "kind": "4",
      "kl": 0.0133026334184451
    },
    {
      "path": "language_model.model.layers.11.self_attn.k_proj",
      "layer": 11,
      "layer_type": "full_attention",
      "nparams": 2621440,
      "bits": 4,
      "kind": "4",
      "kl": 0.007485433004483934
    },
    {
      "path": "language_model.model.layers.11.self_attn.v_proj",
      "layer": 11,
      "layer_type": "full_attention",
      "nparams": 2621440,
      "bits": 4,
      "kind": "4",
      "kl": 0.009912732537830257
    },
    {
      "path": "language_model.model.layers.11.self_attn.o_proj",
      "layer": 11,
      "layer_type": "full_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.01034751691642807
    },
    {
      "path": "language_model.model.layers.11.mlp.gate_proj",
      "layer": 11,
      "layer_type": "full_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.004611820005780289
    },
    {
      "path": "language_model.model.layers.11.mlp.down_proj",
      "layer": 11,
      "layer_type": "full_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.005963056204075085
    },
    {
      "path": "language_model.model.layers.11.mlp.up_proj",
      "layer": 11,
      "layer_type": "full_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.005597160091559011
    },
    {
      "path": "language_model.model.layers.12.linear_attn.in_proj_qkv",
      "layer": 12,
      "layer_type": "linear_attention",
      "nparams": 20971520,
      "bits": 4,
      "kind": "4",
      "kl": 0.005897109182151416
    },
    {
      "path": "language_model.model.layers.12.linear_attn.in_proj_z",
      "layer": 12,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.005947143164879199
    },
    {
      "path": "language_model.model.layers.12.linear_attn.in_proj_b",
      "layer": 12,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.0030364137922867197
    },
    {
      "path": "language_model.model.layers.12.linear_attn.in_proj_a",
      "layer": 12,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.002990399791879363
    },
    {
      "path": "language_model.model.layers.12.linear_attn.out_proj",
      "layer": 12,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.004469233101413777
    },
    {
      "path": "language_model.model.layers.12.mlp.gate_proj",
      "layer": 12,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.005604113181054048
    },
    {
      "path": "language_model.model.layers.12.mlp.down_proj",
      "layer": 12,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.006729049078832552
    },
    {
      "path": "language_model.model.layers.12.mlp.up_proj",
      "layer": 12,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.0087499496219987
    },
    {
      "path": "language_model.model.layers.13.linear_attn.in_proj_qkv",
      "layer": 13,
      "layer_type": "linear_attention",
      "nparams": 20971520,
      "bits": 4,
      "kind": "4",
      "kl": 0.008554809788733155
    },
    {
      "path": "language_model.model.layers.13.linear_attn.in_proj_z",
      "layer": 13,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.0024209885100068777
    },
    {
      "path": "language_model.model.layers.13.linear_attn.in_proj_b",
      "layer": 13,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 3,
      "kind": "3",
      "kl": 0.003163799510455978
    },
    {
      "path": "language_model.model.layers.13.linear_attn.in_proj_a",
      "layer": 13,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.002690445258387349
    },
    {
      "path": "language_model.model.layers.13.linear_attn.out_proj",
      "layer": 13,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 3,
      "kind": "3",
      "kl": 0.010155192323957927
    },
    {
      "path": "language_model.model.layers.13.mlp.gate_proj",
      "layer": 13,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.011515350872993905
    },
    {
      "path": "language_model.model.layers.13.mlp.down_proj",
      "layer": 13,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.0041446334288496416
    },
    {
      "path": "language_model.model.layers.13.mlp.up_proj",
      "layer": 13,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.004337408969486667
    },
    {
      "path": "language_model.model.layers.14.linear_attn.in_proj_qkv",
      "layer": 14,
      "layer_type": "linear_attention",
      "nparams": 20971520,
      "bits": 4,
      "kind": "4",
      "kl": 0.011339673428033051
    },
    {
      "path": "language_model.model.layers.14.linear_attn.in_proj_z",
      "layer": 14,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 3,
      "kind": "3",
      "kl": 0.007962872862178986
    },
    {
      "path": "language_model.model.layers.14.linear_attn.in_proj_b",
      "layer": 14,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.0019362232417174479
    },
    {
      "path": "language_model.model.layers.14.linear_attn.in_proj_a",
      "layer": 14,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.0015431009217327445
    },
    {
      "path": "language_model.model.layers.14.linear_attn.out_proj",
      "layer": 14,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.004887138324674762
    },
    {
      "path": "language_model.model.layers.14.mlp.gate_proj",
      "layer": 14,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.013886265417909858
    },
    {
      "path": "language_model.model.layers.14.mlp.down_proj",
      "layer": 14,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.01326110203221376
    },
    {
      "path": "language_model.model.layers.14.mlp.up_proj",
      "layer": 14,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.006025693011817328
    },
    {
      "path": "language_model.model.layers.15.self_attn.q_proj",
      "layer": 15,
      "layer_type": "full_attention",
      "nparams": 20971520,
      "bits": 4,
      "kind": "4",
      "kl": 0.018192663011557066
    },
    {
      "path": "language_model.model.layers.15.self_attn.k_proj",
      "layer": 15,
      "layer_type": "full_attention",
      "nparams": 2621440,
      "bits": 4,
      "kind": "4",
      "kl": 0.006153966935711085
    },
    {
      "path": "language_model.model.layers.15.self_attn.v_proj",
      "layer": 15,
      "layer_type": "full_attention",
      "nparams": 2621440,
      "bits": 4,
      "kind": "4",
      "kl": 0.00728601800779855
    },
    {
      "path": "language_model.model.layers.15.self_attn.o_proj",
      "layer": 15,
      "layer_type": "full_attention",
      "nparams": 10485760,
      "bits": 4,
      "kind": "4",
      "kl": 0.00843663950103548
    },
    {
      "path": "language_model.model.layers.15.mlp.gate_proj",
      "layer": 15,
      "layer_type": "full_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.00844169358430183
    },
    {
      "path": "language_model.model.layers.15.mlp.down_proj",
      "layer": 15,
      "layer_type": "full_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.008618414916908151
    },
    {
      "path": "language_model.model.layers.15.mlp.up_proj",
      "layer": 15,
      "layer_type": "full_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.00904957608295576
    },
    {
      "path": "language_model.model.layers.16.linear_attn.in_proj_qkv",
      "layer": 16,
      "layer_type": "linear_attention",
      "nparams": 20971520,
      "bits": 3,
      "kind": "3",
      "kl": 0.012239668651959228
    },
    {
      "path": "language_model.model.layers.16.linear_attn.in_proj_z",
      "layer": 16,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 3,
      "kind": "3",
      "kl": 0.007165175116112102
    },
    {
      "path": "language_model.model.layers.16.linear_attn.in_proj_b",
      "layer": 16,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.001689089058653197
    },
    {
      "path": "language_model.model.layers.16.linear_attn.in_proj_a",
      "layer": 16,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.0021763847058412227
    },
    {
      "path": "language_model.model.layers.16.linear_attn.out_proj",
      "layer": 16,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 3,
      "kind": "3",
      "kl": 0.0050337960426177965
    },
    {
      "path": "language_model.model.layers.16.mlp.gate_proj",
      "layer": 16,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.004508509312858587
    },
    {
      "path": "language_model.model.layers.16.mlp.down_proj",
      "layer": 16,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.006751129047441731
    },
    {
      "path": "language_model.model.layers.16.mlp.up_proj",
      "layer": 16,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.008521210010591636
    },
    {
      "path": "language_model.model.layers.17.linear_attn.in_proj_qkv",
      "layer": 17,
      "layer_type": "linear_attention",
      "nparams": 20971520,
      "bits": 3,
      "kind": "3",
      "kl": 0.006143374253497072
    },
    {
      "path": "language_model.model.layers.17.linear_attn.in_proj_z",
      "layer": 17,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 3,
      "kind": "3",
      "kl": 0.0033700770476747474
    },
    {
      "path": "language_model.model.layers.17.linear_attn.in_proj_b",
      "layer": 17,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.0011950391956117611
    },
    {
      "path": "language_model.model.layers.17.linear_attn.in_proj_a",
      "layer": 17,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.0013687938506446078
    },
    {
      "path": "language_model.model.layers.17.linear_attn.out_proj",
      "layer": 17,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 3,
      "kind": "3",
      "kl": 0.0034261684359799434
    },
    {
      "path": "language_model.model.layers.17.mlp.gate_proj",
      "layer": 17,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.005377442401188507
    },
    {
      "path": "language_model.model.layers.17.mlp.down_proj",
      "layer": 17,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.012502019640959393
    },
    {
      "path": "language_model.model.layers.17.mlp.up_proj",
      "layer": 17,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.005043160515902214
    },
    {
      "path": "language_model.model.layers.18.linear_attn.in_proj_qkv",
      "layer": 18,
      "layer_type": "linear_attention",
      "nparams": 20971520,
      "bits": 3,
      "kind": "3",
      "kl": 0.006653324278347113
    },
    {
      "path": "language_model.model.layers.18.linear_attn.in_proj_z",
      "layer": 18,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 3,
      "kind": "3",
      "kl": 0.0034842532869403657
    },
    {
      "path": "language_model.model.layers.18.linear_attn.in_proj_b",
      "layer": 18,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.0021424430907280296
    },
    {
      "path": "language_model.model.layers.18.linear_attn.in_proj_a",
      "layer": 18,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.002039650694065247
    },
    {
      "path": "language_model.model.layers.18.linear_attn.out_proj",
      "layer": 18,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 3,
      "kind": "3",
      "kl": 0.005711511275365844
    },
    {
      "path": "language_model.model.layers.18.mlp.gate_proj",
      "layer": 18,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.00516365691585215
    },
    {
      "path": "language_model.model.layers.18.mlp.down_proj",
      "layer": 18,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.005297452417896363
    },
    {
      "path": "language_model.model.layers.18.mlp.up_proj",
      "layer": 18,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.0057767763788979624
    },
    {
      "path": "language_model.model.layers.19.self_attn.q_proj",
      "layer": 19,
      "layer_type": "full_attention",
      "nparams": 20971520,
      "bits": 2,
      "kind": "2",
      "kl": 0.015324583651697823
    },
    {
      "path": "language_model.model.layers.19.self_attn.k_proj",
      "layer": 19,
      "layer_type": "full_attention",
      "nparams": 2621440,
      "bits": 4,
      "kind": "4",
      "kl": 0.0020935271682160405
    },
    {
      "path": "language_model.model.layers.19.self_attn.v_proj",
      "layer": 19,
      "layer_type": "full_attention",
      "nparams": 2621440,
      "bits": 4,
      "kind": "4",
      "kl": 0.0024838890634897187
    },
    {
      "path": "language_model.model.layers.19.self_attn.o_proj",
      "layer": 19,
      "layer_type": "full_attention",
      "nparams": 10485760,
      "bits": 3,
      "kind": "3",
      "kl": 0.006107687193440991
    },
    {
      "path": "language_model.model.layers.19.mlp.gate_proj",
      "layer": 19,
      "layer_type": "full_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.006521167636485691
    },
    {
      "path": "language_model.model.layers.19.mlp.down_proj",
      "layer": 19,
      "layer_type": "full_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.012130510119700717
    },
    {
      "path": "language_model.model.layers.19.mlp.up_proj",
      "layer": 19,
      "layer_type": "full_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.011730753644203331
    },
    {
      "path": "language_model.model.layers.20.linear_attn.in_proj_qkv",
      "layer": 20,
      "layer_type": "linear_attention",
      "nparams": 20971520,
      "bits": 2,
      "kind": "ternary",
      "kl": 0.004217642276790693
    },
    {
      "path": "language_model.model.layers.20.linear_attn.in_proj_z",
      "layer": 20,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 2,
      "kind": "2",
      "kl": 0.003584793409144641
    },
    {
      "path": "language_model.model.layers.20.linear_attn.in_proj_b",
      "layer": 20,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 2,
      "kind": "2",
      "kl": 0.0016076045508908155
    },
    {
      "path": "language_model.model.layers.20.linear_attn.in_proj_a",
      "layer": 20,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.0014620511291034764
    },
    {
      "path": "language_model.model.layers.20.linear_attn.out_proj",
      "layer": 20,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 2,
      "kind": "2",
      "kl": 0.003913781122171125
    },
    {
      "path": "language_model.model.layers.20.mlp.gate_proj",
      "layer": 20,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.005980544503803167
    },
    {
      "path": "language_model.model.layers.20.mlp.down_proj",
      "layer": 20,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.005546381825443038
    },
    {
      "path": "language_model.model.layers.20.mlp.up_proj",
      "layer": 20,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.005117319684254928
    },
    {
      "path": "language_model.model.layers.21.linear_attn.in_proj_qkv",
      "layer": 21,
      "layer_type": "linear_attention",
      "nparams": 20971520,
      "bits": 2,
      "kind": "2",
      "kl": 0.0038563888301930307
    },
    {
      "path": "language_model.model.layers.21.linear_attn.in_proj_z",
      "layer": 21,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 2,
      "kind": "2",
      "kl": 0.00392118822650533
    },
    {
      "path": "language_model.model.layers.21.linear_attn.in_proj_b",
      "layer": 21,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.0011615152303928113
    },
    {
      "path": "language_model.model.layers.21.linear_attn.in_proj_a",
      "layer": 21,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 3,
      "kind": "3",
      "kl": 0.0013032740645528766
    },
    {
      "path": "language_model.model.layers.21.linear_attn.out_proj",
      "layer": 21,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 2,
      "kind": "2",
      "kl": 0.0038319851560137557
    },
    {
      "path": "language_model.model.layers.21.mlp.gate_proj",
      "layer": 21,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.0044488231220858945
    },
    {
      "path": "language_model.model.layers.21.mlp.down_proj",
      "layer": 21,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.010022162259151325
    },
    {
      "path": "language_model.model.layers.21.mlp.up_proj",
      "layer": 21,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.008296753091727066
    },
    {
      "path": "language_model.model.layers.22.linear_attn.in_proj_qkv",
      "layer": 22,
      "layer_type": "linear_attention",
      "nparams": 20971520,
      "bits": 2,
      "kind": "ternary",
      "kl": 0.005124012435564135
    },
    {
      "path": "language_model.model.layers.22.linear_attn.in_proj_z",
      "layer": 22,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 2,
      "kind": "2",
      "kl": 0.0016929112836727208
    },
    {
      "path": "language_model.model.layers.22.linear_attn.in_proj_b",
      "layer": 22,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.00120441033270699
    },
    {
      "path": "language_model.model.layers.22.linear_attn.in_proj_a",
      "layer": 22,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.0018310609734295002
    },
    {
      "path": "language_model.model.layers.22.linear_attn.out_proj",
      "layer": 22,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 2,
      "kind": "2",
      "kl": 0.0025930998573809503
    },
    {
      "path": "language_model.model.layers.22.mlp.gate_proj",
      "layer": 22,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.007172711070889574
    },
    {
      "path": "language_model.model.layers.22.mlp.down_proj",
      "layer": 22,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.006215302619931402
    },
    {
      "path": "language_model.model.layers.22.mlp.up_proj",
      "layer": 22,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.004271566133998565
    },
    {
      "path": "language_model.model.layers.23.self_attn.q_proj",
      "layer": 23,
      "layer_type": "full_attention",
      "nparams": 20971520,
      "bits": 2,
      "kind": "ternary",
      "kl": 0.00861396856288937
    },
    {
      "path": "language_model.model.layers.23.self_attn.k_proj",
      "layer": 23,
      "layer_type": "full_attention",
      "nparams": 2621440,
      "bits": 4,
      "kind": "4",
      "kl": 0.0019846955283420566
    },
    {
      "path": "language_model.model.layers.23.self_attn.v_proj",
      "layer": 23,
      "layer_type": "full_attention",
      "nparams": 2621440,
      "bits": 4,
      "kind": "4",
      "kl": 0.0020784904667924616
    },
    {
      "path": "language_model.model.layers.23.self_attn.o_proj",
      "layer": 23,
      "layer_type": "full_attention",
      "nparams": 10485760,
      "bits": 3,
      "kind": "3",
      "kl": 0.005887735015824158
    },
    {
      "path": "language_model.model.layers.23.mlp.gate_proj",
      "layer": 23,
      "layer_type": "full_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.0027974898614304316
    },
    {
      "path": "language_model.model.layers.23.mlp.down_proj",
      "layer": 23,
      "layer_type": "full_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.005008540500233844
    },
    {
      "path": "language_model.model.layers.23.mlp.up_proj",
      "layer": 23,
      "layer_type": "full_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.004288408973020578
    },
    {
      "path": "language_model.model.layers.24.linear_attn.in_proj_qkv",
      "layer": 24,
      "layer_type": "linear_attention",
      "nparams": 20971520,
      "bits": 2,
      "kind": "2",
      "kl": 0.0035279809933627783
    },
    {
      "path": "language_model.model.layers.24.linear_attn.in_proj_z",
      "layer": 24,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 2,
      "kind": "2",
      "kl": 0.0028186044364557697
    },
    {
      "path": "language_model.model.layers.24.linear_attn.in_proj_b",
      "layer": 24,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 3,
      "kind": "3",
      "kl": 0.000861179402180736
    },
    {
      "path": "language_model.model.layers.24.linear_attn.in_proj_a",
      "layer": 24,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 2,
      "kind": "ternary",
      "kl": 0.0014708640460124364
    },
    {
      "path": "language_model.model.layers.24.linear_attn.out_proj",
      "layer": 24,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 2,
      "kind": "2",
      "kl": 0.002918557645676256
    },
    {
      "path": "language_model.model.layers.24.mlp.gate_proj",
      "layer": 24,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.003586550376589139
    },
    {
      "path": "language_model.model.layers.24.mlp.down_proj",
      "layer": 24,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.0033804561018519865
    },
    {
      "path": "language_model.model.layers.24.mlp.up_proj",
      "layer": 24,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.005295595565605282
    },
    {
      "path": "language_model.model.layers.25.linear_attn.in_proj_qkv",
      "layer": 25,
      "layer_type": "linear_attention",
      "nparams": 20971520,
      "bits": 2,
      "kind": "2",
      "kl": 0.0053310425548574476
    },
    {
      "path": "language_model.model.layers.25.linear_attn.in_proj_z",
      "layer": 25,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 2,
      "kind": "2",
      "kl": 0.0027591396448848737
    },
    {
      "path": "language_model.model.layers.25.linear_attn.in_proj_b",
      "layer": 25,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.0013798386948937805
    },
    {
      "path": "language_model.model.layers.25.linear_attn.in_proj_a",
      "layer": 25,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 3,
      "kind": "3",
      "kl": 0.001222093718792863
    },
    {
      "path": "language_model.model.layers.25.linear_attn.out_proj",
      "layer": 25,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 2,
      "kind": "2",
      "kl": 0.003876863353402041
    },
    {
      "path": "language_model.model.layers.25.mlp.gate_proj",
      "layer": 25,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.004169148010042012
    },
    {
      "path": "language_model.model.layers.25.mlp.down_proj",
      "layer": 25,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.005103545410745988
    },
    {
      "path": "language_model.model.layers.25.mlp.up_proj",
      "layer": 25,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.002309934380767234
    },
    {
      "path": "language_model.model.layers.26.linear_attn.in_proj_qkv",
      "layer": 26,
      "layer_type": "linear_attention",
      "nparams": 20971520,
      "bits": 2,
      "kind": "ternary",
      "kl": 0.003328468159294269
    },
    {
      "path": "language_model.model.layers.26.linear_attn.in_proj_z",
      "layer": 26,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 2,
      "kind": "2",
      "kl": 0.0025809277220773287
    },
    {
      "path": "language_model.model.layers.26.linear_attn.in_proj_b",
      "layer": 26,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 3,
      "kind": "3",
      "kl": 0.001228033403678685
    },
    {
      "path": "language_model.model.layers.26.linear_attn.in_proj_a",
      "layer": 26,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 2,
      "kind": "ternary",
      "kl": 0.0008598071553789065
    },
    {
      "path": "language_model.model.layers.26.linear_attn.out_proj",
      "layer": 26,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 2,
      "kind": "2",
      "kl": 0.0036962745992717514
    },
    {
      "path": "language_model.model.layers.26.mlp.gate_proj",
      "layer": 26,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.004956540361069822
    },
    {
      "path": "language_model.model.layers.26.mlp.down_proj",
      "layer": 26,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.0034921885774195933
    },
    {
      "path": "language_model.model.layers.26.mlp.up_proj",
      "layer": 26,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "ternary",
      "kl": 0.0044375837627941605
    },
    {
      "path": "language_model.model.layers.27.self_attn.q_proj",
      "layer": 27,
      "layer_type": "full_attention",
      "nparams": 20971520,
      "bits": 3,
      "kind": "3",
      "kl": 0.007407018614069779
    },
    {
      "path": "language_model.model.layers.27.self_attn.k_proj",
      "layer": 27,
      "layer_type": "full_attention",
      "nparams": 2621440,
      "bits": 4,
      "kind": "4",
      "kl": 0.002578442176801118
    },
    {
      "path": "language_model.model.layers.27.self_attn.v_proj",
      "layer": 27,
      "layer_type": "full_attention",
      "nparams": 2621440,
      "bits": 4,
      "kind": "4",
      "kl": 0.0015826920125229685
    },
    {
      "path": "language_model.model.layers.27.self_attn.o_proj",
      "layer": 27,
      "layer_type": "full_attention",
      "nparams": 10485760,
      "bits": 3,
      "kind": "3",
      "kl": 0.0063210920071340064
    },
    {
      "path": "language_model.model.layers.27.mlp.gate_proj",
      "layer": 27,
      "layer_type": "full_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.0024200480880117894
    },
    {
      "path": "language_model.model.layers.27.mlp.down_proj",
      "layer": 27,
      "layer_type": "full_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.0038518660025201163
    },
    {
      "path": "language_model.model.layers.27.mlp.up_proj",
      "layer": 27,
      "layer_type": "full_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.003806384045919116
    },
    {
      "path": "language_model.model.layers.28.linear_attn.in_proj_qkv",
      "layer": 28,
      "layer_type": "linear_attention",
      "nparams": 20971520,
      "bits": 2,
      "kind": "2",
      "kl": 0.002749126565589559
    },
    {
      "path": "language_model.model.layers.28.linear_attn.in_proj_z",
      "layer": 28,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 2,
      "kind": "2",
      "kl": 0.00198772293110002
    },
    {
      "path": "language_model.model.layers.28.linear_attn.in_proj_b",
      "layer": 28,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 3,
      "kind": "3",
      "kl": 0.0004472118012195674
    },
    {
      "path": "language_model.model.layers.28.linear_attn.in_proj_a",
      "layer": 28,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 3,
      "kind": "3",
      "kl": 0.0006654004124117305
    },
    {
      "path": "language_model.model.layers.28.linear_attn.out_proj",
      "layer": 28,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 2,
      "kind": "2",
      "kl": 0.003904530959754009
    },
    {
      "path": "language_model.model.layers.28.mlp.gate_proj",
      "layer": 28,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.0024391735330841595
    },
    {
      "path": "language_model.model.layers.28.mlp.down_proj",
      "layer": 28,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.0035189845281537145
    },
    {
      "path": "language_model.model.layers.28.mlp.up_proj",
      "layer": 28,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.0030495466258141754
    },
    {
      "path": "language_model.model.layers.29.linear_attn.in_proj_qkv",
      "layer": 29,
      "layer_type": "linear_attention",
      "nparams": 20971520,
      "bits": 2,
      "kind": "ternary",
      "kl": 0.002629985415225408
    },
    {
      "path": "language_model.model.layers.29.linear_attn.in_proj_z",
      "layer": 29,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 2,
      "kind": "2",
      "kl": 0.0025190827935064374
    },
    {
      "path": "language_model.model.layers.29.linear_attn.in_proj_b",
      "layer": 29,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 3,
      "kind": "3",
      "kl": 0.0007231384101107027
    },
    {
      "path": "language_model.model.layers.29.linear_attn.in_proj_a",
      "layer": 29,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 2,
      "kind": "2",
      "kl": 0.0005520234299671102
    },
    {
      "path": "language_model.model.layers.29.linear_attn.out_proj",
      "layer": 29,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 2,
      "kind": "ternary",
      "kl": 0.0022329573897954718
    },
    {
      "path": "language_model.model.layers.29.mlp.gate_proj",
      "layer": 29,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.003990216169521104
    },
    {
      "path": "language_model.model.layers.29.mlp.down_proj",
      "layer": 29,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.004243390057269966
    },
    {
      "path": "language_model.model.layers.29.mlp.up_proj",
      "layer": 29,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.008227331830174473
    },
    {
      "path": "language_model.model.layers.30.linear_attn.in_proj_qkv",
      "layer": 30,
      "layer_type": "linear_attention",
      "nparams": 20971520,
      "bits": 2,
      "kind": "2",
      "kl": 0.0052265165882378235
    },
    {
      "path": "language_model.model.layers.30.linear_attn.in_proj_z",
      "layer": 30,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 2,
      "kind": "2",
      "kl": 0.003792861104396105
    },
    {
      "path": "language_model.model.layers.30.linear_attn.in_proj_b",
      "layer": 30,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.0007776507164580607
    },
    {
      "path": "language_model.model.layers.30.linear_attn.in_proj_a",
      "layer": 30,
      "layer_type": "linear_attention",
      "nparams": 81920,
      "bits": 4,
      "kind": "4",
      "kl": 0.0008808346500481211
    },
    {
      "path": "language_model.model.layers.30.linear_attn.out_proj",
      "layer": 30,
      "layer_type": "linear_attention",
      "nparams": 10485760,
      "bits": 3,
      "kind": "3",
      "kl": 0.0030045996317789635
    },
    {
      "path": "language_model.model.layers.30.mlp.gate_proj",
      "layer": 30,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "2",
      "kl": 0.004581500490373279
    },
    {
      "path": "language_model.model.layers.30.mlp.down_proj",
      "layer": 30,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "ternary",
      "kl": 0.0075853140942216245
    },
    {
      "path": "language_model.model.layers.30.mlp.up_proj",
      "layer": 30,
      "layer_type": "linear_attention",
      "nparams": 23592960,
      "bits": 2,
      "kind": "ternary",
      "kl": 0.009086271054308806
    },
    {
      "path": "language_model.model.layers.31.self_attn.q_proj",
      "layer": 31,
      "layer_type": "full_attention",
      "nparams": 20971520,
      "bits": 3,
      "kind": "3",
      "kl": 0.004721018605704054
    },
    {
      "path": "language_model.model.layers.31.self_attn.k_proj",
      "layer": 31,
      "layer_type": "full_attention",
      "nparams": 2621440,
      "bits": 4,
      "kind": "4",
      "kl": 0.001393019356334057
    },
    {
      "path": "language_model.model.layers.31.self_attn.v_proj",
      "layer": 31,
      "layer_type": "full_attention",
      "nparams": 2621440,
      "bits": 4,
      "kind": "4",
      "kl": 0.0009648835963438874
    },
    {
      "path": "language_model.model.layers.31.self_attn.o_proj",
      "layer": 31,
      "layer_type": "full_attention",
      "nparams": 10485760,
      "bits": 3,
      "kind": "3",
      "kl": 0.004874454498413899
    },
    {
      "path": "language_model.model.layers.31.mlp.gate_proj",
      "layer": 31,
      "layer_type": "full_attention",
      "nparams": 23592960,
      "bits": 3,
      "kind": "3",
      "kl": 0.008578638269353754
    },
    {
      "path": "language_model.model.layers.31.mlp.down_proj",
      "layer": 31,
      "layer_type": "full_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.0059679943044754795
    },
    {
      "path": "language_model.model.layers.31.mlp.up_proj",
      "layer": 31,
      "layer_type": "full_attention",
      "nparams": 23592960,
      "bits": 4,
      "kind": "4",
      "kl": 0.00986051337512158
    }
  ]
}