| { |
| "average_bits": 3.4999670749374423, |
| "embed_bits": 8, |
| "embed_uses_tied_head": false, |
| "embed_reason": "The decision readout is last_hidden @ readout.T and does not use the tied lm_head. The embedding is input-only and is stored as 8-bit affine, group 64.", |
| "embed_sum_kl": { |
| "4": 0.055970916987464966, |
| "8": 0.010904520401237319 |
| }, |
| "embed_bytes": { |
| "4": 143032320, |
| "8": 270172160 |
| }, |
| "joint_refine": { |
| "calibration_n": 0, |
| "calibration_ids": [], |
| "joint_kl_before": null, |
| "joint_kl_after": null, |
| "swaps": 0 |
| }, |
| "assignment": [ |
| { |
| "path": "language_model.model.layers.0.linear_attn.in_proj_qkv", |
| "layer": 0, |
| "layer_type": "linear_attention", |
| "nparams": 6291456, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.04734761182665103 |
| }, |
| { |
| "path": "language_model.model.layers.0.linear_attn.in_proj_z", |
| "layer": 0, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.1388501915724426 |
| }, |
| { |
| "path": "language_model.model.layers.0.linear_attn.in_proj_b", |
| "layer": 0, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.01794408019266606 |
| }, |
| { |
| "path": "language_model.model.layers.0.linear_attn.in_proj_a", |
| "layer": 0, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.011710990232290872 |
| }, |
| { |
| "path": "language_model.model.layers.0.linear_attn.out_proj", |
| "layer": 0, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.4239300562384638 |
| }, |
| { |
| "path": "language_model.model.layers.0.mlp.gate_proj", |
| "layer": 0, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.06092327188142901 |
| }, |
| { |
| "path": "language_model.model.layers.0.mlp.down_proj", |
| "layer": 0, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.02344921252365501 |
| }, |
| { |
| "path": "language_model.model.layers.0.mlp.up_proj", |
| "layer": 0, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.027068575012547745 |
| }, |
| { |
| "path": "language_model.model.layers.1.linear_attn.in_proj_qkv", |
| "layer": 1, |
| "layer_type": "linear_attention", |
| "nparams": 6291456, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.023629120959874177 |
| }, |
| { |
| "path": "language_model.model.layers.1.linear_attn.in_proj_z", |
| "layer": 1, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.025880121687977955 |
| }, |
| { |
| "path": "language_model.model.layers.1.linear_attn.in_proj_b", |
| "layer": 1, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.01083456115500153 |
| }, |
| { |
| "path": "language_model.model.layers.1.linear_attn.in_proj_a", |
| "layer": 1, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.01273320237789333 |
| }, |
| { |
| "path": "language_model.model.layers.1.linear_attn.out_proj", |
| "layer": 1, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0927767372959222 |
| }, |
| { |
| "path": "language_model.model.layers.1.mlp.gate_proj", |
| "layer": 1, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.021329556389151568 |
| }, |
| { |
| "path": "language_model.model.layers.1.mlp.down_proj", |
| "layer": 1, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.02737624581982894 |
| }, |
| { |
| "path": "language_model.model.layers.1.mlp.up_proj", |
| "layer": 1, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.037710730514948596 |
| }, |
| { |
| "path": "language_model.model.layers.2.linear_attn.in_proj_qkv", |
| "layer": 2, |
| "layer_type": "linear_attention", |
| "nparams": 6291456, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.05043157109883073 |
| }, |
| { |
| "path": "language_model.model.layers.2.linear_attn.in_proj_z", |
| "layer": 2, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.03697225452164565 |
| }, |
| { |
| "path": "language_model.model.layers.2.linear_attn.in_proj_b", |
| "layer": 2, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.013773699583866855 |
| }, |
| { |
| "path": "language_model.model.layers.2.linear_attn.in_proj_a", |
| "layer": 2, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.011592477724978621 |
| }, |
| { |
| "path": "language_model.model.layers.2.linear_attn.out_proj", |
| "layer": 2, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.09072199036986346 |
| }, |
| { |
| "path": "language_model.model.layers.2.mlp.gate_proj", |
| "layer": 2, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.04075199032161302 |
| }, |
| { |
| "path": "language_model.model.layers.2.mlp.down_proj", |
| "layer": 2, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.057830560223890416 |
| }, |
| { |
| "path": "language_model.model.layers.2.mlp.up_proj", |
| "layer": 2, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.03367332028470472 |
| }, |
| { |
| "path": "language_model.model.layers.3.self_attn.q_proj", |
| "layer": 3, |
| "layer_type": "full_attention", |
| "nparams": 4194304, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0363731313499352 |
| }, |
| { |
| "path": "language_model.model.layers.3.self_attn.k_proj", |
| "layer": 3, |
| "layer_type": "full_attention", |
| "nparams": 524288, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.017387628331155 |
| }, |
| { |
| "path": "language_model.model.layers.3.self_attn.v_proj", |
| "layer": 3, |
| "layer_type": "full_attention", |
| "nparams": 524288, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.03509479391362726 |
| }, |
| { |
| "path": "language_model.model.layers.3.self_attn.o_proj", |
| "layer": 3, |
| "layer_type": "full_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.047125915910451774 |
| }, |
| { |
| "path": "language_model.model.layers.3.mlp.gate_proj", |
| "layer": 3, |
| "layer_type": "full_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.019515000915455945 |
| }, |
| { |
| "path": "language_model.model.layers.3.mlp.down_proj", |
| "layer": 3, |
| "layer_type": "full_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.061700401195738386 |
| }, |
| { |
| "path": "language_model.model.layers.3.mlp.up_proj", |
| "layer": 3, |
| "layer_type": "full_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.042194126852922675 |
| }, |
| { |
| "path": "language_model.model.layers.4.linear_attn.in_proj_qkv", |
| "layer": 4, |
| "layer_type": "linear_attention", |
| "nparams": 6291456, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.02397921018909746 |
| }, |
| { |
| "path": "language_model.model.layers.4.linear_attn.in_proj_z", |
| "layer": 4, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.023035051569588204 |
| }, |
| { |
| "path": "language_model.model.layers.4.linear_attn.in_proj_b", |
| "layer": 4, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.01564397704726879 |
| }, |
| { |
| "path": "language_model.model.layers.4.linear_attn.in_proj_a", |
| "layer": 4, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 2, |
| "kind": "ternary", |
| "kl": 0.013407841441155961 |
| }, |
| { |
| "path": "language_model.model.layers.4.linear_attn.out_proj", |
| "layer": 4, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.023875757661911464 |
| }, |
| { |
| "path": "language_model.model.layers.4.mlp.gate_proj", |
| "layer": 4, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.031587638719582915 |
| }, |
| { |
| "path": "language_model.model.layers.4.mlp.down_proj", |
| "layer": 4, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.136789628508474 |
| }, |
| { |
| "path": "language_model.model.layers.4.mlp.up_proj", |
| "layer": 4, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.03427242462143647 |
| }, |
| { |
| "path": "language_model.model.layers.5.linear_attn.in_proj_qkv", |
| "layer": 5, |
| "layer_type": "linear_attention", |
| "nparams": 6291456, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.03755131517139272 |
| }, |
| { |
| "path": "language_model.model.layers.5.linear_attn.in_proj_z", |
| "layer": 5, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.02201721690262345 |
| }, |
| { |
| "path": "language_model.model.layers.5.linear_attn.in_proj_b", |
| "layer": 5, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.014129301163061457 |
| }, |
| { |
| "path": "language_model.model.layers.5.linear_attn.in_proj_a", |
| "layer": 5, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.00921678707939411 |
| }, |
| { |
| "path": "language_model.model.layers.5.linear_attn.out_proj", |
| "layer": 5, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.05138253773226259 |
| }, |
| { |
| "path": "language_model.model.layers.5.mlp.gate_proj", |
| "layer": 5, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.032154125558556325 |
| }, |
| { |
| "path": "language_model.model.layers.5.mlp.down_proj", |
| "layer": 5, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.043968335455040024 |
| }, |
| { |
| "path": "language_model.model.layers.5.mlp.up_proj", |
| "layer": 5, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.062463296864341974 |
| }, |
| { |
| "path": "language_model.model.layers.6.linear_attn.in_proj_qkv", |
| "layer": 6, |
| "layer_type": "linear_attention", |
| "nparams": 6291456, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.051413092635799004 |
| }, |
| { |
| "path": "language_model.model.layers.6.linear_attn.in_proj_z", |
| "layer": 6, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.02528388093525213 |
| }, |
| { |
| "path": "language_model.model.layers.6.linear_attn.in_proj_b", |
| "layer": 6, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.010495090312282684 |
| }, |
| { |
| "path": "language_model.model.layers.6.linear_attn.in_proj_a", |
| "layer": 6, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.010463668713146171 |
| }, |
| { |
| "path": "language_model.model.layers.6.linear_attn.out_proj", |
| "layer": 6, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.03687874271655201 |
| }, |
| { |
| "path": "language_model.model.layers.6.mlp.gate_proj", |
| "layer": 6, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.041433529224225885 |
| }, |
| { |
| "path": "language_model.model.layers.6.mlp.down_proj", |
| "layer": 6, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.04199239106005209 |
| }, |
| { |
| "path": "language_model.model.layers.6.mlp.up_proj", |
| "layer": 6, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.09370311534083604 |
| }, |
| { |
| "path": "language_model.model.layers.7.self_attn.q_proj", |
| "layer": 7, |
| "layer_type": "full_attention", |
| "nparams": 4194304, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0639473471482219 |
| }, |
| { |
| "path": "language_model.model.layers.7.self_attn.k_proj", |
| "layer": 7, |
| "layer_type": "full_attention", |
| "nparams": 524288, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.01995626164111499 |
| }, |
| { |
| "path": "language_model.model.layers.7.self_attn.v_proj", |
| "layer": 7, |
| "layer_type": "full_attention", |
| "nparams": 524288, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.03879161901214778 |
| }, |
| { |
| "path": "language_model.model.layers.7.self_attn.o_proj", |
| "layer": 7, |
| "layer_type": "full_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.07775133380157301 |
| }, |
| { |
| "path": "language_model.model.layers.7.mlp.gate_proj", |
| "layer": 7, |
| "layer_type": "full_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.01902822349416639 |
| }, |
| { |
| "path": "language_model.model.layers.7.mlp.down_proj", |
| "layer": 7, |
| "layer_type": "full_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.02074294201563859 |
| }, |
| { |
| "path": "language_model.model.layers.7.mlp.up_proj", |
| "layer": 7, |
| "layer_type": "full_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.03005311131768014 |
| }, |
| { |
| "path": "language_model.model.layers.8.linear_attn.in_proj_qkv", |
| "layer": 8, |
| "layer_type": "linear_attention", |
| "nparams": 6291456, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.026959423507237353 |
| }, |
| { |
| "path": "language_model.model.layers.8.linear_attn.in_proj_z", |
| "layer": 8, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.010542849449285302 |
| }, |
| { |
| "path": "language_model.model.layers.8.linear_attn.in_proj_b", |
| "layer": 8, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.006259536017413378 |
| }, |
| { |
| "path": "language_model.model.layers.8.linear_attn.in_proj_a", |
| "layer": 8, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.005066103889982274 |
| }, |
| { |
| "path": "language_model.model.layers.8.linear_attn.out_proj", |
| "layer": 8, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.04619141676349868 |
| }, |
| { |
| "path": "language_model.model.layers.8.mlp.gate_proj", |
| "layer": 8, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.019728178176647466 |
| }, |
| { |
| "path": "language_model.model.layers.8.mlp.down_proj", |
| "layer": 8, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.026803769680060083 |
| }, |
| { |
| "path": "language_model.model.layers.8.mlp.up_proj", |
| "layer": 8, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.022680045334707168 |
| }, |
| { |
| "path": "language_model.model.layers.9.linear_attn.in_proj_qkv", |
| "layer": 9, |
| "layer_type": "linear_attention", |
| "nparams": 6291456, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.03976326973341923 |
| }, |
| { |
| "path": "language_model.model.layers.9.linear_attn.in_proj_z", |
| "layer": 9, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.018086428763740822 |
| }, |
| { |
| "path": "language_model.model.layers.9.linear_attn.in_proj_b", |
| "layer": 9, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.008665403679117296 |
| }, |
| { |
| "path": "language_model.model.layers.9.linear_attn.in_proj_a", |
| "layer": 9, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.005111378159239233 |
| }, |
| { |
| "path": "language_model.model.layers.9.linear_attn.out_proj", |
| "layer": 9, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.024027441586703807 |
| }, |
| { |
| "path": "language_model.model.layers.9.mlp.gate_proj", |
| "layer": 9, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.017517147359535766 |
| }, |
| { |
| "path": "language_model.model.layers.9.mlp.down_proj", |
| "layer": 9, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.018829855820785585 |
| }, |
| { |
| "path": "language_model.model.layers.9.mlp.up_proj", |
| "layer": 9, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.021916257691088493 |
| }, |
| { |
| "path": "language_model.model.layers.10.linear_attn.in_proj_qkv", |
| "layer": 10, |
| "layer_type": "linear_attention", |
| "nparams": 6291456, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.04024679641347246 |
| }, |
| { |
| "path": "language_model.model.layers.10.linear_attn.in_proj_z", |
| "layer": 10, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.011116799937381346 |
| }, |
| { |
| "path": "language_model.model.layers.10.linear_attn.in_proj_b", |
| "layer": 10, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0043433013822391404 |
| }, |
| { |
| "path": "language_model.model.layers.10.linear_attn.in_proj_a", |
| "layer": 10, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.004665172839471169 |
| }, |
| { |
| "path": "language_model.model.layers.10.linear_attn.out_proj", |
| "layer": 10, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.02778384028510572 |
| }, |
| { |
| "path": "language_model.model.layers.10.mlp.gate_proj", |
| "layer": 10, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.007522035321253345 |
| }, |
| { |
| "path": "language_model.model.layers.10.mlp.down_proj", |
| "layer": 10, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.015915166493531592 |
| }, |
| { |
| "path": "language_model.model.layers.10.mlp.up_proj", |
| "layer": 10, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.017182175372423135 |
| }, |
| { |
| "path": "language_model.model.layers.11.self_attn.q_proj", |
| "layer": 11, |
| "layer_type": "full_attention", |
| "nparams": 4194304, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.01918938407635554 |
| }, |
| { |
| "path": "language_model.model.layers.11.self_attn.k_proj", |
| "layer": 11, |
| "layer_type": "full_attention", |
| "nparams": 524288, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.02101637260234458 |
| }, |
| { |
| "path": "language_model.model.layers.11.self_attn.v_proj", |
| "layer": 11, |
| "layer_type": "full_attention", |
| "nparams": 524288, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.03523801972554967 |
| }, |
| { |
| "path": "language_model.model.layers.11.self_attn.o_proj", |
| "layer": 11, |
| "layer_type": "full_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.044128938397831746 |
| }, |
| { |
| "path": "language_model.model.layers.11.mlp.gate_proj", |
| "layer": 11, |
| "layer_type": "full_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.006564071190068745 |
| }, |
| { |
| "path": "language_model.model.layers.11.mlp.down_proj", |
| "layer": 11, |
| "layer_type": "full_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.009954753161174175 |
| }, |
| { |
| "path": "language_model.model.layers.11.mlp.up_proj", |
| "layer": 11, |
| "layer_type": "full_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.016408592796186942 |
| }, |
| { |
| "path": "language_model.model.layers.12.linear_attn.in_proj_qkv", |
| "layer": 12, |
| "layer_type": "linear_attention", |
| "nparams": 6291456, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.012744699638467517 |
| }, |
| { |
| "path": "language_model.model.layers.12.linear_attn.in_proj_z", |
| "layer": 12, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.004262488847540524 |
| }, |
| { |
| "path": "language_model.model.layers.12.linear_attn.in_proj_b", |
| "layer": 12, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.003491112939298953 |
| }, |
| { |
| "path": "language_model.model.layers.12.linear_attn.in_proj_a", |
| "layer": 12, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0023025657529113326 |
| }, |
| { |
| "path": "language_model.model.layers.12.linear_attn.out_proj", |
| "layer": 12, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.006398780878283918 |
| }, |
| { |
| "path": "language_model.model.layers.12.mlp.gate_proj", |
| "layer": 12, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.005122177783788138 |
| }, |
| { |
| "path": "language_model.model.layers.12.mlp.down_proj", |
| "layer": 12, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.00533927442938409 |
| }, |
| { |
| "path": "language_model.model.layers.12.mlp.up_proj", |
| "layer": 12, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0061679559347479485 |
| }, |
| { |
| "path": "language_model.model.layers.13.linear_attn.in_proj_qkv", |
| "layer": 13, |
| "layer_type": "linear_attention", |
| "nparams": 6291456, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.011533852265464987 |
| }, |
| { |
| "path": "language_model.model.layers.13.linear_attn.in_proj_z", |
| "layer": 13, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0028721824605509763 |
| }, |
| { |
| "path": "language_model.model.layers.13.linear_attn.in_proj_b", |
| "layer": 13, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0023588539987408805 |
| }, |
| { |
| "path": "language_model.model.layers.13.linear_attn.in_proj_a", |
| "layer": 13, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.00199120487849583 |
| }, |
| { |
| "path": "language_model.model.layers.13.linear_attn.out_proj", |
| "layer": 13, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.004224189863711925 |
| }, |
| { |
| "path": "language_model.model.layers.13.mlp.gate_proj", |
| "layer": 13, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.012466730321177164 |
| }, |
| { |
| "path": "language_model.model.layers.13.mlp.down_proj", |
| "layer": 13, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.002558384258649209 |
| }, |
| { |
| "path": "language_model.model.layers.13.mlp.up_proj", |
| "layer": 13, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0029687005528531204 |
| }, |
| { |
| "path": "language_model.model.layers.14.linear_attn.in_proj_qkv", |
| "layer": 14, |
| "layer_type": "linear_attention", |
| "nparams": 6291456, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.007997483313712273 |
| }, |
| { |
| "path": "language_model.model.layers.14.linear_attn.in_proj_z", |
| "layer": 14, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.0052743370907232245 |
| }, |
| { |
| "path": "language_model.model.layers.14.linear_attn.in_proj_b", |
| "layer": 14, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0024342861764566695 |
| }, |
| { |
| "path": "language_model.model.layers.14.linear_attn.in_proj_a", |
| "layer": 14, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0016676004025067079 |
| }, |
| { |
| "path": "language_model.model.layers.14.linear_attn.out_proj", |
| "layer": 14, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.004083662991277806 |
| }, |
| { |
| "path": "language_model.model.layers.14.mlp.gate_proj", |
| "layer": 14, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.008108828858961267 |
| }, |
| { |
| "path": "language_model.model.layers.14.mlp.down_proj", |
| "layer": 14, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.004123490271222968 |
| }, |
| { |
| "path": "language_model.model.layers.14.mlp.up_proj", |
| "layer": 14, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.007036047053022412 |
| }, |
| { |
| "path": "language_model.model.layers.15.self_attn.q_proj", |
| "layer": 15, |
| "layer_type": "full_attention", |
| "nparams": 4194304, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.004153576767074394 |
| }, |
| { |
| "path": "language_model.model.layers.15.self_attn.k_proj", |
| "layer": 15, |
| "layer_type": "full_attention", |
| "nparams": 524288, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.004112759303064329 |
| }, |
| { |
| "path": "language_model.model.layers.15.self_attn.v_proj", |
| "layer": 15, |
| "layer_type": "full_attention", |
| "nparams": 524288, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0045544497693278275 |
| }, |
| { |
| "path": "language_model.model.layers.15.self_attn.o_proj", |
| "layer": 15, |
| "layer_type": "full_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.003484462206984148 |
| }, |
| { |
| "path": "language_model.model.layers.15.mlp.gate_proj", |
| "layer": 15, |
| "layer_type": "full_attention", |
| "nparams": 3670016, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.002868431546808957 |
| }, |
| { |
| "path": "language_model.model.layers.15.mlp.down_proj", |
| "layer": 15, |
| "layer_type": "full_attention", |
| "nparams": 3670016, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.006095650806291535 |
| }, |
| { |
| "path": "language_model.model.layers.15.mlp.up_proj", |
| "layer": 15, |
| "layer_type": "full_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.004815552077497851 |
| }, |
| { |
| "path": "language_model.model.layers.16.linear_attn.in_proj_qkv", |
| "layer": 16, |
| "layer_type": "linear_attention", |
| "nparams": 6291456, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.00292960581312596 |
| }, |
| { |
| "path": "language_model.model.layers.16.linear_attn.in_proj_z", |
| "layer": 16, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0034485573524671727 |
| }, |
| { |
| "path": "language_model.model.layers.16.linear_attn.in_proj_b", |
| "layer": 16, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.0012654082102243791 |
| }, |
| { |
| "path": "language_model.model.layers.16.linear_attn.in_proj_a", |
| "layer": 16, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0012813463656705713 |
| }, |
| { |
| "path": "language_model.model.layers.16.linear_attn.out_proj", |
| "layer": 16, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.005193882758604845 |
| }, |
| { |
| "path": "language_model.model.layers.16.mlp.gate_proj", |
| "layer": 16, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.002351040174577789 |
| }, |
| { |
| "path": "language_model.model.layers.16.mlp.down_proj", |
| "layer": 16, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.00440584063789979 |
| }, |
| { |
| "path": "language_model.model.layers.16.mlp.up_proj", |
| "layer": 16, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.007104415338209336 |
| }, |
| { |
| "path": "language_model.model.layers.17.linear_attn.in_proj_qkv", |
| "layer": 17, |
| "layer_type": "linear_attention", |
| "nparams": 6291456, |
| "bits": 2, |
| "kind": "ternary", |
| "kl": 0.0021934894088310236 |
| }, |
| { |
| "path": "language_model.model.layers.17.linear_attn.in_proj_z", |
| "layer": 17, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0032236417576331686 |
| }, |
| { |
| "path": "language_model.model.layers.17.linear_attn.in_proj_b", |
| "layer": 17, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.000886947803901282 |
| }, |
| { |
| "path": "language_model.model.layers.17.linear_attn.in_proj_a", |
| "layer": 17, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.000976995525569164 |
| }, |
| { |
| "path": "language_model.model.layers.17.linear_attn.out_proj", |
| "layer": 17, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.002114814895788902 |
| }, |
| { |
| "path": "language_model.model.layers.17.mlp.gate_proj", |
| "layer": 17, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.004665009833481794 |
| }, |
| { |
| "path": "language_model.model.layers.17.mlp.down_proj", |
| "layer": 17, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.004484325279917158 |
| }, |
| { |
| "path": "language_model.model.layers.17.mlp.up_proj", |
| "layer": 17, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 2, |
| "kind": "ternary", |
| "kl": 0.005121177837445393 |
| }, |
| { |
| "path": "language_model.model.layers.18.linear_attn.in_proj_qkv", |
| "layer": 18, |
| "layer_type": "linear_attention", |
| "nparams": 6291456, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.006290895747047132 |
| }, |
| { |
| "path": "language_model.model.layers.18.linear_attn.in_proj_z", |
| "layer": 18, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.002068792589064126 |
| }, |
| { |
| "path": "language_model.model.layers.18.linear_attn.in_proj_b", |
| "layer": 18, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.0008810069546544934 |
| }, |
| { |
| "path": "language_model.model.layers.18.linear_attn.in_proj_a", |
| "layer": 18, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.000987608700835434 |
| }, |
| { |
| "path": "language_model.model.layers.18.linear_attn.out_proj", |
| "layer": 18, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.002887347314595934 |
| }, |
| { |
| "path": "language_model.model.layers.18.mlp.gate_proj", |
| "layer": 18, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.0018628583013199987 |
| }, |
| { |
| "path": "language_model.model.layers.18.mlp.down_proj", |
| "layer": 18, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0023940537857527703 |
| }, |
| { |
| "path": "language_model.model.layers.18.mlp.up_proj", |
| "layer": 18, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.006746079837461991 |
| }, |
| { |
| "path": "language_model.model.layers.19.self_attn.q_proj", |
| "layer": 19, |
| "layer_type": "full_attention", |
| "nparams": 4194304, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.006032338261324889 |
| }, |
| { |
| "path": "language_model.model.layers.19.self_attn.k_proj", |
| "layer": 19, |
| "layer_type": "full_attention", |
| "nparams": 524288, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0021031284974654106 |
| }, |
| { |
| "path": "language_model.model.layers.19.self_attn.v_proj", |
| "layer": 19, |
| "layer_type": "full_attention", |
| "nparams": 524288, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.004348253480209944 |
| }, |
| { |
| "path": "language_model.model.layers.19.self_attn.o_proj", |
| "layer": 19, |
| "layer_type": "full_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.004557360657928573 |
| }, |
| { |
| "path": "language_model.model.layers.19.mlp.gate_proj", |
| "layer": 19, |
| "layer_type": "full_attention", |
| "nparams": 3670016, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0029232948535193653 |
| }, |
| { |
| "path": "language_model.model.layers.19.mlp.down_proj", |
| "layer": 19, |
| "layer_type": "full_attention", |
| "nparams": 3670016, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0058071099995981815 |
| }, |
| { |
| "path": "language_model.model.layers.19.mlp.up_proj", |
| "layer": 19, |
| "layer_type": "full_attention", |
| "nparams": 3670016, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.0023828522802972463 |
| }, |
| { |
| "path": "language_model.model.layers.20.linear_attn.in_proj_qkv", |
| "layer": 20, |
| "layer_type": "linear_attention", |
| "nparams": 6291456, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.008125929608638258 |
| }, |
| { |
| "path": "language_model.model.layers.20.linear_attn.in_proj_z", |
| "layer": 20, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0017258157022901578 |
| }, |
| { |
| "path": "language_model.model.layers.20.linear_attn.in_proj_b", |
| "layer": 20, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0005154021203862869 |
| }, |
| { |
| "path": "language_model.model.layers.20.linear_attn.in_proj_a", |
| "layer": 20, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 2, |
| "kind": "ternary", |
| "kl": 0.0005305295310921597 |
| }, |
| { |
| "path": "language_model.model.layers.20.linear_attn.out_proj", |
| "layer": 20, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.0030152998807947977 |
| }, |
| { |
| "path": "language_model.model.layers.20.mlp.gate_proj", |
| "layer": 20, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0022819108116763583 |
| }, |
| { |
| "path": "language_model.model.layers.20.mlp.down_proj", |
| "layer": 20, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0015101953736824964 |
| }, |
| { |
| "path": "language_model.model.layers.20.mlp.up_proj", |
| "layer": 20, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 2, |
| "kind": "ternary", |
| "kl": 0.004383521037158472 |
| }, |
| { |
| "path": "language_model.model.layers.21.linear_attn.in_proj_qkv", |
| "layer": 21, |
| "layer_type": "linear_attention", |
| "nparams": 6291456, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0035359140742138107 |
| }, |
| { |
| "path": "language_model.model.layers.21.linear_attn.in_proj_z", |
| "layer": 21, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0033209959668630907 |
| }, |
| { |
| "path": "language_model.model.layers.21.linear_attn.in_proj_b", |
| "layer": 21, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0005307516921509054 |
| }, |
| { |
| "path": "language_model.model.layers.21.linear_attn.in_proj_a", |
| "layer": 21, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0002769620310376814 |
| }, |
| { |
| "path": "language_model.model.layers.21.linear_attn.out_proj", |
| "layer": 21, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.0017829395230716451 |
| }, |
| { |
| "path": "language_model.model.layers.21.mlp.gate_proj", |
| "layer": 21, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 2, |
| "kind": "ternary", |
| "kl": 0.004034291358501529 |
| }, |
| { |
| "path": "language_model.model.layers.21.mlp.down_proj", |
| "layer": 21, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.0008362982179293819 |
| }, |
| { |
| "path": "language_model.model.layers.21.mlp.up_proj", |
| "layer": 21, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.003467660124694521 |
| }, |
| { |
| "path": "language_model.model.layers.22.linear_attn.in_proj_qkv", |
| "layer": 22, |
| "layer_type": "linear_attention", |
| "nparams": 6291456, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.006269973941838318 |
| }, |
| { |
| "path": "language_model.model.layers.22.linear_attn.in_proj_z", |
| "layer": 22, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.0034496159643663885 |
| }, |
| { |
| "path": "language_model.model.layers.22.linear_attn.in_proj_b", |
| "layer": 22, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0005182467336934553 |
| }, |
| { |
| "path": "language_model.model.layers.22.linear_attn.in_proj_a", |
| "layer": 22, |
| "layer_type": "linear_attention", |
| "nparams": 16384, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.0003643164638947492 |
| }, |
| { |
| "path": "language_model.model.layers.22.linear_attn.out_proj", |
| "layer": 22, |
| "layer_type": "linear_attention", |
| "nparams": 2097152, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.003477276947365615 |
| }, |
| { |
| "path": "language_model.model.layers.22.mlp.gate_proj", |
| "layer": 22, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.005100013411379256 |
| }, |
| { |
| "path": "language_model.model.layers.22.mlp.down_proj", |
| "layer": 22, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.0033566558855619606 |
| }, |
| { |
| "path": "language_model.model.layers.22.mlp.up_proj", |
| "layer": 22, |
| "layer_type": "linear_attention", |
| "nparams": 3670016, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.00313410047627708 |
| }, |
| { |
| "path": "language_model.model.layers.23.self_attn.q_proj", |
| "layer": 23, |
| "layer_type": "full_attention", |
| "nparams": 4194304, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.004782156469623039 |
| }, |
| { |
| "path": "language_model.model.layers.23.self_attn.k_proj", |
| "layer": 23, |
| "layer_type": "full_attention", |
| "nparams": 524288, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.006514890193217124 |
| }, |
| { |
| "path": "language_model.model.layers.23.self_attn.v_proj", |
| "layer": 23, |
| "layer_type": "full_attention", |
| "nparams": 524288, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.002427561433381982 |
| }, |
| { |
| "path": "language_model.model.layers.23.self_attn.o_proj", |
| "layer": 23, |
| "layer_type": "full_attention", |
| "nparams": 2097152, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.002634655396783468 |
| }, |
| { |
| "path": "language_model.model.layers.23.mlp.gate_proj", |
| "layer": 23, |
| "layer_type": "full_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.001242476476743588 |
| }, |
| { |
| "path": "language_model.model.layers.23.mlp.down_proj", |
| "layer": 23, |
| "layer_type": "full_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.029779743010173505 |
| }, |
| { |
| "path": "language_model.model.layers.23.mlp.up_proj", |
| "layer": 23, |
| "layer_type": "full_attention", |
| "nparams": 3670016, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.006386583194459594 |
| } |
| ] |
| } |
|
|
