| { |
| "average_bits": 3.0498071979434447, |
| "embed_bits": 8, |
| "embed_uses_tied_head": false, |
| "embed_reason": "The decision readout is last_hidden @ readout.T and does not use the tied lm_head. The embedding is input-only and is stored as 8-bit affine, group 64.", |
| "embed_sum_kl": { |
| "4": 0.011286197474019719, |
| "8": 0.004745219702161558 |
| }, |
| "embed_bytes": { |
| "4": 357580800, |
| "8": 675430400 |
| }, |
| "joint_refine": { |
| "calibration_n": 20, |
| "calibration_ids": [ |
| "mnli/validation_matched/3117/relation", |
| "banking77/test/1454/intent", |
| "boolq/validation/2354/answer", |
| "banking77/test/1793/permuted/intent", |
| "composition/v3-composition-dev/conjunction/3/relevant/a/decision", |
| "banking77/test/2120/none_absent/intent", |
| "amazon/test/605/stars", |
| "mnli/validation_matched/79/relation", |
| "dbpedia14/test/40231/category", |
| "dbpedia14/test/48873/category", |
| "mnli/validation_matched/1139/relation", |
| "composition/v3-composition-dev/nested_and/3/relevant/a/decision", |
| "contrastive/age_eligibility/v3-control-dev-age_eligibility-0012/b/decision", |
| "yelp/test/10368/rating", |
| "agnews/test/1493/topic", |
| "composition/v3-composition-dev/conditional/2/irrelevant/b/decision", |
| "boolq/validation/2399/answer", |
| "agnews/test/2095/topic", |
| "mnli/validation_matched/8140/relation", |
| "contrastive/age_eligibility/v3-control-dev-age_eligibility-0013/b/decision" |
| ], |
| "joint_kl_before": 0.27327187773454925, |
| "joint_kl_after": 0.2351137807324731, |
| "swaps": 4, |
| "accepted": [ |
| { |
| "up": "language_model.model.layers.5.linear_attn.in_proj_z", |
| "down": "language_model.model.layers.16.linear_attn.in_proj_z", |
| "up_to": "4", |
| "down_to": "3", |
| "joint_kl": 0.24233961590783762, |
| "predicted": 5.7052450264859396e-05 |
| }, |
| { |
| "up": "language_model.model.layers.24.linear_attn.in_proj_a", |
| "down": "language_model.model.layers.13.linear_attn.in_proj_b", |
| "up_to": "3", |
| "down_to": "3", |
| "joint_kl": 0.23988770723326833, |
| "predicted": 0.0001434830382605536 |
| }, |
| { |
| "up": "language_model.model.layers.5.linear_attn.in_proj_b", |
| "down": "language_model.model.layers.24.linear_attn.in_proj_a", |
| "up_to": "4", |
| "down_to": "ternary", |
| "joint_kl": 0.23789553741053193, |
| "predicted": 1.5698582511071034e-05 |
| }, |
| { |
| "up": "language_model.model.layers.25.linear_attn.in_proj_a", |
| "down": "language_model.model.layers.5.linear_attn.in_proj_b", |
| "up_to": "3", |
| "down_to": "3", |
| "joint_kl": 0.2351137807324731, |
| "predicted": 1.9772355902315797e-05 |
| } |
| ] |
| }, |
| "assignment": [ |
| { |
| "path": "language_model.model.layers.0.linear_attn.in_proj_qkv", |
| "layer": 0, |
| "layer_type": "linear_attention", |
| "nparams": 20971520, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.010448446673969113 |
| }, |
| { |
| "path": "language_model.model.layers.0.linear_attn.in_proj_z", |
| "layer": 0, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.007454903243540781 |
| }, |
| { |
| "path": "language_model.model.layers.0.linear_attn.in_proj_b", |
| "layer": 0, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.0033223642585997228 |
| }, |
| { |
| "path": "language_model.model.layers.0.linear_attn.in_proj_a", |
| "layer": 0, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0035474445607511793 |
| }, |
| { |
| "path": "language_model.model.layers.0.linear_attn.out_proj", |
| "layer": 0, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.06154597570566438 |
| }, |
| { |
| "path": "language_model.model.layers.0.mlp.gate_proj", |
| "layer": 0, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.012915829515700692 |
| }, |
| { |
| "path": "language_model.model.layers.0.mlp.down_proj", |
| "layer": 0, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.006924833638929763 |
| }, |
| { |
| "path": "language_model.model.layers.0.mlp.up_proj", |
| "layer": 0, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.010924317433800512 |
| }, |
| { |
| "path": "language_model.model.layers.1.linear_attn.in_proj_qkv", |
| "layer": 1, |
| "layer_type": "linear_attention", |
| "nparams": 20971520, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.010014838784017786 |
| }, |
| { |
| "path": "language_model.model.layers.1.linear_attn.in_proj_z", |
| "layer": 1, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.006952277388950299 |
| }, |
| { |
| "path": "language_model.model.layers.1.linear_attn.in_proj_b", |
| "layer": 1, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0032649088695483855 |
| }, |
| { |
| "path": "language_model.model.layers.1.linear_attn.in_proj_a", |
| "layer": 1, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0040076704179294684 |
| }, |
| { |
| "path": "language_model.model.layers.1.linear_attn.out_proj", |
| "layer": 1, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.005802257928629042 |
| }, |
| { |
| "path": "language_model.model.layers.1.mlp.gate_proj", |
| "layer": 1, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.01244909772269 |
| }, |
| { |
| "path": "language_model.model.layers.1.mlp.down_proj", |
| "layer": 1, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0065094802589678565 |
| }, |
| { |
| "path": "language_model.model.layers.1.mlp.up_proj", |
| "layer": 1, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.014826100961388559 |
| }, |
| { |
| "path": "language_model.model.layers.2.linear_attn.in_proj_qkv", |
| "layer": 2, |
| "layer_type": "linear_attention", |
| "nparams": 20971520, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.012259290078230753 |
| }, |
| { |
| "path": "language_model.model.layers.2.linear_attn.in_proj_z", |
| "layer": 2, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.006292201548291481 |
| }, |
| { |
| "path": "language_model.model.layers.2.linear_attn.in_proj_b", |
| "layer": 2, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.004659571511338003 |
| }, |
| { |
| "path": "language_model.model.layers.2.linear_attn.in_proj_a", |
| "layer": 2, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.00400962032856176 |
| }, |
| { |
| "path": "language_model.model.layers.2.linear_attn.out_proj", |
| "layer": 2, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.008425798009871279 |
| }, |
| { |
| "path": "language_model.model.layers.2.mlp.gate_proj", |
| "layer": 2, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.008391449294443439 |
| }, |
| { |
| "path": "language_model.model.layers.2.mlp.down_proj", |
| "layer": 2, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.005654083152917417 |
| }, |
| { |
| "path": "language_model.model.layers.2.mlp.up_proj", |
| "layer": 2, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0231763729959581 |
| }, |
| { |
| "path": "language_model.model.layers.3.self_attn.q_proj", |
| "layer": 3, |
| "layer_type": "full_attention", |
| "nparams": 20971520, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0060903110340457785 |
| }, |
| { |
| "path": "language_model.model.layers.3.self_attn.k_proj", |
| "layer": 3, |
| "layer_type": "full_attention", |
| "nparams": 2621440, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0033800215613122805 |
| }, |
| { |
| "path": "language_model.model.layers.3.self_attn.v_proj", |
| "layer": 3, |
| "layer_type": "full_attention", |
| "nparams": 2621440, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.01283825919462436 |
| }, |
| { |
| "path": "language_model.model.layers.3.self_attn.o_proj", |
| "layer": 3, |
| "layer_type": "full_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.009046701374450225 |
| }, |
| { |
| "path": "language_model.model.layers.3.mlp.gate_proj", |
| "layer": 3, |
| "layer_type": "full_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.01050212096987629 |
| }, |
| { |
| "path": "language_model.model.layers.3.mlp.down_proj", |
| "layer": 3, |
| "layer_type": "full_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0061818342546688335 |
| }, |
| { |
| "path": "language_model.model.layers.3.mlp.up_proj", |
| "layer": 3, |
| "layer_type": "full_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.01611825974729804 |
| }, |
| { |
| "path": "language_model.model.layers.4.linear_attn.in_proj_qkv", |
| "layer": 4, |
| "layer_type": "linear_attention", |
| "nparams": 20971520, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.013024711441356987 |
| }, |
| { |
| "path": "language_model.model.layers.4.linear_attn.in_proj_z", |
| "layer": 4, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.00429976138543272 |
| }, |
| { |
| "path": "language_model.model.layers.4.linear_attn.in_proj_b", |
| "layer": 4, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0034435942018261646 |
| }, |
| { |
| "path": "language_model.model.layers.4.linear_attn.in_proj_a", |
| "layer": 4, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.00230237893619504 |
| }, |
| { |
| "path": "language_model.model.layers.4.linear_attn.out_proj", |
| "layer": 4, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.005720972922117528 |
| }, |
| { |
| "path": "language_model.model.layers.4.mlp.gate_proj", |
| "layer": 4, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.008319276305372148 |
| }, |
| { |
| "path": "language_model.model.layers.4.mlp.down_proj", |
| "layer": 4, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.00833979089090999 |
| }, |
| { |
| "path": "language_model.model.layers.4.mlp.up_proj", |
| "layer": 4, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.008235880725585083 |
| }, |
| { |
| "path": "language_model.model.layers.5.linear_attn.in_proj_qkv", |
| "layer": 5, |
| "layer_type": "linear_attention", |
| "nparams": 20971520, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.008891306822033977 |
| }, |
| { |
| "path": "language_model.model.layers.5.linear_attn.in_proj_z", |
| "layer": 5, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0049944552043391915 |
| }, |
| { |
| "path": "language_model.model.layers.5.linear_attn.in_proj_b", |
| "layer": 5, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.004080156970837005 |
| }, |
| { |
| "path": "language_model.model.layers.5.linear_attn.in_proj_a", |
| "layer": 5, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0026296631091524943 |
| }, |
| { |
| "path": "language_model.model.layers.5.linear_attn.out_proj", |
| "layer": 5, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.005161598731319505 |
| }, |
| { |
| "path": "language_model.model.layers.5.mlp.gate_proj", |
| "layer": 5, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0062223512029718495 |
| }, |
| { |
| "path": "language_model.model.layers.5.mlp.down_proj", |
| "layer": 5, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.004617157800457818 |
| }, |
| { |
| "path": "language_model.model.layers.5.mlp.up_proj", |
| "layer": 5, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.01450103365186557 |
| }, |
| { |
| "path": "language_model.model.layers.6.linear_attn.in_proj_qkv", |
| "layer": 6, |
| "layer_type": "linear_attention", |
| "nparams": 20971520, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.010250001698275295 |
| }, |
| { |
| "path": "language_model.model.layers.6.linear_attn.in_proj_z", |
| "layer": 6, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.005800948212441958 |
| }, |
| { |
| "path": "language_model.model.layers.6.linear_attn.in_proj_b", |
| "layer": 6, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0033723909629199142 |
| }, |
| { |
| "path": "language_model.model.layers.6.linear_attn.in_proj_a", |
| "layer": 6, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0035418093499744747 |
| }, |
| { |
| "path": "language_model.model.layers.6.linear_attn.out_proj", |
| "layer": 6, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.00722899541177021 |
| }, |
| { |
| "path": "language_model.model.layers.6.mlp.gate_proj", |
| "layer": 6, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.00866014900850607 |
| }, |
| { |
| "path": "language_model.model.layers.6.mlp.down_proj", |
| "layer": 6, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.010980755737561482 |
| }, |
| { |
| "path": "language_model.model.layers.6.mlp.up_proj", |
| "layer": 6, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.01381060262887852 |
| }, |
| { |
| "path": "language_model.model.layers.7.self_attn.q_proj", |
| "layer": 7, |
| "layer_type": "full_attention", |
| "nparams": 20971520, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.009426559410376102 |
| }, |
| { |
| "path": "language_model.model.layers.7.self_attn.k_proj", |
| "layer": 7, |
| "layer_type": "full_attention", |
| "nparams": 2621440, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.004889550787478134 |
| }, |
| { |
| "path": "language_model.model.layers.7.self_attn.v_proj", |
| "layer": 7, |
| "layer_type": "full_attention", |
| "nparams": 2621440, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.01626780704191527 |
| }, |
| { |
| "path": "language_model.model.layers.7.self_attn.o_proj", |
| "layer": 7, |
| "layer_type": "full_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.02200452754786024 |
| }, |
| { |
| "path": "language_model.model.layers.7.mlp.gate_proj", |
| "layer": 7, |
| "layer_type": "full_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.01603272086664161 |
| }, |
| { |
| "path": "language_model.model.layers.7.mlp.down_proj", |
| "layer": 7, |
| "layer_type": "full_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.008829930829181317 |
| }, |
| { |
| "path": "language_model.model.layers.7.mlp.up_proj", |
| "layer": 7, |
| "layer_type": "full_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.007551976471688681 |
| }, |
| { |
| "path": "language_model.model.layers.8.linear_attn.in_proj_qkv", |
| "layer": 8, |
| "layer_type": "linear_attention", |
| "nparams": 20971520, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.00636630976733739 |
| }, |
| { |
| "path": "language_model.model.layers.8.linear_attn.in_proj_z", |
| "layer": 8, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.008684421003327794 |
| }, |
| { |
| "path": "language_model.model.layers.8.linear_attn.in_proj_b", |
| "layer": 8, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0031855265832704638 |
| }, |
| { |
| "path": "language_model.model.layers.8.linear_attn.in_proj_a", |
| "layer": 8, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.002266341492353467 |
| }, |
| { |
| "path": "language_model.model.layers.8.linear_attn.out_proj", |
| "layer": 8, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.007247695609163844 |
| }, |
| { |
| "path": "language_model.model.layers.8.mlp.gate_proj", |
| "layer": 8, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.009257387943368597 |
| }, |
| { |
| "path": "language_model.model.layers.8.mlp.down_proj", |
| "layer": 8, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.011886287054785323 |
| }, |
| { |
| "path": "language_model.model.layers.8.mlp.up_proj", |
| "layer": 8, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.006176910843308165 |
| }, |
| { |
| "path": "language_model.model.layers.9.linear_attn.in_proj_qkv", |
| "layer": 9, |
| "layer_type": "linear_attention", |
| "nparams": 20971520, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.007901331850660961 |
| }, |
| { |
| "path": "language_model.model.layers.9.linear_attn.in_proj_z", |
| "layer": 9, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.006090545368305263 |
| }, |
| { |
| "path": "language_model.model.layers.9.linear_attn.in_proj_b", |
| "layer": 9, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0021647317109205087 |
| }, |
| { |
| "path": "language_model.model.layers.9.linear_attn.in_proj_a", |
| "layer": 9, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0034912846735995606 |
| }, |
| { |
| "path": "language_model.model.layers.9.linear_attn.out_proj", |
| "layer": 9, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.006964942701371921 |
| }, |
| { |
| "path": "language_model.model.layers.9.mlp.gate_proj", |
| "layer": 9, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.010906623393109898 |
| }, |
| { |
| "path": "language_model.model.layers.9.mlp.down_proj", |
| "layer": 9, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.011869402951866571 |
| }, |
| { |
| "path": "language_model.model.layers.9.mlp.up_proj", |
| "layer": 9, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.008047571413088556 |
| }, |
| { |
| "path": "language_model.model.layers.10.linear_attn.in_proj_qkv", |
| "layer": 10, |
| "layer_type": "linear_attention", |
| "nparams": 20971520, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.009507280075560192 |
| }, |
| { |
| "path": "language_model.model.layers.10.linear_attn.in_proj_z", |
| "layer": 10, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.006265319777060277 |
| }, |
| { |
| "path": "language_model.model.layers.10.linear_attn.in_proj_b", |
| "layer": 10, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0038279725069975206 |
| }, |
| { |
| "path": "language_model.model.layers.10.linear_attn.in_proj_a", |
| "layer": 10, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.003955201643937205 |
| }, |
| { |
| "path": "language_model.model.layers.10.linear_attn.out_proj", |
| "layer": 10, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0074089394346225565 |
| }, |
| { |
| "path": "language_model.model.layers.10.mlp.gate_proj", |
| "layer": 10, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.00640157690977524 |
| }, |
| { |
| "path": "language_model.model.layers.10.mlp.down_proj", |
| "layer": 10, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.008578138881508952 |
| }, |
| { |
| "path": "language_model.model.layers.10.mlp.up_proj", |
| "layer": 10, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.010964694488962036 |
| }, |
| { |
| "path": "language_model.model.layers.11.self_attn.q_proj", |
| "layer": 11, |
| "layer_type": "full_attention", |
| "nparams": 20971520, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0133026334184451 |
| }, |
| { |
| "path": "language_model.model.layers.11.self_attn.k_proj", |
| "layer": 11, |
| "layer_type": "full_attention", |
| "nparams": 2621440, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.007485433004483934 |
| }, |
| { |
| "path": "language_model.model.layers.11.self_attn.v_proj", |
| "layer": 11, |
| "layer_type": "full_attention", |
| "nparams": 2621440, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.009912732537830257 |
| }, |
| { |
| "path": "language_model.model.layers.11.self_attn.o_proj", |
| "layer": 11, |
| "layer_type": "full_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.01034751691642807 |
| }, |
| { |
| "path": "language_model.model.layers.11.mlp.gate_proj", |
| "layer": 11, |
| "layer_type": "full_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.004611820005780289 |
| }, |
| { |
| "path": "language_model.model.layers.11.mlp.down_proj", |
| "layer": 11, |
| "layer_type": "full_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.005963056204075085 |
| }, |
| { |
| "path": "language_model.model.layers.11.mlp.up_proj", |
| "layer": 11, |
| "layer_type": "full_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.005597160091559011 |
| }, |
| { |
| "path": "language_model.model.layers.12.linear_attn.in_proj_qkv", |
| "layer": 12, |
| "layer_type": "linear_attention", |
| "nparams": 20971520, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.005897109182151416 |
| }, |
| { |
| "path": "language_model.model.layers.12.linear_attn.in_proj_z", |
| "layer": 12, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.005947143164879199 |
| }, |
| { |
| "path": "language_model.model.layers.12.linear_attn.in_proj_b", |
| "layer": 12, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0030364137922867197 |
| }, |
| { |
| "path": "language_model.model.layers.12.linear_attn.in_proj_a", |
| "layer": 12, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.002990399791879363 |
| }, |
| { |
| "path": "language_model.model.layers.12.linear_attn.out_proj", |
| "layer": 12, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.004469233101413777 |
| }, |
| { |
| "path": "language_model.model.layers.12.mlp.gate_proj", |
| "layer": 12, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.005604113181054048 |
| }, |
| { |
| "path": "language_model.model.layers.12.mlp.down_proj", |
| "layer": 12, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.006729049078832552 |
| }, |
| { |
| "path": "language_model.model.layers.12.mlp.up_proj", |
| "layer": 12, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0087499496219987 |
| }, |
| { |
| "path": "language_model.model.layers.13.linear_attn.in_proj_qkv", |
| "layer": 13, |
| "layer_type": "linear_attention", |
| "nparams": 20971520, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.008554809788733155 |
| }, |
| { |
| "path": "language_model.model.layers.13.linear_attn.in_proj_z", |
| "layer": 13, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0024209885100068777 |
| }, |
| { |
| "path": "language_model.model.layers.13.linear_attn.in_proj_b", |
| "layer": 13, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.003163799510455978 |
| }, |
| { |
| "path": "language_model.model.layers.13.linear_attn.in_proj_a", |
| "layer": 13, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.002690445258387349 |
| }, |
| { |
| "path": "language_model.model.layers.13.linear_attn.out_proj", |
| "layer": 13, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.010155192323957927 |
| }, |
| { |
| "path": "language_model.model.layers.13.mlp.gate_proj", |
| "layer": 13, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.011515350872993905 |
| }, |
| { |
| "path": "language_model.model.layers.13.mlp.down_proj", |
| "layer": 13, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0041446334288496416 |
| }, |
| { |
| "path": "language_model.model.layers.13.mlp.up_proj", |
| "layer": 13, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.004337408969486667 |
| }, |
| { |
| "path": "language_model.model.layers.14.linear_attn.in_proj_qkv", |
| "layer": 14, |
| "layer_type": "linear_attention", |
| "nparams": 20971520, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.011339673428033051 |
| }, |
| { |
| "path": "language_model.model.layers.14.linear_attn.in_proj_z", |
| "layer": 14, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.007962872862178986 |
| }, |
| { |
| "path": "language_model.model.layers.14.linear_attn.in_proj_b", |
| "layer": 14, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0019362232417174479 |
| }, |
| { |
| "path": "language_model.model.layers.14.linear_attn.in_proj_a", |
| "layer": 14, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0015431009217327445 |
| }, |
| { |
| "path": "language_model.model.layers.14.linear_attn.out_proj", |
| "layer": 14, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.004887138324674762 |
| }, |
| { |
| "path": "language_model.model.layers.14.mlp.gate_proj", |
| "layer": 14, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.013886265417909858 |
| }, |
| { |
| "path": "language_model.model.layers.14.mlp.down_proj", |
| "layer": 14, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.01326110203221376 |
| }, |
| { |
| "path": "language_model.model.layers.14.mlp.up_proj", |
| "layer": 14, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.006025693011817328 |
| }, |
| { |
| "path": "language_model.model.layers.15.self_attn.q_proj", |
| "layer": 15, |
| "layer_type": "full_attention", |
| "nparams": 20971520, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.018192663011557066 |
| }, |
| { |
| "path": "language_model.model.layers.15.self_attn.k_proj", |
| "layer": 15, |
| "layer_type": "full_attention", |
| "nparams": 2621440, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.006153966935711085 |
| }, |
| { |
| "path": "language_model.model.layers.15.self_attn.v_proj", |
| "layer": 15, |
| "layer_type": "full_attention", |
| "nparams": 2621440, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.00728601800779855 |
| }, |
| { |
| "path": "language_model.model.layers.15.self_attn.o_proj", |
| "layer": 15, |
| "layer_type": "full_attention", |
| "nparams": 10485760, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.00843663950103548 |
| }, |
| { |
| "path": "language_model.model.layers.15.mlp.gate_proj", |
| "layer": 15, |
| "layer_type": "full_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.00844169358430183 |
| }, |
| { |
| "path": "language_model.model.layers.15.mlp.down_proj", |
| "layer": 15, |
| "layer_type": "full_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.008618414916908151 |
| }, |
| { |
| "path": "language_model.model.layers.15.mlp.up_proj", |
| "layer": 15, |
| "layer_type": "full_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.00904957608295576 |
| }, |
| { |
| "path": "language_model.model.layers.16.linear_attn.in_proj_qkv", |
| "layer": 16, |
| "layer_type": "linear_attention", |
| "nparams": 20971520, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.012239668651959228 |
| }, |
| { |
| "path": "language_model.model.layers.16.linear_attn.in_proj_z", |
| "layer": 16, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.007165175116112102 |
| }, |
| { |
| "path": "language_model.model.layers.16.linear_attn.in_proj_b", |
| "layer": 16, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.001689089058653197 |
| }, |
| { |
| "path": "language_model.model.layers.16.linear_attn.in_proj_a", |
| "layer": 16, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0021763847058412227 |
| }, |
| { |
| "path": "language_model.model.layers.16.linear_attn.out_proj", |
| "layer": 16, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.0050337960426177965 |
| }, |
| { |
| "path": "language_model.model.layers.16.mlp.gate_proj", |
| "layer": 16, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.004508509312858587 |
| }, |
| { |
| "path": "language_model.model.layers.16.mlp.down_proj", |
| "layer": 16, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.006751129047441731 |
| }, |
| { |
| "path": "language_model.model.layers.16.mlp.up_proj", |
| "layer": 16, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.008521210010591636 |
| }, |
| { |
| "path": "language_model.model.layers.17.linear_attn.in_proj_qkv", |
| "layer": 17, |
| "layer_type": "linear_attention", |
| "nparams": 20971520, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.006143374253497072 |
| }, |
| { |
| "path": "language_model.model.layers.17.linear_attn.in_proj_z", |
| "layer": 17, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.0033700770476747474 |
| }, |
| { |
| "path": "language_model.model.layers.17.linear_attn.in_proj_b", |
| "layer": 17, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0011950391956117611 |
| }, |
| { |
| "path": "language_model.model.layers.17.linear_attn.in_proj_a", |
| "layer": 17, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0013687938506446078 |
| }, |
| { |
| "path": "language_model.model.layers.17.linear_attn.out_proj", |
| "layer": 17, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.0034261684359799434 |
| }, |
| { |
| "path": "language_model.model.layers.17.mlp.gate_proj", |
| "layer": 17, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.005377442401188507 |
| }, |
| { |
| "path": "language_model.model.layers.17.mlp.down_proj", |
| "layer": 17, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.012502019640959393 |
| }, |
| { |
| "path": "language_model.model.layers.17.mlp.up_proj", |
| "layer": 17, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.005043160515902214 |
| }, |
| { |
| "path": "language_model.model.layers.18.linear_attn.in_proj_qkv", |
| "layer": 18, |
| "layer_type": "linear_attention", |
| "nparams": 20971520, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.006653324278347113 |
| }, |
| { |
| "path": "language_model.model.layers.18.linear_attn.in_proj_z", |
| "layer": 18, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.0034842532869403657 |
| }, |
| { |
| "path": "language_model.model.layers.18.linear_attn.in_proj_b", |
| "layer": 18, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0021424430907280296 |
| }, |
| { |
| "path": "language_model.model.layers.18.linear_attn.in_proj_a", |
| "layer": 18, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.002039650694065247 |
| }, |
| { |
| "path": "language_model.model.layers.18.linear_attn.out_proj", |
| "layer": 18, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.005711511275365844 |
| }, |
| { |
| "path": "language_model.model.layers.18.mlp.gate_proj", |
| "layer": 18, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.00516365691585215 |
| }, |
| { |
| "path": "language_model.model.layers.18.mlp.down_proj", |
| "layer": 18, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.005297452417896363 |
| }, |
| { |
| "path": "language_model.model.layers.18.mlp.up_proj", |
| "layer": 18, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.0057767763788979624 |
| }, |
| { |
| "path": "language_model.model.layers.19.self_attn.q_proj", |
| "layer": 19, |
| "layer_type": "full_attention", |
| "nparams": 20971520, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.015324583651697823 |
| }, |
| { |
| "path": "language_model.model.layers.19.self_attn.k_proj", |
| "layer": 19, |
| "layer_type": "full_attention", |
| "nparams": 2621440, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0020935271682160405 |
| }, |
| { |
| "path": "language_model.model.layers.19.self_attn.v_proj", |
| "layer": 19, |
| "layer_type": "full_attention", |
| "nparams": 2621440, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0024838890634897187 |
| }, |
| { |
| "path": "language_model.model.layers.19.self_attn.o_proj", |
| "layer": 19, |
| "layer_type": "full_attention", |
| "nparams": 10485760, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.006107687193440991 |
| }, |
| { |
| "path": "language_model.model.layers.19.mlp.gate_proj", |
| "layer": 19, |
| "layer_type": "full_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.006521167636485691 |
| }, |
| { |
| "path": "language_model.model.layers.19.mlp.down_proj", |
| "layer": 19, |
| "layer_type": "full_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.012130510119700717 |
| }, |
| { |
| "path": "language_model.model.layers.19.mlp.up_proj", |
| "layer": 19, |
| "layer_type": "full_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.011730753644203331 |
| }, |
| { |
| "path": "language_model.model.layers.20.linear_attn.in_proj_qkv", |
| "layer": 20, |
| "layer_type": "linear_attention", |
| "nparams": 20971520, |
| "bits": 2, |
| "kind": "ternary", |
| "kl": 0.004217642276790693 |
| }, |
| { |
| "path": "language_model.model.layers.20.linear_attn.in_proj_z", |
| "layer": 20, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.003584793409144641 |
| }, |
| { |
| "path": "language_model.model.layers.20.linear_attn.in_proj_b", |
| "layer": 20, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0016076045508908155 |
| }, |
| { |
| "path": "language_model.model.layers.20.linear_attn.in_proj_a", |
| "layer": 20, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0014620511291034764 |
| }, |
| { |
| "path": "language_model.model.layers.20.linear_attn.out_proj", |
| "layer": 20, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.003913781122171125 |
| }, |
| { |
| "path": "language_model.model.layers.20.mlp.gate_proj", |
| "layer": 20, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.005980544503803167 |
| }, |
| { |
| "path": "language_model.model.layers.20.mlp.down_proj", |
| "layer": 20, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.005546381825443038 |
| }, |
| { |
| "path": "language_model.model.layers.20.mlp.up_proj", |
| "layer": 20, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.005117319684254928 |
| }, |
| { |
| "path": "language_model.model.layers.21.linear_attn.in_proj_qkv", |
| "layer": 21, |
| "layer_type": "linear_attention", |
| "nparams": 20971520, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0038563888301930307 |
| }, |
| { |
| "path": "language_model.model.layers.21.linear_attn.in_proj_z", |
| "layer": 21, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.00392118822650533 |
| }, |
| { |
| "path": "language_model.model.layers.21.linear_attn.in_proj_b", |
| "layer": 21, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0011615152303928113 |
| }, |
| { |
| "path": "language_model.model.layers.21.linear_attn.in_proj_a", |
| "layer": 21, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.0013032740645528766 |
| }, |
| { |
| "path": "language_model.model.layers.21.linear_attn.out_proj", |
| "layer": 21, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0038319851560137557 |
| }, |
| { |
| "path": "language_model.model.layers.21.mlp.gate_proj", |
| "layer": 21, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0044488231220858945 |
| }, |
| { |
| "path": "language_model.model.layers.21.mlp.down_proj", |
| "layer": 21, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.010022162259151325 |
| }, |
| { |
| "path": "language_model.model.layers.21.mlp.up_proj", |
| "layer": 21, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.008296753091727066 |
| }, |
| { |
| "path": "language_model.model.layers.22.linear_attn.in_proj_qkv", |
| "layer": 22, |
| "layer_type": "linear_attention", |
| "nparams": 20971520, |
| "bits": 2, |
| "kind": "ternary", |
| "kl": 0.005124012435564135 |
| }, |
| { |
| "path": "language_model.model.layers.22.linear_attn.in_proj_z", |
| "layer": 22, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0016929112836727208 |
| }, |
| { |
| "path": "language_model.model.layers.22.linear_attn.in_proj_b", |
| "layer": 22, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.00120441033270699 |
| }, |
| { |
| "path": "language_model.model.layers.22.linear_attn.in_proj_a", |
| "layer": 22, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0018310609734295002 |
| }, |
| { |
| "path": "language_model.model.layers.22.linear_attn.out_proj", |
| "layer": 22, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0025930998573809503 |
| }, |
| { |
| "path": "language_model.model.layers.22.mlp.gate_proj", |
| "layer": 22, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.007172711070889574 |
| }, |
| { |
| "path": "language_model.model.layers.22.mlp.down_proj", |
| "layer": 22, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.006215302619931402 |
| }, |
| { |
| "path": "language_model.model.layers.22.mlp.up_proj", |
| "layer": 22, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.004271566133998565 |
| }, |
| { |
| "path": "language_model.model.layers.23.self_attn.q_proj", |
| "layer": 23, |
| "layer_type": "full_attention", |
| "nparams": 20971520, |
| "bits": 2, |
| "kind": "ternary", |
| "kl": 0.00861396856288937 |
| }, |
| { |
| "path": "language_model.model.layers.23.self_attn.k_proj", |
| "layer": 23, |
| "layer_type": "full_attention", |
| "nparams": 2621440, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0019846955283420566 |
| }, |
| { |
| "path": "language_model.model.layers.23.self_attn.v_proj", |
| "layer": 23, |
| "layer_type": "full_attention", |
| "nparams": 2621440, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0020784904667924616 |
| }, |
| { |
| "path": "language_model.model.layers.23.self_attn.o_proj", |
| "layer": 23, |
| "layer_type": "full_attention", |
| "nparams": 10485760, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.005887735015824158 |
| }, |
| { |
| "path": "language_model.model.layers.23.mlp.gate_proj", |
| "layer": 23, |
| "layer_type": "full_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0027974898614304316 |
| }, |
| { |
| "path": "language_model.model.layers.23.mlp.down_proj", |
| "layer": 23, |
| "layer_type": "full_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.005008540500233844 |
| }, |
| { |
| "path": "language_model.model.layers.23.mlp.up_proj", |
| "layer": 23, |
| "layer_type": "full_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.004288408973020578 |
| }, |
| { |
| "path": "language_model.model.layers.24.linear_attn.in_proj_qkv", |
| "layer": 24, |
| "layer_type": "linear_attention", |
| "nparams": 20971520, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0035279809933627783 |
| }, |
| { |
| "path": "language_model.model.layers.24.linear_attn.in_proj_z", |
| "layer": 24, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0028186044364557697 |
| }, |
| { |
| "path": "language_model.model.layers.24.linear_attn.in_proj_b", |
| "layer": 24, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.000861179402180736 |
| }, |
| { |
| "path": "language_model.model.layers.24.linear_attn.in_proj_a", |
| "layer": 24, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 2, |
| "kind": "ternary", |
| "kl": 0.0014708640460124364 |
| }, |
| { |
| "path": "language_model.model.layers.24.linear_attn.out_proj", |
| "layer": 24, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.002918557645676256 |
| }, |
| { |
| "path": "language_model.model.layers.24.mlp.gate_proj", |
| "layer": 24, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.003586550376589139 |
| }, |
| { |
| "path": "language_model.model.layers.24.mlp.down_proj", |
| "layer": 24, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0033804561018519865 |
| }, |
| { |
| "path": "language_model.model.layers.24.mlp.up_proj", |
| "layer": 24, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.005295595565605282 |
| }, |
| { |
| "path": "language_model.model.layers.25.linear_attn.in_proj_qkv", |
| "layer": 25, |
| "layer_type": "linear_attention", |
| "nparams": 20971520, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0053310425548574476 |
| }, |
| { |
| "path": "language_model.model.layers.25.linear_attn.in_proj_z", |
| "layer": 25, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0027591396448848737 |
| }, |
| { |
| "path": "language_model.model.layers.25.linear_attn.in_proj_b", |
| "layer": 25, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0013798386948937805 |
| }, |
| { |
| "path": "language_model.model.layers.25.linear_attn.in_proj_a", |
| "layer": 25, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.001222093718792863 |
| }, |
| { |
| "path": "language_model.model.layers.25.linear_attn.out_proj", |
| "layer": 25, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.003876863353402041 |
| }, |
| { |
| "path": "language_model.model.layers.25.mlp.gate_proj", |
| "layer": 25, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.004169148010042012 |
| }, |
| { |
| "path": "language_model.model.layers.25.mlp.down_proj", |
| "layer": 25, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.005103545410745988 |
| }, |
| { |
| "path": "language_model.model.layers.25.mlp.up_proj", |
| "layer": 25, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.002309934380767234 |
| }, |
| { |
| "path": "language_model.model.layers.26.linear_attn.in_proj_qkv", |
| "layer": 26, |
| "layer_type": "linear_attention", |
| "nparams": 20971520, |
| "bits": 2, |
| "kind": "ternary", |
| "kl": 0.003328468159294269 |
| }, |
| { |
| "path": "language_model.model.layers.26.linear_attn.in_proj_z", |
| "layer": 26, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0025809277220773287 |
| }, |
| { |
| "path": "language_model.model.layers.26.linear_attn.in_proj_b", |
| "layer": 26, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.001228033403678685 |
| }, |
| { |
| "path": "language_model.model.layers.26.linear_attn.in_proj_a", |
| "layer": 26, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 2, |
| "kind": "ternary", |
| "kl": 0.0008598071553789065 |
| }, |
| { |
| "path": "language_model.model.layers.26.linear_attn.out_proj", |
| "layer": 26, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0036962745992717514 |
| }, |
| { |
| "path": "language_model.model.layers.26.mlp.gate_proj", |
| "layer": 26, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.004956540361069822 |
| }, |
| { |
| "path": "language_model.model.layers.26.mlp.down_proj", |
| "layer": 26, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.0034921885774195933 |
| }, |
| { |
| "path": "language_model.model.layers.26.mlp.up_proj", |
| "layer": 26, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "ternary", |
| "kl": 0.0044375837627941605 |
| }, |
| { |
| "path": "language_model.model.layers.27.self_attn.q_proj", |
| "layer": 27, |
| "layer_type": "full_attention", |
| "nparams": 20971520, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.007407018614069779 |
| }, |
| { |
| "path": "language_model.model.layers.27.self_attn.k_proj", |
| "layer": 27, |
| "layer_type": "full_attention", |
| "nparams": 2621440, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.002578442176801118 |
| }, |
| { |
| "path": "language_model.model.layers.27.self_attn.v_proj", |
| "layer": 27, |
| "layer_type": "full_attention", |
| "nparams": 2621440, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0015826920125229685 |
| }, |
| { |
| "path": "language_model.model.layers.27.self_attn.o_proj", |
| "layer": 27, |
| "layer_type": "full_attention", |
| "nparams": 10485760, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.0063210920071340064 |
| }, |
| { |
| "path": "language_model.model.layers.27.mlp.gate_proj", |
| "layer": 27, |
| "layer_type": "full_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0024200480880117894 |
| }, |
| { |
| "path": "language_model.model.layers.27.mlp.down_proj", |
| "layer": 27, |
| "layer_type": "full_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0038518660025201163 |
| }, |
| { |
| "path": "language_model.model.layers.27.mlp.up_proj", |
| "layer": 27, |
| "layer_type": "full_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.003806384045919116 |
| }, |
| { |
| "path": "language_model.model.layers.28.linear_attn.in_proj_qkv", |
| "layer": 28, |
| "layer_type": "linear_attention", |
| "nparams": 20971520, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.002749126565589559 |
| }, |
| { |
| "path": "language_model.model.layers.28.linear_attn.in_proj_z", |
| "layer": 28, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.00198772293110002 |
| }, |
| { |
| "path": "language_model.model.layers.28.linear_attn.in_proj_b", |
| "layer": 28, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.0004472118012195674 |
| }, |
| { |
| "path": "language_model.model.layers.28.linear_attn.in_proj_a", |
| "layer": 28, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.0006654004124117305 |
| }, |
| { |
| "path": "language_model.model.layers.28.linear_attn.out_proj", |
| "layer": 28, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.003904530959754009 |
| }, |
| { |
| "path": "language_model.model.layers.28.mlp.gate_proj", |
| "layer": 28, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0024391735330841595 |
| }, |
| { |
| "path": "language_model.model.layers.28.mlp.down_proj", |
| "layer": 28, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0035189845281537145 |
| }, |
| { |
| "path": "language_model.model.layers.28.mlp.up_proj", |
| "layer": 28, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0030495466258141754 |
| }, |
| { |
| "path": "language_model.model.layers.29.linear_attn.in_proj_qkv", |
| "layer": 29, |
| "layer_type": "linear_attention", |
| "nparams": 20971520, |
| "bits": 2, |
| "kind": "ternary", |
| "kl": 0.002629985415225408 |
| }, |
| { |
| "path": "language_model.model.layers.29.linear_attn.in_proj_z", |
| "layer": 29, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0025190827935064374 |
| }, |
| { |
| "path": "language_model.model.layers.29.linear_attn.in_proj_b", |
| "layer": 29, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.0007231384101107027 |
| }, |
| { |
| "path": "language_model.model.layers.29.linear_attn.in_proj_a", |
| "layer": 29, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0005520234299671102 |
| }, |
| { |
| "path": "language_model.model.layers.29.linear_attn.out_proj", |
| "layer": 29, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 2, |
| "kind": "ternary", |
| "kl": 0.0022329573897954718 |
| }, |
| { |
| "path": "language_model.model.layers.29.mlp.gate_proj", |
| "layer": 29, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.003990216169521104 |
| }, |
| { |
| "path": "language_model.model.layers.29.mlp.down_proj", |
| "layer": 29, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.004243390057269966 |
| }, |
| { |
| "path": "language_model.model.layers.29.mlp.up_proj", |
| "layer": 29, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.008227331830174473 |
| }, |
| { |
| "path": "language_model.model.layers.30.linear_attn.in_proj_qkv", |
| "layer": 30, |
| "layer_type": "linear_attention", |
| "nparams": 20971520, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.0052265165882378235 |
| }, |
| { |
| "path": "language_model.model.layers.30.linear_attn.in_proj_z", |
| "layer": 30, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.003792861104396105 |
| }, |
| { |
| "path": "language_model.model.layers.30.linear_attn.in_proj_b", |
| "layer": 30, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0007776507164580607 |
| }, |
| { |
| "path": "language_model.model.layers.30.linear_attn.in_proj_a", |
| "layer": 30, |
| "layer_type": "linear_attention", |
| "nparams": 81920, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0008808346500481211 |
| }, |
| { |
| "path": "language_model.model.layers.30.linear_attn.out_proj", |
| "layer": 30, |
| "layer_type": "linear_attention", |
| "nparams": 10485760, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.0030045996317789635 |
| }, |
| { |
| "path": "language_model.model.layers.30.mlp.gate_proj", |
| "layer": 30, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "2", |
| "kl": 0.004581500490373279 |
| }, |
| { |
| "path": "language_model.model.layers.30.mlp.down_proj", |
| "layer": 30, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "ternary", |
| "kl": 0.0075853140942216245 |
| }, |
| { |
| "path": "language_model.model.layers.30.mlp.up_proj", |
| "layer": 30, |
| "layer_type": "linear_attention", |
| "nparams": 23592960, |
| "bits": 2, |
| "kind": "ternary", |
| "kl": 0.009086271054308806 |
| }, |
| { |
| "path": "language_model.model.layers.31.self_attn.q_proj", |
| "layer": 31, |
| "layer_type": "full_attention", |
| "nparams": 20971520, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.004721018605704054 |
| }, |
| { |
| "path": "language_model.model.layers.31.self_attn.k_proj", |
| "layer": 31, |
| "layer_type": "full_attention", |
| "nparams": 2621440, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.001393019356334057 |
| }, |
| { |
| "path": "language_model.model.layers.31.self_attn.v_proj", |
| "layer": 31, |
| "layer_type": "full_attention", |
| "nparams": 2621440, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0009648835963438874 |
| }, |
| { |
| "path": "language_model.model.layers.31.self_attn.o_proj", |
| "layer": 31, |
| "layer_type": "full_attention", |
| "nparams": 10485760, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.004874454498413899 |
| }, |
| { |
| "path": "language_model.model.layers.31.mlp.gate_proj", |
| "layer": 31, |
| "layer_type": "full_attention", |
| "nparams": 23592960, |
| "bits": 3, |
| "kind": "3", |
| "kl": 0.008578638269353754 |
| }, |
| { |
| "path": "language_model.model.layers.31.mlp.down_proj", |
| "layer": 31, |
| "layer_type": "full_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.0059679943044754795 |
| }, |
| { |
| "path": "language_model.model.layers.31.mlp.up_proj", |
| "layer": 31, |
| "layer_type": "full_attention", |
| "nparams": 23592960, |
| "bits": 4, |
| "kind": "4", |
| "kl": 0.00986051337512158 |
| } |
| ] |
| } |
|
|
