{ "source_dir": "/Users/dustinnielson/DEV_INT/lance-mlx-research/lance-mlx-models/Lance-3B-bf16", "stats_dir": "/Users/dustinnielson/DEV_INT/lance-mlx-research/lance-mlx/notes/phase5n_diagnostics/phase5c3_awq_port/act_stats", "bits": 4, "group_size": 128, "mode": "affine+awq", "n_grid": 20, "bf16_bytes": 12371046496, "quantized_bytes": 3306811488, "compression_ratio": 0.2673024864201432, "fusion_groups_applied": 144, "fusion_groups_skipped": 0, "skip_patterns_always": [ "time_embedder.proj_in", "time_embedder.proj_out", "llm2vae" ], "awq_per_group": { "layers.0.input_layernorm": { "best_alpha": 0.35, "best_err": 0.00458593072835356, "consumers": [ "layers.0.self_attn.q_proj", "layers.0.self_attn.k_proj", "layers.0.self_attn.v_proj" ] }, "layers.0.input_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.0021403880091384053, "consumers": [ "layers.0.self_attn.q_proj_moe_gen", "layers.0.self_attn.k_proj_moe_gen", "layers.0.self_attn.v_proj_moe_gen" ] }, "layers.0.post_attention_layernorm": { "best_alpha": 0.35, "best_err": 0.007643559714779258, "consumers": [ "layers.0.mlp.gate_proj", "layers.0.mlp.up_proj" ] }, "layers.0.post_attention_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.003576420247554779, "consumers": [ "layers.0.mlp_moe_gen.gate_proj", "layers.0.mlp_moe_gen.up_proj" ] }, "layers.1.input_layernorm": { "best_alpha": 0.3, "best_err": 0.07270884048193693, "consumers": [ "layers.1.self_attn.q_proj", "layers.1.self_attn.k_proj", "layers.1.self_attn.v_proj" ] }, "layers.1.input_layernorm_moe_gen": { "best_alpha": 0.3, "best_err": 0.05615841131657362, "consumers": [ "layers.1.self_attn.q_proj_moe_gen", "layers.1.self_attn.k_proj_moe_gen", "layers.1.self_attn.v_proj_moe_gen" ] }, "layers.1.post_attention_layernorm": { "best_alpha": 0.35, "best_err": 0.01055069756694138, "consumers": [ "layers.1.mlp.gate_proj", "layers.1.mlp.up_proj" ] }, "layers.1.post_attention_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.0033836811780929565, "consumers": [ "layers.1.mlp_moe_gen.gate_proj", "layers.1.mlp_moe_gen.up_proj" ] }, "layers.2.input_layernorm": { "best_alpha": 0.3, "best_err": 0.03681113151833415, "consumers": [ "layers.2.self_attn.q_proj", "layers.2.self_attn.k_proj", "layers.2.self_attn.v_proj" ] }, "layers.2.input_layernorm_moe_gen": { "best_alpha": 0.3, "best_err": 0.029231476597487926, "consumers": [ "layers.2.self_attn.q_proj_moe_gen", "layers.2.self_attn.k_proj_moe_gen", "layers.2.self_attn.v_proj_moe_gen" ] }, "layers.2.post_attention_layernorm": { "best_alpha": 0.35, "best_err": 0.009307235479354858, "consumers": [ "layers.2.mlp.gate_proj", "layers.2.mlp.up_proj" ] }, "layers.2.post_attention_layernorm_moe_gen": { "best_alpha": 0.3, "best_err": 0.0031711964402347803, "consumers": [ "layers.2.mlp_moe_gen.gate_proj", "layers.2.mlp_moe_gen.up_proj" ] }, "layers.3.input_layernorm": { "best_alpha": 0.3, "best_err": 0.022754479199647903, "consumers": [ "layers.3.self_attn.q_proj", "layers.3.self_attn.k_proj", "layers.3.self_attn.v_proj" ] }, "layers.3.input_layernorm_moe_gen": { "best_alpha": 0.3, "best_err": 0.020830828696489334, "consumers": [ "layers.3.self_attn.q_proj_moe_gen", "layers.3.self_attn.k_proj_moe_gen", "layers.3.self_attn.v_proj_moe_gen" ] }, "layers.3.post_attention_layernorm": { "best_alpha": 0.5, "best_err": 0.016386197414249182, "consumers": [ "layers.3.mlp.gate_proj", "layers.3.mlp.up_proj" ] }, "layers.3.post_attention_layernorm_moe_gen": { "best_alpha": 0.5, "best_err": 0.005372598301619291, "consumers": [ "layers.3.mlp_moe_gen.gate_proj", "layers.3.mlp_moe_gen.up_proj" ] }, "layers.4.input_layernorm": { "best_alpha": 0.3, "best_err": 0.01877523958683014, "consumers": [ "layers.4.self_attn.q_proj", "layers.4.self_attn.k_proj", "layers.4.self_attn.v_proj" ] }, "layers.4.input_layernorm_moe_gen": { "best_alpha": 0.35, "best_err": 0.015583840198814869, "consumers": [ "layers.4.self_attn.q_proj_moe_gen", "layers.4.self_attn.k_proj_moe_gen", "layers.4.self_attn.v_proj_moe_gen" ] }, "layers.4.post_attention_layernorm": { "best_alpha": 0.45, "best_err": 0.009718624409288168, "consumers": [ "layers.4.mlp.gate_proj", "layers.4.mlp.up_proj" ] }, "layers.4.post_attention_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.004090428119525313, "consumers": [ "layers.4.mlp_moe_gen.gate_proj", "layers.4.mlp_moe_gen.up_proj" ] }, "layers.5.input_layernorm": { "best_alpha": 0.3, "best_err": 0.011677489150315523, "consumers": [ "layers.5.self_attn.q_proj", "layers.5.self_attn.k_proj", "layers.5.self_attn.v_proj" ] }, "layers.5.input_layernorm_moe_gen": { "best_alpha": 0.35, "best_err": 0.011435149004682899, "consumers": [ "layers.5.self_attn.q_proj_moe_gen", "layers.5.self_attn.k_proj_moe_gen", "layers.5.self_attn.v_proj_moe_gen" ] }, "layers.5.post_attention_layernorm": { "best_alpha": 0.35, "best_err": 0.004989819368347526, "consumers": [ "layers.5.mlp.gate_proj", "layers.5.mlp.up_proj" ] }, "layers.5.post_attention_layernorm_moe_gen": { "best_alpha": 0.35, "best_err": 0.0024960024747997522, "consumers": [ "layers.5.mlp_moe_gen.gate_proj", "layers.5.mlp_moe_gen.up_proj" ] }, "layers.6.input_layernorm": { "best_alpha": 0.3, "best_err": 0.06253223866224289, "consumers": [ "layers.6.self_attn.q_proj", "layers.6.self_attn.k_proj", "layers.6.self_attn.v_proj" ] }, "layers.6.input_layernorm_moe_gen": { "best_alpha": 0.3, "best_err": 0.03238382376730442, "consumers": [ "layers.6.self_attn.q_proj_moe_gen", "layers.6.self_attn.k_proj_moe_gen", "layers.6.self_attn.v_proj_moe_gen" ] }, "layers.6.post_attention_layernorm": { "best_alpha": 0.35, "best_err": 0.005734460661187768, "consumers": [ "layers.6.mlp.gate_proj", "layers.6.mlp.up_proj" ] }, "layers.6.post_attention_layernorm_moe_gen": { "best_alpha": 0.35, "best_err": 0.0033549898071214557, "consumers": [ "layers.6.mlp_moe_gen.gate_proj", "layers.6.mlp_moe_gen.up_proj" ] }, "layers.7.input_layernorm": { "best_alpha": 0.3, "best_err": 0.022551667410880327, "consumers": [ "layers.7.self_attn.q_proj", "layers.7.self_attn.k_proj", "layers.7.self_attn.v_proj" ] }, "layers.7.input_layernorm_moe_gen": { "best_alpha": 0.45, "best_err": 0.01729161897674203, "consumers": [ "layers.7.self_attn.q_proj_moe_gen", "layers.7.self_attn.k_proj_moe_gen", "layers.7.self_attn.v_proj_moe_gen" ] }, "layers.7.post_attention_layernorm": { "best_alpha": 0.4, "best_err": 0.007926714839413762, "consumers": [ "layers.7.mlp.gate_proj", "layers.7.mlp.up_proj" ] }, "layers.7.post_attention_layernorm_moe_gen": { "best_alpha": 0.35, "best_err": 0.004999170079827309, "consumers": [ "layers.7.mlp_moe_gen.gate_proj", "layers.7.mlp_moe_gen.up_proj" ] }, "layers.8.input_layernorm": { "best_alpha": 0.3, "best_err": 0.016717045567929745, "consumers": [ "layers.8.self_attn.q_proj", "layers.8.self_attn.k_proj", "layers.8.self_attn.v_proj" ] }, "layers.8.input_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.015823927707970142, "consumers": [ "layers.8.self_attn.q_proj_moe_gen", "layers.8.self_attn.k_proj_moe_gen", "layers.8.self_attn.v_proj_moe_gen" ] }, "layers.8.post_attention_layernorm": { "best_alpha": 0.35, "best_err": 0.006246325559914112, "consumers": [ "layers.8.mlp.gate_proj", "layers.8.mlp.up_proj" ] }, "layers.8.post_attention_layernorm_moe_gen": { "best_alpha": 0.3, "best_err": 0.004818351706489921, "consumers": [ "layers.8.mlp_moe_gen.gate_proj", "layers.8.mlp_moe_gen.up_proj" ] }, "layers.9.input_layernorm": { "best_alpha": 0.3, "best_err": 0.025030743330717087, "consumers": [ "layers.9.self_attn.q_proj", "layers.9.self_attn.k_proj", "layers.9.self_attn.v_proj" ] }, "layers.9.input_layernorm_moe_gen": { "best_alpha": 0.35, "best_err": 0.019082439597696066, "consumers": [ "layers.9.self_attn.q_proj_moe_gen", "layers.9.self_attn.k_proj_moe_gen", "layers.9.self_attn.v_proj_moe_gen" ] }, "layers.9.post_attention_layernorm": { "best_alpha": 0.35, "best_err": 0.004797130124643445, "consumers": [ "layers.9.mlp.gate_proj", "layers.9.mlp.up_proj" ] }, "layers.9.post_attention_layernorm_moe_gen": { "best_alpha": 0.3, "best_err": 0.00391571584623307, "consumers": [ "layers.9.mlp_moe_gen.gate_proj", "layers.9.mlp_moe_gen.up_proj" ] }, "layers.10.input_layernorm": { "best_alpha": 0.3, "best_err": 0.007787489332258701, "consumers": [ "layers.10.self_attn.q_proj", "layers.10.self_attn.k_proj", "layers.10.self_attn.v_proj" ] }, "layers.10.input_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.008221070980653167, "consumers": [ "layers.10.self_attn.q_proj_moe_gen", "layers.10.self_attn.k_proj_moe_gen", "layers.10.self_attn.v_proj_moe_gen" ] }, "layers.10.post_attention_layernorm": { "best_alpha": 0.3, "best_err": 0.003546421998180449, "consumers": [ "layers.10.mlp.gate_proj", "layers.10.mlp.up_proj" ] }, "layers.10.post_attention_layernorm_moe_gen": { "best_alpha": 0.3, "best_err": 0.0033734560711309314, "consumers": [ "layers.10.mlp_moe_gen.gate_proj", "layers.10.mlp_moe_gen.up_proj" ] }, "layers.11.input_layernorm": { "best_alpha": 0.3, "best_err": 0.006951858755201101, "consumers": [ "layers.11.self_attn.q_proj", "layers.11.self_attn.k_proj", "layers.11.self_attn.v_proj" ] }, "layers.11.input_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.007689060410484672, "consumers": [ "layers.11.self_attn.q_proj_moe_gen", "layers.11.self_attn.k_proj_moe_gen", "layers.11.self_attn.v_proj_moe_gen" ] }, "layers.11.post_attention_layernorm": { "best_alpha": 0.3, "best_err": 0.003067293087951839, "consumers": [ "layers.11.mlp.gate_proj", "layers.11.mlp.up_proj" ] }, "layers.11.post_attention_layernorm_moe_gen": { "best_alpha": 0.3, "best_err": 0.003110864432528615, "consumers": [ "layers.11.mlp_moe_gen.gate_proj", "layers.11.mlp_moe_gen.up_proj" ] }, "layers.12.input_layernorm": { "best_alpha": 0.3, "best_err": 0.02311427053064108, "consumers": [ "layers.12.self_attn.q_proj", "layers.12.self_attn.k_proj", "layers.12.self_attn.v_proj" ] }, "layers.12.input_layernorm_moe_gen": { "best_alpha": 0.35, "best_err": 0.02629701793193817, "consumers": [ "layers.12.self_attn.q_proj_moe_gen", "layers.12.self_attn.k_proj_moe_gen", "layers.12.self_attn.v_proj_moe_gen" ] }, "layers.12.post_attention_layernorm": { "best_alpha": 0.35, "best_err": 0.0031268250895664096, "consumers": [ "layers.12.mlp.gate_proj", "layers.12.mlp.up_proj" ] }, "layers.12.post_attention_layernorm_moe_gen": { "best_alpha": 0.3, "best_err": 0.0030514480313286185, "consumers": [ "layers.12.mlp_moe_gen.gate_proj", "layers.12.mlp_moe_gen.up_proj" ] }, "layers.13.input_layernorm": { "best_alpha": 0.35, "best_err": 0.007222589338198304, "consumers": [ "layers.13.self_attn.q_proj", "layers.13.self_attn.k_proj", "layers.13.self_attn.v_proj" ] }, "layers.13.input_layernorm_moe_gen": { "best_alpha": 0.35, "best_err": 0.0087540487293154, "consumers": [ "layers.13.self_attn.q_proj_moe_gen", "layers.13.self_attn.k_proj_moe_gen", "layers.13.self_attn.v_proj_moe_gen" ] }, "layers.13.post_attention_layernorm": { "best_alpha": 0.35, "best_err": 0.003296053037047386, "consumers": [ "layers.13.mlp.gate_proj", "layers.13.mlp.up_proj" ] }, "layers.13.post_attention_layernorm_moe_gen": { "best_alpha": 0.3, "best_err": 0.003499676357023418, "consumers": [ "layers.13.mlp_moe_gen.gate_proj", "layers.13.mlp_moe_gen.up_proj" ] }, "layers.14.input_layernorm": { "best_alpha": 0.3, "best_err": 0.006669960916042328, "consumers": [ "layers.14.self_attn.q_proj", "layers.14.self_attn.k_proj", "layers.14.self_attn.v_proj" ] }, "layers.14.input_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.008936543483287096, "consumers": [ "layers.14.self_attn.q_proj_moe_gen", "layers.14.self_attn.k_proj_moe_gen", "layers.14.self_attn.v_proj_moe_gen" ] }, "layers.14.post_attention_layernorm": { "best_alpha": 0.4, "best_err": 0.0037863863399252295, "consumers": [ "layers.14.mlp.gate_proj", "layers.14.mlp.up_proj" ] }, "layers.14.post_attention_layernorm_moe_gen": { "best_alpha": 0.35, "best_err": 0.004165650112554431, "consumers": [ "layers.14.mlp_moe_gen.gate_proj", "layers.14.mlp_moe_gen.up_proj" ] }, "layers.15.input_layernorm": { "best_alpha": 0.3, "best_err": 0.00765526108443737, "consumers": [ "layers.15.self_attn.q_proj", "layers.15.self_attn.k_proj", "layers.15.self_attn.v_proj" ] }, "layers.15.input_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.010691209929063916, "consumers": [ "layers.15.self_attn.q_proj_moe_gen", "layers.15.self_attn.k_proj_moe_gen", "layers.15.self_attn.v_proj_moe_gen" ] }, "layers.15.post_attention_layernorm": { "best_alpha": 0.4, "best_err": 0.003892258508130908, "consumers": [ "layers.15.mlp.gate_proj", "layers.15.mlp.up_proj" ] }, "layers.15.post_attention_layernorm_moe_gen": { "best_alpha": 0.35, "best_err": 0.004360296297818422, "consumers": [ "layers.15.mlp_moe_gen.gate_proj", "layers.15.mlp_moe_gen.up_proj" ] }, "layers.16.input_layernorm": { "best_alpha": 0.3, "best_err": 0.009200290311127901, "consumers": [ "layers.16.self_attn.q_proj", "layers.16.self_attn.k_proj", "layers.16.self_attn.v_proj" ] }, "layers.16.input_layernorm_moe_gen": { "best_alpha": 0.45, "best_err": 0.013372566550970078, "consumers": [ "layers.16.self_attn.q_proj_moe_gen", "layers.16.self_attn.k_proj_moe_gen", "layers.16.self_attn.v_proj_moe_gen" ] }, "layers.16.post_attention_layernorm": { "best_alpha": 0.4, "best_err": 0.003502526553347707, "consumers": [ "layers.16.mlp.gate_proj", "layers.16.mlp.up_proj" ] }, "layers.16.post_attention_layernorm_moe_gen": { "best_alpha": 0.35, "best_err": 0.003909384133294225, "consumers": [ "layers.16.mlp_moe_gen.gate_proj", "layers.16.mlp_moe_gen.up_proj" ] }, "layers.17.input_layernorm": { "best_alpha": 0.3, "best_err": 0.01001800550147891, "consumers": [ "layers.17.self_attn.q_proj", "layers.17.self_attn.k_proj", "layers.17.self_attn.v_proj" ] }, "layers.17.input_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.014895168598741293, "consumers": [ "layers.17.self_attn.q_proj_moe_gen", "layers.17.self_attn.k_proj_moe_gen", "layers.17.self_attn.v_proj_moe_gen" ] }, "layers.17.post_attention_layernorm": { "best_alpha": 0.4, "best_err": 0.003303446574136615, "consumers": [ "layers.17.mlp.gate_proj", "layers.17.mlp.up_proj" ] }, "layers.17.post_attention_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.0037332052597776055, "consumers": [ "layers.17.mlp_moe_gen.gate_proj", "layers.17.mlp_moe_gen.up_proj" ] }, "layers.18.input_layernorm": { "best_alpha": 0.3, "best_err": 0.010829327395185828, "consumers": [ "layers.18.self_attn.q_proj", "layers.18.self_attn.k_proj", "layers.18.self_attn.v_proj" ] }, "layers.18.input_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.015611939132213593, "consumers": [ "layers.18.self_attn.q_proj_moe_gen", "layers.18.self_attn.k_proj_moe_gen", "layers.18.self_attn.v_proj_moe_gen" ] }, "layers.18.post_attention_layernorm": { "best_alpha": 0.45, "best_err": 0.003714948776178062, "consumers": [ "layers.18.mlp.gate_proj", "layers.18.mlp.up_proj" ] }, "layers.18.post_attention_layernorm_moe_gen": { "best_alpha": 0.35, "best_err": 0.004194366978481412, "consumers": [ "layers.18.mlp_moe_gen.gate_proj", "layers.18.mlp_moe_gen.up_proj" ] }, "layers.19.input_layernorm": { "best_alpha": 0.35, "best_err": 0.01116279885172844, "consumers": [ "layers.19.self_attn.q_proj", "layers.19.self_attn.k_proj", "layers.19.self_attn.v_proj" ] }, "layers.19.input_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.018096713349223137, "consumers": [ "layers.19.self_attn.q_proj_moe_gen", "layers.19.self_attn.k_proj_moe_gen", "layers.19.self_attn.v_proj_moe_gen" ] }, "layers.19.post_attention_layernorm": { "best_alpha": 0.4, "best_err": 0.0038424793165177107, "consumers": [ "layers.19.mlp.gate_proj", "layers.19.mlp.up_proj" ] }, "layers.19.post_attention_layernorm_moe_gen": { "best_alpha": 0.45, "best_err": 0.004652554169297218, "consumers": [ "layers.19.mlp_moe_gen.gate_proj", "layers.19.mlp_moe_gen.up_proj" ] }, "layers.20.input_layernorm": { "best_alpha": 0.3, "best_err": 0.01315189665183425, "consumers": [ "layers.20.self_attn.q_proj", "layers.20.self_attn.k_proj", "layers.20.self_attn.v_proj" ] }, "layers.20.input_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.02170101460069418, "consumers": [ "layers.20.self_attn.q_proj_moe_gen", "layers.20.self_attn.k_proj_moe_gen", "layers.20.self_attn.v_proj_moe_gen" ] }, "layers.20.post_attention_layernorm": { "best_alpha": 0.45, "best_err": 0.0043482230976223946, "consumers": [ "layers.20.mlp.gate_proj", "layers.20.mlp.up_proj" ] }, "layers.20.post_attention_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.005148760508745909, "consumers": [ "layers.20.mlp_moe_gen.gate_proj", "layers.20.mlp_moe_gen.up_proj" ] }, "layers.21.input_layernorm": { "best_alpha": 0.3, "best_err": 0.014262943528592587, "consumers": [ "layers.21.self_attn.q_proj", "layers.21.self_attn.k_proj", "layers.21.self_attn.v_proj" ] }, "layers.21.input_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.026137451641261578, "consumers": [ "layers.21.self_attn.q_proj_moe_gen", "layers.21.self_attn.k_proj_moe_gen", "layers.21.self_attn.v_proj_moe_gen" ] }, "layers.21.post_attention_layernorm": { "best_alpha": 0.45, "best_err": 0.004324082052335143, "consumers": [ "layers.21.mlp.gate_proj", "layers.21.mlp.up_proj" ] }, "layers.21.post_attention_layernorm_moe_gen": { "best_alpha": 0.45, "best_err": 0.005147382616996765, "consumers": [ "layers.21.mlp_moe_gen.gate_proj", "layers.21.mlp_moe_gen.up_proj" ] }, "layers.22.input_layernorm": { "best_alpha": 0.35, "best_err": 0.017558230087161064, "consumers": [ "layers.22.self_attn.q_proj", "layers.22.self_attn.k_proj", "layers.22.self_attn.v_proj" ] }, "layers.22.input_layernorm_moe_gen": { "best_alpha": 0.45, "best_err": 0.02854403667151928, "consumers": [ "layers.22.self_attn.q_proj_moe_gen", "layers.22.self_attn.k_proj_moe_gen", "layers.22.self_attn.v_proj_moe_gen" ] }, "layers.22.post_attention_layernorm": { "best_alpha": 0.4, "best_err": 0.00529686501249671, "consumers": [ "layers.22.mlp.gate_proj", "layers.22.mlp.up_proj" ] }, "layers.22.post_attention_layernorm_moe_gen": { "best_alpha": 0.35, "best_err": 0.006503301672637463, "consumers": [ "layers.22.mlp_moe_gen.gate_proj", "layers.22.mlp_moe_gen.up_proj" ] }, "layers.23.input_layernorm": { "best_alpha": 0.35, "best_err": 0.017843658570200205, "consumers": [ "layers.23.self_attn.q_proj", "layers.23.self_attn.k_proj", "layers.23.self_attn.v_proj" ] }, "layers.23.input_layernorm_moe_gen": { "best_alpha": 0.45, "best_err": 0.028661489952355623, "consumers": [ "layers.23.self_attn.q_proj_moe_gen", "layers.23.self_attn.k_proj_moe_gen", "layers.23.self_attn.v_proj_moe_gen" ] }, "layers.23.post_attention_layernorm": { "best_alpha": 0.45, "best_err": 0.006145666586235166, "consumers": [ "layers.23.mlp.gate_proj", "layers.23.mlp.up_proj" ] }, "layers.23.post_attention_layernorm_moe_gen": { "best_alpha": 0.35, "best_err": 0.007576683536171913, "consumers": [ "layers.23.mlp_moe_gen.gate_proj", "layers.23.mlp_moe_gen.up_proj" ] }, "layers.24.input_layernorm": { "best_alpha": 0.35, "best_err": 0.020195242948830128, "consumers": [ "layers.24.self_attn.q_proj", "layers.24.self_attn.k_proj", "layers.24.self_attn.v_proj" ] }, "layers.24.input_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.039047873578965664, "consumers": [ "layers.24.self_attn.q_proj_moe_gen", "layers.24.self_attn.k_proj_moe_gen", "layers.24.self_attn.v_proj_moe_gen" ] }, "layers.24.post_attention_layernorm": { "best_alpha": 0.4, "best_err": 0.006804545875638723, "consumers": [ "layers.24.mlp.gate_proj", "layers.24.mlp.up_proj" ] }, "layers.24.post_attention_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.008703016676008701, "consumers": [ "layers.24.mlp_moe_gen.gate_proj", "layers.24.mlp_moe_gen.up_proj" ] }, "layers.25.input_layernorm": { "best_alpha": 0.4, "best_err": 0.027022873517125845, "consumers": [ "layers.25.self_attn.q_proj", "layers.25.self_attn.k_proj", "layers.25.self_attn.v_proj" ] }, "layers.25.input_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.0456560542806983, "consumers": [ "layers.25.self_attn.q_proj_moe_gen", "layers.25.self_attn.k_proj_moe_gen", "layers.25.self_attn.v_proj_moe_gen" ] }, "layers.25.post_attention_layernorm": { "best_alpha": 0.45, "best_err": 0.008004232309758663, "consumers": [ "layers.25.mlp.gate_proj", "layers.25.mlp.up_proj" ] }, "layers.25.post_attention_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.01017605559900403, "consumers": [ "layers.25.mlp_moe_gen.gate_proj", "layers.25.mlp_moe_gen.up_proj" ] }, "layers.26.input_layernorm": { "best_alpha": 0.35, "best_err": 0.0346508389338851, "consumers": [ "layers.26.self_attn.q_proj", "layers.26.self_attn.k_proj", "layers.26.self_attn.v_proj" ] }, "layers.26.input_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.06741268001496792, "consumers": [ "layers.26.self_attn.q_proj_moe_gen", "layers.26.self_attn.k_proj_moe_gen", "layers.26.self_attn.v_proj_moe_gen" ] }, "layers.26.post_attention_layernorm": { "best_alpha": 0.45, "best_err": 0.008322665002197027, "consumers": [ "layers.26.mlp.gate_proj", "layers.26.mlp.up_proj" ] }, "layers.26.post_attention_layernorm_moe_gen": { "best_alpha": 0.5, "best_err": 0.010978639125823975, "consumers": [ "layers.26.mlp_moe_gen.gate_proj", "layers.26.mlp_moe_gen.up_proj" ] }, "layers.27.input_layernorm": { "best_alpha": 0.4, "best_err": 0.04764394834637642, "consumers": [ "layers.27.self_attn.q_proj", "layers.27.self_attn.k_proj", "layers.27.self_attn.v_proj" ] }, "layers.27.input_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.08542009443044662, "consumers": [ "layers.27.self_attn.q_proj_moe_gen", "layers.27.self_attn.k_proj_moe_gen", "layers.27.self_attn.v_proj_moe_gen" ] }, "layers.27.post_attention_layernorm": { "best_alpha": 0.45, "best_err": 0.008581496309489012, "consumers": [ "layers.27.mlp.gate_proj", "layers.27.mlp.up_proj" ] }, "layers.27.post_attention_layernorm_moe_gen": { "best_alpha": 0.45, "best_err": 0.011591496411710978, "consumers": [ "layers.27.mlp_moe_gen.gate_proj", "layers.27.mlp_moe_gen.up_proj" ] }, "layers.28.input_layernorm": { "best_alpha": 0.3, "best_err": 0.05938507616519928, "consumers": [ "layers.28.self_attn.q_proj", "layers.28.self_attn.k_proj", "layers.28.self_attn.v_proj" ] }, "layers.28.input_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.12293312698602676, "consumers": [ "layers.28.self_attn.q_proj_moe_gen", "layers.28.self_attn.k_proj_moe_gen", "layers.28.self_attn.v_proj_moe_gen" ] }, "layers.28.post_attention_layernorm": { "best_alpha": 0.4, "best_err": 0.008860271889716387, "consumers": [ "layers.28.mlp.gate_proj", "layers.28.mlp.up_proj" ] }, "layers.28.post_attention_layernorm_moe_gen": { "best_alpha": 0.45, "best_err": 0.01251167617738247, "consumers": [ "layers.28.mlp_moe_gen.gate_proj", "layers.28.mlp_moe_gen.up_proj" ] }, "layers.29.input_layernorm": { "best_alpha": 0.35, "best_err": 0.06842886283993721, "consumers": [ "layers.29.self_attn.q_proj", "layers.29.self_attn.k_proj", "layers.29.self_attn.v_proj" ] }, "layers.29.input_layernorm_moe_gen": { "best_alpha": 0.45, "best_err": 0.15185140073299408, "consumers": [ "layers.29.self_attn.q_proj_moe_gen", "layers.29.self_attn.k_proj_moe_gen", "layers.29.self_attn.v_proj_moe_gen" ] }, "layers.29.post_attention_layernorm": { "best_alpha": 0.4, "best_err": 0.009515361860394478, "consumers": [ "layers.29.mlp.gate_proj", "layers.29.mlp.up_proj" ] }, "layers.29.post_attention_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.013548723887652159, "consumers": [ "layers.29.mlp_moe_gen.gate_proj", "layers.29.mlp_moe_gen.up_proj" ] }, "layers.30.input_layernorm": { "best_alpha": 0.3, "best_err": 0.07855793833732605, "consumers": [ "layers.30.self_attn.q_proj", "layers.30.self_attn.k_proj", "layers.30.self_attn.v_proj" ] }, "layers.30.input_layernorm_moe_gen": { "best_alpha": 0.35, "best_err": 0.14745976030826569, "consumers": [ "layers.30.self_attn.q_proj_moe_gen", "layers.30.self_attn.k_proj_moe_gen", "layers.30.self_attn.v_proj_moe_gen" ] }, "layers.30.post_attention_layernorm": { "best_alpha": 0.35, "best_err": 0.010349276009947062, "consumers": [ "layers.30.mlp.gate_proj", "layers.30.mlp.up_proj" ] }, "layers.30.post_attention_layernorm_moe_gen": { "best_alpha": 0.35, "best_err": 0.014536601025611162, "consumers": [ "layers.30.mlp_moe_gen.gate_proj", "layers.30.mlp_moe_gen.up_proj" ] }, "layers.31.input_layernorm": { "best_alpha": 0.3, "best_err": 0.10708250477910042, "consumers": [ "layers.31.self_attn.q_proj", "layers.31.self_attn.k_proj", "layers.31.self_attn.v_proj" ] }, "layers.31.input_layernorm_moe_gen": { "best_alpha": 0.45, "best_err": 0.2615283280611038, "consumers": [ "layers.31.self_attn.q_proj_moe_gen", "layers.31.self_attn.k_proj_moe_gen", "layers.31.self_attn.v_proj_moe_gen" ] }, "layers.31.post_attention_layernorm": { "best_alpha": 0.35, "best_err": 0.011434435844421387, "consumers": [ "layers.31.mlp.gate_proj", "layers.31.mlp.up_proj" ] }, "layers.31.post_attention_layernorm_moe_gen": { "best_alpha": 0.35, "best_err": 0.016750527545809746, "consumers": [ "layers.31.mlp_moe_gen.gate_proj", "layers.31.mlp_moe_gen.up_proj" ] }, "layers.32.input_layernorm": { "best_alpha": 0.25, "best_err": 0.11255145259201527, "consumers": [ "layers.32.self_attn.q_proj", "layers.32.self_attn.k_proj", "layers.32.self_attn.v_proj" ] }, "layers.32.input_layernorm_moe_gen": { "best_alpha": 0.45, "best_err": 0.1822471246123314, "consumers": [ "layers.32.self_attn.q_proj_moe_gen", "layers.32.self_attn.k_proj_moe_gen", "layers.32.self_attn.v_proj_moe_gen" ] }, "layers.32.post_attention_layernorm": { "best_alpha": 0.3, "best_err": 0.013713904656469822, "consumers": [ "layers.32.mlp.gate_proj", "layers.32.mlp.up_proj" ] }, "layers.32.post_attention_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.017479958944022655, "consumers": [ "layers.32.mlp_moe_gen.gate_proj", "layers.32.mlp_moe_gen.up_proj" ] }, "layers.33.input_layernorm": { "best_alpha": 0.3, "best_err": 0.17432252317667007, "consumers": [ "layers.33.self_attn.q_proj", "layers.33.self_attn.k_proj", "layers.33.self_attn.v_proj" ] }, "layers.33.input_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.22210053354501724, "consumers": [ "layers.33.self_attn.q_proj_moe_gen", "layers.33.self_attn.k_proj_moe_gen", "layers.33.self_attn.v_proj_moe_gen" ] }, "layers.33.post_attention_layernorm": { "best_alpha": 0.35, "best_err": 0.01600974891334772, "consumers": [ "layers.33.mlp.gate_proj", "layers.33.mlp.up_proj" ] }, "layers.33.post_attention_layernorm_moe_gen": { "best_alpha": 0.45, "best_err": 0.02009424101561308, "consumers": [ "layers.33.mlp_moe_gen.gate_proj", "layers.33.mlp_moe_gen.up_proj" ] }, "layers.34.input_layernorm": { "best_alpha": 0.3, "best_err": 0.19069868326187134, "consumers": [ "layers.34.self_attn.q_proj", "layers.34.self_attn.k_proj", "layers.34.self_attn.v_proj" ] }, "layers.34.input_layernorm_moe_gen": { "best_alpha": 0.45, "best_err": 0.49640433490276337, "consumers": [ "layers.34.self_attn.q_proj_moe_gen", "layers.34.self_attn.k_proj_moe_gen", "layers.34.self_attn.v_proj_moe_gen" ] }, "layers.34.post_attention_layernorm": { "best_alpha": 0.35, "best_err": 0.0172486649826169, "consumers": [ "layers.34.mlp.gate_proj", "layers.34.mlp.up_proj" ] }, "layers.34.post_attention_layernorm_moe_gen": { "best_alpha": 0.45, "best_err": 0.021609684452414513, "consumers": [ "layers.34.mlp_moe_gen.gate_proj", "layers.34.mlp_moe_gen.up_proj" ] }, "layers.35.input_layernorm": { "best_alpha": 0.3, "best_err": 0.19978100806474686, "consumers": [ "layers.35.self_attn.q_proj", "layers.35.self_attn.k_proj", "layers.35.self_attn.v_proj" ] }, "layers.35.input_layernorm_moe_gen": { "best_alpha": 0.4, "best_err": 0.30665621906518936, "consumers": [ "layers.35.self_attn.q_proj_moe_gen", "layers.35.self_attn.k_proj_moe_gen", "layers.35.self_attn.v_proj_moe_gen" ] }, "layers.35.post_attention_layernorm": { "best_alpha": 0.4, "best_err": 0.012281936127692461, "consumers": [ "layers.35.mlp.gate_proj", "layers.35.mlp.up_proj" ] }, "layers.35.post_attention_layernorm_moe_gen": { "best_alpha": 0.45, "best_err": 0.011713964398950338, "consumers": [ "layers.35.mlp_moe_gen.gate_proj", "layers.35.mlp_moe_gen.up_proj" ] } } }