{ "policies": { "1.58bit (brief)": { "args": { "sliding": 2, "gbits": 4, "mlp": 4 }, "memory_report": { "attn_unquant_MB": 1284.5056, "mlp_unquant_MB": 6606.028799999999, "ple_unquant_MB": 110.10048, "attn_packed_MB": 206.954496, "mlp_packed_MB": 1653.44256, "ple_packed_MB": 27.761664, "savings_attn_MB": 1077.551104, "savings_mlp_MB": 4952.586239999999, "savings_ple_MB": 82.33881600000001, "kv": { "sliding_compression": 0.25, "global_compression": 0.15625, "sliding_budget_bits_per_kv_token": 1024, "global_budget_bits_per_kv_token": 1280 }, "ple": { "hot_table_resident": false, "hot_table_MB": 0, "ple_full_MB": 5637.144576, "hot_token_topk": 5000, "hits": 0, "misses": 0, "hit_rate": 0.0, "lru_size": 0 }, "n_sliding_layers": 35, "n_global_layers": 7, "quant_policy": { "bits_sliding_attn": 2, "bits_sliding_mlp": 4, "bits_global_attn": 4, "bits_global_mlp": 4, "bits_ple_per_layer": 4 }, "ple_policy": { "hot_token_topk": 5000, "cold_strategy": "lazy", "lru_capacity": 64 }, "kv_policy": { "sliding_unrotated_bits": 1, "sliding_rotated_bits": 4, "global_unrotated_bits": 2, "global_rotated_bits": 4 } }, "bench_sliding": { "layer": 0, "is_global": false, "input_norm_ms": 2.164599995012395, "ple_ms": 196.9771000003675, "ple_gate_packed_bytes": 328192, "ple_proj_packed_bytes": 332800, "ple_gate_recon_l2": 0.19998590118338486, "ple_proj_recon_l2": 0.1615555011755687, "attn_q_packed_bytes": 1314816, "attn_k_packed_bytes": 328704, "attn_v_packed_bytes": 328704, "attn_o_packed_bytes": 1315840, "attn_q_recon_l2": 1.113928914956695, "attn_k_recon_l2": 1.1164674390988485, "attn_v_recon_l2": 1.116220747187745, "attn_o_recon_l2": 1.108164146438398, "attn_ms": 285.2296000055503, "post_attn_ms": 0.5234999989625067, "mlp_gate_packed_bytes": 13127680, "mlp_up_packed_bytes": 13127680, "mlp_down_packed_bytes": 13112320, "mlp_gate_recon_l2": 0.18576650965389796, "mlp_up_recon_l2": 0.17382894054311962, "mlp_down_recon_l2": 0.20356717484649897, "mlp_ms": 1613.3233000000473, "total_ms": 2100.952099994174, "layer_total_packed_bytes": 43316736 }, "bench_global": { "layer": 5, "is_global": true, "input_norm_ms": 0.3660000074887648, "ple_ms": 44.33210000570398, "ple_gate_packed_bytes": 328192, "ple_proj_packed_bytes": 332800, "ple_gate_recon_l2": 0.2747719913918256, "ple_proj_recon_l2": 0.11705859502340919, "attn_q_packed_bytes": 5251072, "attn_k_packed_bytes": 1312768, "attn_v_packed_bytes": 1312768, "attn_o_packed_bytes": 5248000, "attn_q_recon_l2": 0.19458868030799337, "attn_k_recon_l2": 0.2619226705703569, "attn_v_recon_l2": 0.17262208836753853, "attn_o_recon_l2": 0.18573301232711664, "attn_ms": 542.601499997545, "post_attn_ms": 0.7747000054223463, "mlp_gate_packed_bytes": 13127680, "mlp_up_packed_bytes": 13127680, "mlp_down_packed_bytes": 13112320, "mlp_gate_recon_l2": 0.22854034296442252, "mlp_up_recon_l2": 0.21990214053335333, "mlp_down_recon_l2": 0.21393051150008158, "mlp_ms": 1537.8826999949524, "total_ms": 2127.720100004808, "layer_total_packed_bytes": 53153280 } }, "3bit": { "args": { "sliding": 3, "gbits": 4, "mlp": 4 }, "memory_report": { "attn_unquant_MB": 1284.5056, "mlp_unquant_MB": 6606.028799999999, "ple_unquant_MB": 110.10048, "attn_packed_MB": 321.642496, "mlp_packed_MB": 1653.44256, "ple_packed_MB": 27.761664, "savings_attn_MB": 962.863104, "savings_mlp_MB": 4952.586239999999, "savings_ple_MB": 82.33881600000001, "kv": { "sliding_compression": 0.25, "global_compression": 0.15625, "sliding_budget_bits_per_kv_token": 1024, "global_budget_bits_per_kv_token": 1280 }, "ple": { "hot_table_resident": false, "hot_table_MB": 0, "ple_full_MB": 5637.144576, "hot_token_topk": 5000, "hits": 0, "misses": 0, "hit_rate": 0.0, "lru_size": 0 }, "n_sliding_layers": 35, "n_global_layers": 7, "quant_policy": { "bits_sliding_attn": 3, "bits_sliding_mlp": 4, "bits_global_attn": 4, "bits_global_mlp": 4, "bits_ple_per_layer": 4 }, "ple_policy": { "hot_token_topk": 5000, "cold_strategy": "lazy", "lru_capacity": 64 }, "kv_policy": { "sliding_unrotated_bits": 1, "sliding_rotated_bits": 4, "global_unrotated_bits": 2, "global_rotated_bits": 4 } }, "bench_sliding": { "layer": 0, "is_global": false, "input_norm_ms": 0.4782999894814566, "ple_ms": 206.5034000115702, "ple_gate_packed_bytes": 328192, "ple_proj_packed_bytes": 332800, "ple_gate_recon_l2": 0.19998590118338486, "ple_proj_recon_l2": 0.1615555011755687, "attn_q_packed_bytes": 2625536, "attn_k_packed_bytes": 656384, "attn_v_packed_bytes": 656384, "attn_o_packed_bytes": 2626560, "attn_q_recon_l2": 0.30230211215011915, "attn_k_recon_l2": 0.2887961155809107, "attn_v_recon_l2": 0.28545971874666093, "attn_o_recon_l2": 0.29116891566865305, "attn_ms": 244.29490001057275, "post_attn_ms": 0.9511999960523099, "mlp_gate_packed_bytes": 13127680, "mlp_up_packed_bytes": 13127680, "mlp_down_packed_bytes": 13112320, "mlp_gate_recon_l2": 0.18576650965389796, "mlp_up_recon_l2": 0.17382894054311962, "mlp_down_recon_l2": 0.20356717484649897, "mlp_ms": 1559.267800010275, "total_ms": 2013.148399986676, "layer_total_packed_bytes": 46593536 }, "bench_global": { "layer": 5, "is_global": true, "input_norm_ms": 0.18650000856723636, "ple_ms": 44.89550000289455, "ple_gate_packed_bytes": 328192, "ple_proj_packed_bytes": 332800, "ple_gate_recon_l2": 0.2747719913918256, "ple_proj_recon_l2": 0.11705859502340919, "attn_q_packed_bytes": 5251072, "attn_k_packed_bytes": 1312768, "attn_v_packed_bytes": 1312768, "attn_o_packed_bytes": 5248000, "attn_q_recon_l2": 0.19458868030799337, "attn_k_recon_l2": 0.2619226705703569, "attn_v_recon_l2": 0.17262208836753853, "attn_o_recon_l2": 0.18573301232711664, "attn_ms": 534.7028000105638, "post_attn_ms": 2.530099998693913, "mlp_gate_packed_bytes": 13127680, "mlp_up_packed_bytes": 13127680, "mlp_down_packed_bytes": 13112320, "mlp_gate_recon_l2": 0.22854034296442252, "mlp_up_recon_l2": 0.21990214053335333, "mlp_down_recon_l2": 0.21393051150008158, "mlp_ms": 1545.74469999352, "total_ms": 2129.8614000115776, "layer_total_packed_bytes": 53153280 } }, "4bit (uniform)": { "args": { "sliding": 4, "gbits": 4, "mlp": 4 }, "memory_report": { "attn_unquant_MB": 1284.5056, "mlp_unquant_MB": 6606.028799999999, "ple_unquant_MB": 110.10048, "attn_packed_MB": 321.642496, "mlp_packed_MB": 1653.44256, "ple_packed_MB": 27.761664, "savings_attn_MB": 962.863104, "savings_mlp_MB": 4952.586239999999, "savings_ple_MB": 82.33881600000001, "kv": { "sliding_compression": 0.25, "global_compression": 0.15625, "sliding_budget_bits_per_kv_token": 1024, "global_budget_bits_per_kv_token": 1280 }, "ple": { "hot_table_resident": false, "hot_table_MB": 0, "ple_full_MB": 5637.144576, "hot_token_topk": 5000, "hits": 0, "misses": 0, "hit_rate": 0.0, "lru_size": 0 }, "n_sliding_layers": 35, "n_global_layers": 7, "quant_policy": { "bits_sliding_attn": 4, "bits_sliding_mlp": 4, "bits_global_attn": 4, "bits_global_mlp": 4, "bits_ple_per_layer": 4 }, "ple_policy": { "hot_token_topk": 5000, "cold_strategy": "lazy", "lru_capacity": 64 }, "kv_policy": { "sliding_unrotated_bits": 1, "sliding_rotated_bits": 4, "global_unrotated_bits": 2, "global_rotated_bits": 4 } }, "bench_sliding": { "layer": 0, "is_global": false, "input_norm_ms": 0.39339999784715474, "ple_ms": 209.62190000864211, "ple_gate_packed_bytes": 328192, "ple_proj_packed_bytes": 332800, "ple_gate_recon_l2": 0.19998590118338486, "ple_proj_recon_l2": 0.1615555011755687, "attn_q_packed_bytes": 2625536, "attn_k_packed_bytes": 656384, "attn_v_packed_bytes": 656384, "attn_o_packed_bytes": 2626560, "attn_q_recon_l2": 0.17265795799733186, "attn_k_recon_l2": 0.16476979727635283, "attn_v_recon_l2": 0.16278332317657349, "attn_o_recon_l2": 0.16589325781467756, "attn_ms": 258.8467000023229, "post_attn_ms": 0.668799999402836, "mlp_gate_packed_bytes": 13127680, "mlp_up_packed_bytes": 13127680, "mlp_down_packed_bytes": 13112320, "mlp_gate_recon_l2": 0.18576650965389796, "mlp_up_recon_l2": 0.17382894054311962, "mlp_down_recon_l2": 0.20356717484649897, "mlp_ms": 1634.9234999943292, "total_ms": 2106.1582999973325, "layer_total_packed_bytes": 46593536 }, "bench_global": { "layer": 5, "is_global": true, "input_norm_ms": 0.3288000007160008, "ple_ms": 40.49749999830965, "ple_gate_packed_bytes": 328192, "ple_proj_packed_bytes": 332800, "ple_gate_recon_l2": 0.2747719913918256, "ple_proj_recon_l2": 0.11705859502340919, "attn_q_packed_bytes": 5251072, "attn_k_packed_bytes": 1312768, "attn_v_packed_bytes": 1312768, "attn_o_packed_bytes": 5248000, "attn_q_recon_l2": 0.19458868030799337, "attn_k_recon_l2": 0.2619226705703569, "attn_v_recon_l2": 0.17262208836753853, "attn_o_recon_l2": 0.18573301232711664, "attn_ms": 519.8174000106519, "post_attn_ms": 1.9394000119064003, "mlp_gate_packed_bytes": 13127680, "mlp_up_packed_bytes": 13127680, "mlp_down_packed_bytes": 13112320, "mlp_gate_recon_l2": 0.22854034296442252, "mlp_up_recon_l2": 0.21990214053335333, "mlp_down_recon_l2": 0.21393051150008158, "mlp_ms": 1523.029600008158, "total_ms": 2088.1206000049133, "layer_total_packed_bytes": 53153280 } }, "16bit (no quant)": { "args": { "sliding": 16, "gbits": 16, "mlp": 16 }, "memory_report": { "attn_unquant_MB": 1284.5056, "mlp_unquant_MB": 6606.028799999999, "ple_unquant_MB": 110.10048, "attn_packed_MB": 1284.5056, "mlp_packed_MB": 6606.028799999999, "ple_packed_MB": 110.10048, "savings_attn_MB": 0.0, "savings_mlp_MB": 0.0, "savings_ple_MB": 0.0, "kv": { "sliding_compression": 0.25, "global_compression": 0.15625, "sliding_budget_bits_per_kv_token": 1024, "global_budget_bits_per_kv_token": 1280 }, "ple": { "hot_table_resident": false, "hot_table_MB": 0, "ple_full_MB": 5637.144576, "hot_token_topk": 5000, "hits": 0, "misses": 0, "hit_rate": 0.0, "lru_size": 0 }, "n_sliding_layers": 35, "n_global_layers": 7, "quant_policy": { "bits_sliding_attn": 16, "bits_sliding_mlp": 16, "bits_global_attn": 16, "bits_global_mlp": 16, "bits_ple_per_layer": 16 }, "ple_policy": { "hot_token_topk": 5000, "cold_strategy": "lazy", "lru_capacity": 64 }, "kv_policy": { "sliding_unrotated_bits": 1, "sliding_rotated_bits": 4, "global_unrotated_bits": 2, "global_rotated_bits": 4 } }, "bench_sliding": { "layer": 0, "is_global": false, "input_norm_ms": 0.367300002835691, "ple_ms": 191.01950000913348, "ple_gate_packed_bytes": 1311232, "ple_proj_packed_bytes": 1315840, "ple_gate_recon_l2": 0.0, "ple_proj_recon_l2": 0.0, "attn_q_packed_bytes": 10485760, "attn_k_packed_bytes": 2621440, "attn_v_packed_bytes": 2621440, "attn_o_packed_bytes": 10485760, "attn_q_recon_l2": 0.0, "attn_k_recon_l2": 0.0, "attn_v_recon_l2": 0.0, "attn_o_recon_l2": 0.0, "attn_ms": 89.54559999983758, "post_attn_ms": 0.7689999911235645, "mlp_gate_packed_bytes": 52428800, "mlp_up_packed_bytes": 52428800, "mlp_down_packed_bytes": 52428800, "mlp_gate_recon_l2": 0.0, "mlp_up_recon_l2": 0.0, "mlp_down_recon_l2": 0.0, "mlp_ms": 555.5779999995138, "total_ms": 838.7913000042317, "layer_total_packed_bytes": 186127872 }, "bench_global": { "layer": 5, "is_global": true, "input_norm_ms": 0.45989999489393085, "ple_ms": 37.67229999357369, "ple_gate_packed_bytes": 1311232, "ple_proj_packed_bytes": 1315840, "ple_gate_recon_l2": 0.0, "ple_proj_recon_l2": 0.0, "attn_q_packed_bytes": 20971520, "attn_k_packed_bytes": 5242880, "attn_v_packed_bytes": 5242880, "attn_o_packed_bytes": 20971520, "attn_q_recon_l2": 0.0, "attn_k_recon_l2": 0.0, "attn_v_recon_l2": 0.0, "attn_o_recon_l2": 0.0, "attn_ms": 194.55090000701603, "post_attn_ms": 0.6990999972913414, "mlp_gate_packed_bytes": 52428800, "mlp_up_packed_bytes": 52428800, "mlp_down_packed_bytes": 52428800, "mlp_gate_recon_l2": 0.0, "mlp_up_recon_l2": 0.0, "mlp_down_recon_l2": 0.0, "mlp_ms": 505.15009999799076, "total_ms": 740.3529000002891, "layer_total_packed_bytes": 212342272 } } }, "common": { "n_tokens": 200, "seq_len": 16 }, "ple_workload": { "hot_table_resident": true, "hot_table_MB": 107.52, "ple_full_MB": 5637.144576, "hot_token_topk": 5000, "hits": 7098, "misses": 1302, "hit_rate": 0.845, "lru_size": 31, "elapsed_s": 0.3837204000010388, "lookups_per_sec": 21890.939340147826 } }