| { | |
| "policies": { | |
| "1.58bit (brief)": { | |
| "args": { | |
| "sliding": 2, | |
| "gbits": 4, | |
| "mlp": 4 | |
| }, | |
| "memory_report": { | |
| "attn_unquant_MB": 1284.5056, | |
| "mlp_unquant_MB": 6606.028799999999, | |
| "ple_unquant_MB": 110.10048, | |
| "attn_packed_MB": 206.954496, | |
| "mlp_packed_MB": 1653.44256, | |
| "ple_packed_MB": 27.761664, | |
| "savings_attn_MB": 1077.551104, | |
| "savings_mlp_MB": 4952.586239999999, | |
| "savings_ple_MB": 82.33881600000001, | |
| "kv": { | |
| "sliding_compression": 0.25, | |
| "global_compression": 0.15625, | |
| "sliding_budget_bits_per_kv_token": 1024, | |
| "global_budget_bits_per_kv_token": 1280 | |
| }, | |
| "ple": { | |
| "hot_table_resident": false, | |
| "hot_table_MB": 0, | |
| "ple_full_MB": 5637.144576, | |
| "hot_token_topk": 5000, | |
| "hits": 0, | |
| "misses": 0, | |
| "hit_rate": 0.0, | |
| "lru_size": 0 | |
| }, | |
| "n_sliding_layers": 35, | |
| "n_global_layers": 7, | |
| "quant_policy": { | |
| "bits_sliding_attn": 2, | |
| "bits_sliding_mlp": 4, | |
| "bits_global_attn": 4, | |
| "bits_global_mlp": 4, | |
| "bits_ple_per_layer": 4 | |
| }, | |
| "ple_policy": { | |
| "hot_token_topk": 5000, | |
| "cold_strategy": "lazy", | |
| "lru_capacity": 64 | |
| }, | |
| "kv_policy": { | |
| "sliding_unrotated_bits": 1, | |
| "sliding_rotated_bits": 4, | |
| "global_unrotated_bits": 2, | |
| "global_rotated_bits": 4 | |
| } | |
| }, | |
| "bench_sliding": { | |
| "layer": 0, | |
| "is_global": false, | |
| "input_norm_ms": 2.164599995012395, | |
| "ple_ms": 196.9771000003675, | |
| "ple_gate_packed_bytes": 328192, | |
| "ple_proj_packed_bytes": 332800, | |
| "ple_gate_recon_l2": 0.19998590118338486, | |
| "ple_proj_recon_l2": 0.1615555011755687, | |
| "attn_q_packed_bytes": 1314816, | |
| "attn_k_packed_bytes": 328704, | |
| "attn_v_packed_bytes": 328704, | |
| "attn_o_packed_bytes": 1315840, | |
| "attn_q_recon_l2": 1.113928914956695, | |
| "attn_k_recon_l2": 1.1164674390988485, | |
| "attn_v_recon_l2": 1.116220747187745, | |
| "attn_o_recon_l2": 1.108164146438398, | |
| "attn_ms": 285.2296000055503, | |
| "post_attn_ms": 0.5234999989625067, | |
| "mlp_gate_packed_bytes": 13127680, | |
| "mlp_up_packed_bytes": 13127680, | |
| "mlp_down_packed_bytes": 13112320, | |
| "mlp_gate_recon_l2": 0.18576650965389796, | |
| "mlp_up_recon_l2": 0.17382894054311962, | |
| "mlp_down_recon_l2": 0.20356717484649897, | |
| "mlp_ms": 1613.3233000000473, | |
| "total_ms": 2100.952099994174, | |
| "layer_total_packed_bytes": 43316736 | |
| }, | |
| "bench_global": { | |
| "layer": 5, | |
| "is_global": true, | |
| "input_norm_ms": 0.3660000074887648, | |
| "ple_ms": 44.33210000570398, | |
| "ple_gate_packed_bytes": 328192, | |
| "ple_proj_packed_bytes": 332800, | |
| "ple_gate_recon_l2": 0.2747719913918256, | |
| "ple_proj_recon_l2": 0.11705859502340919, | |
| "attn_q_packed_bytes": 5251072, | |
| "attn_k_packed_bytes": 1312768, | |
| "attn_v_packed_bytes": 1312768, | |
| "attn_o_packed_bytes": 5248000, | |
| "attn_q_recon_l2": 0.19458868030799337, | |
| "attn_k_recon_l2": 0.2619226705703569, | |
| "attn_v_recon_l2": 0.17262208836753853, | |
| "attn_o_recon_l2": 0.18573301232711664, | |
| "attn_ms": 542.601499997545, | |
| "post_attn_ms": 0.7747000054223463, | |
| "mlp_gate_packed_bytes": 13127680, | |
| "mlp_up_packed_bytes": 13127680, | |
| "mlp_down_packed_bytes": 13112320, | |
| "mlp_gate_recon_l2": 0.22854034296442252, | |
| "mlp_up_recon_l2": 0.21990214053335333, | |
| "mlp_down_recon_l2": 0.21393051150008158, | |
| "mlp_ms": 1537.8826999949524, | |
| "total_ms": 2127.720100004808, | |
| "layer_total_packed_bytes": 53153280 | |
| } | |
| }, | |
| "3bit": { | |
| "args": { | |
| "sliding": 3, | |
| "gbits": 4, | |
| "mlp": 4 | |
| }, | |
| "memory_report": { | |
| "attn_unquant_MB": 1284.5056, | |
| "mlp_unquant_MB": 6606.028799999999, | |
| "ple_unquant_MB": 110.10048, | |
| "attn_packed_MB": 321.642496, | |
| "mlp_packed_MB": 1653.44256, | |
| "ple_packed_MB": 27.761664, | |
| "savings_attn_MB": 962.863104, | |
| "savings_mlp_MB": 4952.586239999999, | |
| "savings_ple_MB": 82.33881600000001, | |
| "kv": { | |
| "sliding_compression": 0.25, | |
| "global_compression": 0.15625, | |
| "sliding_budget_bits_per_kv_token": 1024, | |
| "global_budget_bits_per_kv_token": 1280 | |
| }, | |
| "ple": { | |
| "hot_table_resident": false, | |
| "hot_table_MB": 0, | |
| "ple_full_MB": 5637.144576, | |
| "hot_token_topk": 5000, | |
| "hits": 0, | |
| "misses": 0, | |
| "hit_rate": 0.0, | |
| "lru_size": 0 | |
| }, | |
| "n_sliding_layers": 35, | |
| "n_global_layers": 7, | |
| "quant_policy": { | |
| "bits_sliding_attn": 3, | |
| "bits_sliding_mlp": 4, | |
| "bits_global_attn": 4, | |
| "bits_global_mlp": 4, | |
| "bits_ple_per_layer": 4 | |
| }, | |
| "ple_policy": { | |
| "hot_token_topk": 5000, | |
| "cold_strategy": "lazy", | |
| "lru_capacity": 64 | |
| }, | |
| "kv_policy": { | |
| "sliding_unrotated_bits": 1, | |
| "sliding_rotated_bits": 4, | |
| "global_unrotated_bits": 2, | |
| "global_rotated_bits": 4 | |
| } | |
| }, | |
| "bench_sliding": { | |
| "layer": 0, | |
| "is_global": false, | |
| "input_norm_ms": 0.4782999894814566, | |
| "ple_ms": 206.5034000115702, | |
| "ple_gate_packed_bytes": 328192, | |
| "ple_proj_packed_bytes": 332800, | |
| "ple_gate_recon_l2": 0.19998590118338486, | |
| "ple_proj_recon_l2": 0.1615555011755687, | |
| "attn_q_packed_bytes": 2625536, | |
| "attn_k_packed_bytes": 656384, | |
| "attn_v_packed_bytes": 656384, | |
| "attn_o_packed_bytes": 2626560, | |
| "attn_q_recon_l2": 0.30230211215011915, | |
| "attn_k_recon_l2": 0.2887961155809107, | |
| "attn_v_recon_l2": 0.28545971874666093, | |
| "attn_o_recon_l2": 0.29116891566865305, | |
| "attn_ms": 244.29490001057275, | |
| "post_attn_ms": 0.9511999960523099, | |
| "mlp_gate_packed_bytes": 13127680, | |
| "mlp_up_packed_bytes": 13127680, | |
| "mlp_down_packed_bytes": 13112320, | |
| "mlp_gate_recon_l2": 0.18576650965389796, | |
| "mlp_up_recon_l2": 0.17382894054311962, | |
| "mlp_down_recon_l2": 0.20356717484649897, | |
| "mlp_ms": 1559.267800010275, | |
| "total_ms": 2013.148399986676, | |
| "layer_total_packed_bytes": 46593536 | |
| }, | |
| "bench_global": { | |
| "layer": 5, | |
| "is_global": true, | |
| "input_norm_ms": 0.18650000856723636, | |
| "ple_ms": 44.89550000289455, | |
| "ple_gate_packed_bytes": 328192, | |
| "ple_proj_packed_bytes": 332800, | |
| "ple_gate_recon_l2": 0.2747719913918256, | |
| "ple_proj_recon_l2": 0.11705859502340919, | |
| "attn_q_packed_bytes": 5251072, | |
| "attn_k_packed_bytes": 1312768, | |
| "attn_v_packed_bytes": 1312768, | |
| "attn_o_packed_bytes": 5248000, | |
| "attn_q_recon_l2": 0.19458868030799337, | |
| "attn_k_recon_l2": 0.2619226705703569, | |
| "attn_v_recon_l2": 0.17262208836753853, | |
| "attn_o_recon_l2": 0.18573301232711664, | |
| "attn_ms": 534.7028000105638, | |
| "post_attn_ms": 2.530099998693913, | |
| "mlp_gate_packed_bytes": 13127680, | |
| "mlp_up_packed_bytes": 13127680, | |
| "mlp_down_packed_bytes": 13112320, | |
| "mlp_gate_recon_l2": 0.22854034296442252, | |
| "mlp_up_recon_l2": 0.21990214053335333, | |
| "mlp_down_recon_l2": 0.21393051150008158, | |
| "mlp_ms": 1545.74469999352, | |
| "total_ms": 2129.8614000115776, | |
| "layer_total_packed_bytes": 53153280 | |
| } | |
| }, | |
| "4bit (uniform)": { | |
| "args": { | |
| "sliding": 4, | |
| "gbits": 4, | |
| "mlp": 4 | |
| }, | |
| "memory_report": { | |
| "attn_unquant_MB": 1284.5056, | |
| "mlp_unquant_MB": 6606.028799999999, | |
| "ple_unquant_MB": 110.10048, | |
| "attn_packed_MB": 321.642496, | |
| "mlp_packed_MB": 1653.44256, | |
| "ple_packed_MB": 27.761664, | |
| "savings_attn_MB": 962.863104, | |
| "savings_mlp_MB": 4952.586239999999, | |
| "savings_ple_MB": 82.33881600000001, | |
| "kv": { | |
| "sliding_compression": 0.25, | |
| "global_compression": 0.15625, | |
| "sliding_budget_bits_per_kv_token": 1024, | |
| "global_budget_bits_per_kv_token": 1280 | |
| }, | |
| "ple": { | |
| "hot_table_resident": false, | |
| "hot_table_MB": 0, | |
| "ple_full_MB": 5637.144576, | |
| "hot_token_topk": 5000, | |
| "hits": 0, | |
| "misses": 0, | |
| "hit_rate": 0.0, | |
| "lru_size": 0 | |
| }, | |
| "n_sliding_layers": 35, | |
| "n_global_layers": 7, | |
| "quant_policy": { | |
| "bits_sliding_attn": 4, | |
| "bits_sliding_mlp": 4, | |
| "bits_global_attn": 4, | |
| "bits_global_mlp": 4, | |
| "bits_ple_per_layer": 4 | |
| }, | |
| "ple_policy": { | |
| "hot_token_topk": 5000, | |
| "cold_strategy": "lazy", | |
| "lru_capacity": 64 | |
| }, | |
| "kv_policy": { | |
| "sliding_unrotated_bits": 1, | |
| "sliding_rotated_bits": 4, | |
| "global_unrotated_bits": 2, | |
| "global_rotated_bits": 4 | |
| } | |
| }, | |
| "bench_sliding": { | |
| "layer": 0, | |
| "is_global": false, | |
| "input_norm_ms": 0.39339999784715474, | |
| "ple_ms": 209.62190000864211, | |
| "ple_gate_packed_bytes": 328192, | |
| "ple_proj_packed_bytes": 332800, | |
| "ple_gate_recon_l2": 0.19998590118338486, | |
| "ple_proj_recon_l2": 0.1615555011755687, | |
| "attn_q_packed_bytes": 2625536, | |
| "attn_k_packed_bytes": 656384, | |
| "attn_v_packed_bytes": 656384, | |
| "attn_o_packed_bytes": 2626560, | |
| "attn_q_recon_l2": 0.17265795799733186, | |
| "attn_k_recon_l2": 0.16476979727635283, | |
| "attn_v_recon_l2": 0.16278332317657349, | |
| "attn_o_recon_l2": 0.16589325781467756, | |
| "attn_ms": 258.8467000023229, | |
| "post_attn_ms": 0.668799999402836, | |
| "mlp_gate_packed_bytes": 13127680, | |
| "mlp_up_packed_bytes": 13127680, | |
| "mlp_down_packed_bytes": 13112320, | |
| "mlp_gate_recon_l2": 0.18576650965389796, | |
| "mlp_up_recon_l2": 0.17382894054311962, | |
| "mlp_down_recon_l2": 0.20356717484649897, | |
| "mlp_ms": 1634.9234999943292, | |
| "total_ms": 2106.1582999973325, | |
| "layer_total_packed_bytes": 46593536 | |
| }, | |
| "bench_global": { | |
| "layer": 5, | |
| "is_global": true, | |
| "input_norm_ms": 0.3288000007160008, | |
| "ple_ms": 40.49749999830965, | |
| "ple_gate_packed_bytes": 328192, | |
| "ple_proj_packed_bytes": 332800, | |
| "ple_gate_recon_l2": 0.2747719913918256, | |
| "ple_proj_recon_l2": 0.11705859502340919, | |
| "attn_q_packed_bytes": 5251072, | |
| "attn_k_packed_bytes": 1312768, | |
| "attn_v_packed_bytes": 1312768, | |
| "attn_o_packed_bytes": 5248000, | |
| "attn_q_recon_l2": 0.19458868030799337, | |
| "attn_k_recon_l2": 0.2619226705703569, | |
| "attn_v_recon_l2": 0.17262208836753853, | |
| "attn_o_recon_l2": 0.18573301232711664, | |
| "attn_ms": 519.8174000106519, | |
| "post_attn_ms": 1.9394000119064003, | |
| "mlp_gate_packed_bytes": 13127680, | |
| "mlp_up_packed_bytes": 13127680, | |
| "mlp_down_packed_bytes": 13112320, | |
| "mlp_gate_recon_l2": 0.22854034296442252, | |
| "mlp_up_recon_l2": 0.21990214053335333, | |
| "mlp_down_recon_l2": 0.21393051150008158, | |
| "mlp_ms": 1523.029600008158, | |
| "total_ms": 2088.1206000049133, | |
| "layer_total_packed_bytes": 53153280 | |
| } | |
| }, | |
| "16bit (no quant)": { | |
| "args": { | |
| "sliding": 16, | |
| "gbits": 16, | |
| "mlp": 16 | |
| }, | |
| "memory_report": { | |
| "attn_unquant_MB": 1284.5056, | |
| "mlp_unquant_MB": 6606.028799999999, | |
| "ple_unquant_MB": 110.10048, | |
| "attn_packed_MB": 1284.5056, | |
| "mlp_packed_MB": 6606.028799999999, | |
| "ple_packed_MB": 110.10048, | |
| "savings_attn_MB": 0.0, | |
| "savings_mlp_MB": 0.0, | |
| "savings_ple_MB": 0.0, | |
| "kv": { | |
| "sliding_compression": 0.25, | |
| "global_compression": 0.15625, | |
| "sliding_budget_bits_per_kv_token": 1024, | |
| "global_budget_bits_per_kv_token": 1280 | |
| }, | |
| "ple": { | |
| "hot_table_resident": false, | |
| "hot_table_MB": 0, | |
| "ple_full_MB": 5637.144576, | |
| "hot_token_topk": 5000, | |
| "hits": 0, | |
| "misses": 0, | |
| "hit_rate": 0.0, | |
| "lru_size": 0 | |
| }, | |
| "n_sliding_layers": 35, | |
| "n_global_layers": 7, | |
| "quant_policy": { | |
| "bits_sliding_attn": 16, | |
| "bits_sliding_mlp": 16, | |
| "bits_global_attn": 16, | |
| "bits_global_mlp": 16, | |
| "bits_ple_per_layer": 16 | |
| }, | |
| "ple_policy": { | |
| "hot_token_topk": 5000, | |
| "cold_strategy": "lazy", | |
| "lru_capacity": 64 | |
| }, | |
| "kv_policy": { | |
| "sliding_unrotated_bits": 1, | |
| "sliding_rotated_bits": 4, | |
| "global_unrotated_bits": 2, | |
| "global_rotated_bits": 4 | |
| } | |
| }, | |
| "bench_sliding": { | |
| "layer": 0, | |
| "is_global": false, | |
| "input_norm_ms": 0.367300002835691, | |
| "ple_ms": 191.01950000913348, | |
| "ple_gate_packed_bytes": 1311232, | |
| "ple_proj_packed_bytes": 1315840, | |
| "ple_gate_recon_l2": 0.0, | |
| "ple_proj_recon_l2": 0.0, | |
| "attn_q_packed_bytes": 10485760, | |
| "attn_k_packed_bytes": 2621440, | |
| "attn_v_packed_bytes": 2621440, | |
| "attn_o_packed_bytes": 10485760, | |
| "attn_q_recon_l2": 0.0, | |
| "attn_k_recon_l2": 0.0, | |
| "attn_v_recon_l2": 0.0, | |
| "attn_o_recon_l2": 0.0, | |
| "attn_ms": 89.54559999983758, | |
| "post_attn_ms": 0.7689999911235645, | |
| "mlp_gate_packed_bytes": 52428800, | |
| "mlp_up_packed_bytes": 52428800, | |
| "mlp_down_packed_bytes": 52428800, | |
| "mlp_gate_recon_l2": 0.0, | |
| "mlp_up_recon_l2": 0.0, | |
| "mlp_down_recon_l2": 0.0, | |
| "mlp_ms": 555.5779999995138, | |
| "total_ms": 838.7913000042317, | |
| "layer_total_packed_bytes": 186127872 | |
| }, | |
| "bench_global": { | |
| "layer": 5, | |
| "is_global": true, | |
| "input_norm_ms": 0.45989999489393085, | |
| "ple_ms": 37.67229999357369, | |
| "ple_gate_packed_bytes": 1311232, | |
| "ple_proj_packed_bytes": 1315840, | |
| "ple_gate_recon_l2": 0.0, | |
| "ple_proj_recon_l2": 0.0, | |
| "attn_q_packed_bytes": 20971520, | |
| "attn_k_packed_bytes": 5242880, | |
| "attn_v_packed_bytes": 5242880, | |
| "attn_o_packed_bytes": 20971520, | |
| "attn_q_recon_l2": 0.0, | |
| "attn_k_recon_l2": 0.0, | |
| "attn_v_recon_l2": 0.0, | |
| "attn_o_recon_l2": 0.0, | |
| "attn_ms": 194.55090000701603, | |
| "post_attn_ms": 0.6990999972913414, | |
| "mlp_gate_packed_bytes": 52428800, | |
| "mlp_up_packed_bytes": 52428800, | |
| "mlp_down_packed_bytes": 52428800, | |
| "mlp_gate_recon_l2": 0.0, | |
| "mlp_up_recon_l2": 0.0, | |
| "mlp_down_recon_l2": 0.0, | |
| "mlp_ms": 505.15009999799076, | |
| "total_ms": 740.3529000002891, | |
| "layer_total_packed_bytes": 212342272 | |
| } | |
| } | |
| }, | |
| "common": { | |
| "n_tokens": 200, | |
| "seq_len": 16 | |
| }, | |
| "ple_workload": { | |
| "hot_table_resident": true, | |
| "hot_table_MB": 107.52, | |
| "ple_full_MB": 5637.144576, | |
| "hot_token_topk": 5000, | |
| "hits": 7098, | |
| "misses": 1302, | |
| "hit_rate": 0.845, | |
| "lru_size": 31, | |
| "elapsed_s": 0.3837204000010388, | |
| "lookups_per_sec": 21890.939340147826 | |
| } | |
| } |