| { |
| "overall": { |
| "sparsity": 0.12496772179236779, |
| "params": 6979321856, |
| "zero_params": 872189952, |
| "remaining_params": 6107131904, |
| "delta_sparsity": 0.12496772179236779, |
| "params_pruned_this_step": 872189952 |
| }, |
| "by_type": { |
| "attn_out": { |
| "layers": 32, |
| "sparsity": 0.124969482421875, |
| "zero_rows": 0, |
| "rows": 131072, |
| "zero_cols": 16380, |
| "cols": 131072 |
| }, |
| "ff_out": { |
| "layers": 32, |
| "sparsity": 0.12496693929036458, |
| "zero_rows": 0, |
| "rows": 131072, |
| "zero_cols": 49139, |
| "cols": 393216 |
| }, |
| "ff_proj": { |
| "layers": 32, |
| "sparsity": 0.12496693929036458, |
| "zero_rows": 49139, |
| "rows": 393216, |
| "zero_cols": 0, |
| "cols": 131072 |
| }, |
| "k_proj": { |
| "layers": 32, |
| "sparsity": 0.124969482421875, |
| "zero_rows": 16380, |
| "rows": 131072, |
| "zero_cols": 0, |
| "cols": 131072 |
| }, |
| "q_proj": { |
| "layers": 32, |
| "sparsity": 0.124969482421875, |
| "zero_rows": 16380, |
| "rows": 131072, |
| "zero_cols": 0, |
| "cols": 131072 |
| }, |
| "up_proj": { |
| "layers": 32, |
| "sparsity": 0.12496693929036458, |
| "zero_rows": 49139, |
| "rows": 393216, |
| "zero_cols": 0, |
| "cols": 131072 |
| }, |
| "v_proj": { |
| "layers": 32, |
| "sparsity": 0.124969482421875, |
| "zero_rows": 16380, |
| "rows": 131072, |
| "zero_cols": 0, |
| "cols": 131072 |
| } |
| }, |
| "by_block": { |
| "0": 0.8305100661057693, |
| "1": 0.7952599158653846, |
| "2": 0.7151066706730769, |
| "3": 0.554443359375, |
| "4": 0.3346980168269231, |
| "5": 0.20494666466346154, |
| "6": 0.1375450721153846, |
| "7": 0.041616586538461536, |
| "8": 0.04790790264423077, |
| "9": 0.028301532451923076, |
| "10": 0.021052433894230768, |
| "11": 0.01891150841346154, |
| "12": 0.017014723557692308, |
| "13": 0.005239633413461538, |
| "14": 0.005521334134615385, |
| "15": 0.021146334134615384, |
| "16": 0.010422926682692308, |
| "17": 0.013258713942307692, |
| "18": 0.011887770432692308, |
| "19": 0.007756159855769231, |
| "20": 0.007943960336538462, |
| "21": 0.009540264423076924, |
| "22": 0.012413611778846154, |
| "23": 0.007493239182692308, |
| "24": 0.006873497596153846, |
| "25": 0.011211688701923076, |
| "26": 0.011831430288461538, |
| "27": 0.010591947115384616, |
| "28": 0.01128680889423077, |
| "29": 0.02013221153846154, |
| "30": 0.03168194110576923, |
| "31": 0.03541917067307692 |
| } |
| } |
