| { |
| "overall": { |
| "sparsity": 0.23431807297926682, |
| "params": 6979321856, |
| "zero_params": 1635381248, |
| "remaining_params": 5343940608, |
| "delta_sparsity": 0.10935035118689904, |
| "params_pruned_this_step": 763191296 |
| }, |
| "by_type": { |
| "attn_out": { |
| "layers": 32, |
| "sparsity": 0.23432159423828125, |
| "zero_rows": 0, |
| "rows": 131072, |
| "zero_cols": 30713, |
| "cols": 131072 |
| }, |
| "ff_out": { |
| "layers": 32, |
| "sparsity": 0.2343165079752604, |
| "zero_rows": 0, |
| "rows": 131072, |
| "zero_cols": 92137, |
| "cols": 393216 |
| }, |
| "ff_proj": { |
| "layers": 32, |
| "sparsity": 0.2343165079752604, |
| "zero_rows": 92137, |
| "rows": 393216, |
| "zero_cols": 0, |
| "cols": 131072 |
| }, |
| "k_proj": { |
| "layers": 32, |
| "sparsity": 0.23432159423828125, |
| "zero_rows": 30713, |
| "rows": 131072, |
| "zero_cols": 0, |
| "cols": 131072 |
| }, |
| "q_proj": { |
| "layers": 32, |
| "sparsity": 0.23432159423828125, |
| "zero_rows": 30713, |
| "rows": 131072, |
| "zero_cols": 0, |
| "cols": 131072 |
| }, |
| "up_proj": { |
| "layers": 32, |
| "sparsity": 0.2343165079752604, |
| "zero_rows": 92137, |
| "rows": 393216, |
| "zero_cols": 0, |
| "cols": 131072 |
| }, |
| "v_proj": { |
| "layers": 32, |
| "sparsity": 0.23432159423828125, |
| "zero_rows": 30713, |
| "rows": 131072, |
| "zero_cols": 0, |
| "cols": 131072 |
| } |
| }, |
| "by_block": { |
| "0": 0.8541541466346154, |
| "1": 0.8175893930288461, |
| "2": 0.7328913762019231, |
| "3": 0.5879657451923077, |
| "4": 0.38467172475961536, |
| "5": 0.25839468149038464, |
| "6": 0.23409329927884615, |
| "7": 0.1275916466346154, |
| "8": 0.1435546875, |
| "9": 0.13823993389423078, |
| "10": 0.1140887920673077, |
| "11": 0.11716871995192307, |
| "12": 0.11831430288461539, |
| "13": 0.09320537860576923, |
| "14": 0.13196739783653846, |
| "15": 0.15320763221153846, |
| "16": 0.1298828125, |
| "17": 0.1385779747596154, |
| "18": 0.14967698317307693, |
| "19": 0.10494290865384616, |
| "20": 0.1157977764423077, |
| "21": 0.1205303485576923, |
| "22": 0.11307466947115384, |
| "23": 0.09205979567307693, |
| "24": 0.0767352764423077, |
| "25": 0.12766676682692307, |
| "26": 0.15681340144230768, |
| "27": 0.22130408653846154, |
| "28": 0.27283653846153844, |
| "29": 0.2716721754807692, |
| "30": 0.23375525841346154, |
| "31": 0.16575270432692307 |
| } |
| } |
