results/bert/bert_structural_prune.json

{
  "L1": {
    "keep_heads": [
      0,
      2,
      4,
      5,
      6,
      8,
      9,
      11
    ],
    "ffn_keep": 1319,
    "ffn_dead": 1753,
    "1_pruned_acc": 0.9208715596330275,
    "2_retrained_acc": 0.9197247706422018,
    "orig_params": 7087872,
    "final_params": 3606311,
    "reduction_pct": 49.119975642901,
    "saved": 3481561
  },
  "L2": {
    "keep_heads": [
      4,
      5,
      6,
      7,
      8,
      9,
      10,
      11
    ],
    "ffn_keep": 1533,
    "ffn_dead": 1539,
    "1_pruned_acc": 0.9220183486238532,
    "2_retrained_acc": 0.9208715596330275,
    "orig_params": 7087872,
    "final_params": 3935229,
    "reduction_pct": 44.479400869541664,
    "saved": 3152643
  },
  "L3": {
    "keep_heads": [
      0,
      1,
      3,
      4,
      5,
      7,
      8,
      11
    ],
    "ffn_keep": 1932,
    "ffn_dead": 1140,
    "1_pruned_acc": 0.9231651376146789,
    "2_retrained_acc": 0.9243119266055045,
    "orig_params": 7087872,
    "final_params": 4548492,
    "reduction_pct": 35.827114259399714,
    "saved": 2539380
  },
  "L4": {
    "keep_heads": [
      1,
      2,
      4,
      6,
      7,
      8,
      9,
      11
    ],
    "ffn_keep": 2301,
    "ffn_dead": 771,
    "1_pruned_acc": 0.9197247706422018,
    "2_retrained_acc": 0.9231651376146789,
    "orig_params": 7087872,
    "final_params": 5115645,
    "reduction_pct": 27.825375514681983,
    "saved": 1972227
  },
  "L5": {
    "keep_heads": [
      2,
      3,
      4,
      7,
      8,
      9,
      10,
      11
    ],
    "ffn_keep": 2255,
    "ffn_dead": 817,
    "1_pruned_acc": 0.9254587155963303,
    "2_retrained_acc": 0.9208715596330275,
    "orig_params": 7087872,
    "final_params": 5044943,
    "reduction_pct": 28.822882241665766,
    "saved": 2042929
  },
  "L6": {
    "keep_heads": [
      3,
      4,
      5,
      6,
      7,
      9,
      10,
      11
    ],
    "ffn_keep": 2380,
    "ffn_dead": 692,
    "1_pruned_acc": 0.9220183486238532,
    "2_retrained_acc": 0.9220183486238532,
    "orig_params": 7087872,
    "final_params": 5237068,
    "reduction_pct": 26.11226613573157,
    "saved": 1850804
  },
  "L7": {
    "keep_heads": [
      0,
      2,
      3,
      4,
      5,
      6,
      8,
      9
    ],
    "ffn_keep": 2478,
    "ffn_dead": 594,
    "1_pruned_acc": 0.9220183486238532,
    "2_retrained_acc": 0.9231651376146789,
    "orig_params": 7087872,
    "final_params": 5387694,
    "reduction_pct": 23.987143108679163,
    "saved": 1700178
  },
  "L8": {
    "keep_heads": [
      1,
      2,
      3,
      4,
      6,
      8,
      10,
      11
    ],
    "ffn_keep": 2715,
    "ffn_dead": 357,
    "1_pruned_acc": 0.9220183486238532,
    "2_retrained_acc": 0.9208715596330275,
    "orig_params": 7087872,
    "final_params": 5751963,
    "reduction_pct": 18.847814971827933,
    "saved": 1335909
  },
  "L9": {
    "keep_heads": [
      3,
      4,
      5,
      6,
      7,
      8,
      10,
      11
    ],
    "ffn_keep": 2647,
    "ffn_dead": 425,
    "1_pruned_acc": 0.9220183486238532,
    "2_retrained_acc": 0.9220183486238532,
    "orig_params": 7087872,
    "final_params": 5647447,
    "reduction_pct": 20.322390133456132,
    "saved": 1440425
  },
  "L10": {
    "keep_heads": [
      3,
      4,
      6,
      7,
      8,
      9,
      10,
      11
    ],
    "ffn_keep": 2627,
    "ffn_dead": 445,
    "1_pruned_acc": 0.9197247706422018,
    "2_retrained_acc": 0.9277522935779816,
    "orig_params": 7087872,
    "final_params": 5616707,
    "reduction_pct": 20.756088710405606,
    "saved": 1471165
  },
  "L11": {
    "keep_heads": [
      4,
      5,
      6,
      7,
      8,
      9,
      10,
      11
    ],
    "ffn_keep": 2594,
    "ffn_dead": 478,
    "1_pruned_acc": 0.9231651376146789,
    "2_retrained_acc": 0.9243119266055045,
    "orig_params": 7087872,
    "final_params": 5565986,
    "reduction_pct": 21.471691362372233,
    "saved": 1521886
  },
  "L12": {
    "keep_heads": [
      4,
      5,
      6,
      7,
      8,
      9,
      10,
      11
    ],
    "ffn_keep": 2818,
    "ffn_dead": 254,
    "1_pruned_acc": 0.9243119266055045,
    "2_retrained_acc": 0.9231651376146789,
    "orig_params": 7087872,
    "final_params": 5910274,
    "reduction_pct": 16.614267300538156,
    "saved": 1177598
  },
  "summary": {
    "orig_acc": 0.9243119266055045,
    "total_saved": 23686705
  }
}