stage_2b/head_config.json

{
  "per_block_num_heads": [
    12,
    12,
    12,
    11,
    11,
    12,
    11,
    11,
    12,
    9,
    11,
    10
  ],
  "per_block_kept_heads": {
    "0": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      8,
      9,
      10,
      11
    ],
    "1": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      8,
      9,
      10,
      11
    ],
    "2": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      8,
      9,
      10,
      11
    ],
    "3": [
      0,
      1,
      2,
      3,
      4,
      6,
      7,
      8,
      9,
      10,
      11
    ],
    "4": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      9,
      10,
      11
    ],
    "5": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      8,
      9,
      10,
      11
    ],
    "6": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      8,
      10,
      11
    ],
    "7": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      8,
      9,
      10
    ],
    "8": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      8,
      9,
      10,
      11
    ],
    "9": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      8
    ],
    "10": [
      0,
      1,
      2,
      3,
      5,
      6,
      7,
      8,
      9,
      10,
      11
    ],
    "11": [
      0,
      2,
      3,
      4,
      5,
      6,
      7,
      8,
      10,
      11
    ]
  },
  "per_block_removed_heads": {
    "0": [],
    "1": [],
    "2": [],
    "3": [
      5
    ],
    "4": [
      8
    ],
    "5": [],
    "6": [
      9
    ],
    "7": [
      11
    ],
    "8": [],
    "9": [
      11,
      10,
      9
    ],
    "10": [
      4
    ],
    "11": [
      1,
      9
    ]
  },
  "head_dim": 64,
  "dim": 768
}