{
  "model": "Qwen/Qwen3-4B-Instruct-2507",
  "ckpt": "/tmp/checkpoints_packed_d128/search_step_1000.pt",
  "by_K": {
    "16": {
      "raw_qk": {
        "per_layer": {
          "4": 0.31979141881068546,
          "8": 0.3186202198266983,
          "12": 0.14174955089886984,
          "16": 0.20945234596729279,
          "20": 0.2801314915219943,
          "24": 0.49455158909161884
        },
        "avg": 0.2940494360195266
      },
      "learned": {
        "per_layer": {
          "4": 0.29460810621579486,
          "8": 0.2879024048646291,
          "12": 0.38745074967543286,
          "16": 0.2947010373075803,
          "20": 0.3400069276491801,
          "24": 0.5236761594812075
        },
        "avg": 0.3547242308656375
      }
    },
    "32": {
      "raw_qk": {
        "per_layer": {
          "4": 0.341281255086263,
          "8": 0.36785539239645004,
          "12": 0.21019610886772475,
          "16": 0.28323352212707203,
          "20": 0.3406280030806859,
          "24": 0.5262841482957205
        },
        "avg": 0.3449130716423194
      },
      "learned": {
        "per_layer": {
          "4": 0.31440528233846027,
          "8": 0.318613442281882,
          "12": 0.42397742718458176,
          "16": 0.33971526473760605,
          "20": 0.3927851542830467,
          "24": 0.5538897663354874
        },
        "avg": 0.390564389526844
      }
    },
    "64": {
      "raw_qk": {
        "per_layer": {
          "4": 0.37279334167639416,
          "8": 0.4341067870457967,
          "12": 0.295872134466966,
          "16": 0.37213313827912015,
          "20": 0.4122797225912412,
          "24": 0.5643380333979925
        },
        "avg": 0.40858719290958506
      },
      "learned": {
        "per_layer": {
          "4": 0.34206587572892505,
          "8": 0.3614834249019623,
          "12": 0.47101640701293945,
          "16": 0.40079283465941745,
          "20": 0.46224164466063183,
          "24": 0.5945162226756414
        },
        "avg": 0.4386860682732529
      }
    },
    "128": {
      "raw_qk": {
        "per_layer": {
          "4": 0.4215788046518962,
          "8": 0.5176494866609573,
          "12": 0.4037476380666097,
          "16": 0.47523776690165204,
          "20": 0.49859579155842465,
          "24": 0.6136594414710999
        },
        "avg": 0.48841148821844
      },
      "learned": {
        "per_layer": {
          "4": 0.38235953201850253,
          "8": 0.4212384819984436,
          "12": 0.5328463464975357,
          "16": 0.4814741685986519,
          "20": 0.551252673069636,
          "24": 0.6483117590347925
        },
        "avg": 0.5029138268695937
      }
    },
    "256": {
      "raw_qk": {
        "per_layer": {
          "4": 0.4964489738146464,
          "8": 0.6146760632594427,
          "12": 0.5339744637409846,
          "16": 0.5874835352102915,
          "20": 0.602066790064176,
          "24": 0.6779818137486776
        },
        "avg": 0.5854386066397032
      },
      "learned": {
        "per_layer": {
          "4": 0.4403117299079895,
          "8": 0.5022278105219206,
          "12": 0.6128821273644766,
          "16": 0.582685723900795,
          "20": 0.65766608218352,
          "24": 0.716396709283193
        },
        "avg": 0.5853616971936492
      }
    }
  },
  "learned_over_raw_K128": 1.0296928696416485
}