{
  "model": "Qwen/Qwen3-4B-Instruct-2507",
  "ckpt": "/tmp/checkpoints_packed_d256/search_step_1000.pt",
  "by_K": {
    "16": {
      "raw_qk": {
        "per_layer": {
          "4": 0.31979141881068546,
          "8": 0.3186202198266983,
          "12": 0.14174955089886984,
          "16": 0.20945234596729279,
          "20": 0.2801314915219943,
          "24": 0.49455158909161884
        },
        "avg": 0.2940494360195266
      },
      "learned": {
        "per_layer": {
          "4": 0.2933313399553299,
          "8": 0.29298429439465207,
          "12": 0.39142733812332153,
          "16": 0.2983766943216324,
          "20": 0.3438563868403435,
          "24": 0.525188535451889
        },
        "avg": 0.35752743151452804
      }
    },
    "32": {
      "raw_qk": {
        "per_layer": {
          "4": 0.341281255086263,
          "8": 0.36785539239645004,
          "12": 0.21019610886772475,
          "16": 0.28323352212707203,
          "20": 0.3406280030806859,
          "24": 0.5262841482957205
        },
        "avg": 0.3449130716423194
      },
      "learned": {
        "per_layer": {
          "4": 0.3145047202706337,
          "8": 0.3253612567981084,
          "12": 0.4279355009396871,
          "16": 0.344377006093661,
          "20": 0.3977118283510208,
          "24": 0.556083157658577
        },
        "avg": 0.39432891168528134
      }
    },
    "64": {
      "raw_qk": {
        "per_layer": {
          "4": 0.37279334167639416,
          "8": 0.4341067870457967,
          "12": 0.295872134466966,
          "16": 0.37213313827912015,
          "20": 0.4122797225912412,
          "24": 0.5643380333979925
        },
        "avg": 0.40858719290958506
      },
      "learned": {
        "per_layer": {
          "4": 0.34352220843235654,
          "8": 0.37063097457091015,
          "12": 0.4752006282409032,
          "16": 0.4067305897672971,
          "20": 0.46815096338589984,
          "24": 0.5977186262607574
        },
        "avg": 0.44365899844302076
      }
    },
    "128": {
      "raw_qk": {
        "per_layer": {
          "4": 0.4215788046518962,
          "8": 0.5176494866609573,
          "12": 0.4037476380666097,
          "16": 0.47523776690165204,
          "20": 0.49859579155842465,
          "24": 0.6136594414710999
        },
        "avg": 0.48841148821844
      },
      "learned": {
        "per_layer": {
          "4": 0.3849342664082845,
          "8": 0.43360541264216107,
          "12": 0.5375104447205862,
          "16": 0.488710917532444,
          "20": 0.5576231330633163,
          "24": 0.6525773753722509
        },
        "avg": 0.5091602582898406
      }
    },
    "256": {
      "raw_qk": {
        "per_layer": {
          "4": 0.4964489738146464,
          "8": 0.6146760632594427,
          "12": 0.5339744637409846,
          "16": 0.5874835352102915,
          "20": 0.602066790064176,
          "24": 0.6779818137486776
        },
        "avg": 0.5854386066397032
      },
      "learned": {
        "per_layer": {
          "4": 0.44419410079717636,
          "8": 0.5183951507012049,
          "12": 0.6179872999588648,
          "16": 0.5908408363660177,
          "20": 0.6638876696427664,
          "24": 0.7216567993164062
        },
        "avg": 0.592826976130406
      }
    }
  },
  "learned_over_raw_K128": 1.0424821499328059
}