{
  "model": "Qwen/Qwen3-4B-Instruct-2507",
  "ckpt": "/tmp/checkpoints_packed_d64/search_step_1000.pt",
  "by_K": {
    "16": {
      "raw_qk": {
        "per_layer": {
          "4": 0.31979141881068546,
          "8": 0.3186202198266983,
          "12": 0.14174955089886984,
          "16": 0.20945234596729279,
          "20": 0.2801314915219943,
          "24": 0.49455158909161884
        },
        "avg": 0.2940494360195266
      },
      "learned": {
        "per_layer": {
          "4": 0.29329264909029007,
          "8": 0.28124504536390305,
          "12": 0.37904051691293716,
          "16": 0.2890646557013194,
          "20": 0.3339161326487859,
          "24": 0.5209685415029526
        },
        "avg": 0.34958792353669804
      }
    },
    "32": {
      "raw_qk": {
        "per_layer": {
          "4": 0.341281255086263,
          "8": 0.36785539239645004,
          "12": 0.21019610886772475,
          "16": 0.28323352212707203,
          "20": 0.3406280030806859,
          "24": 0.5262841482957205
        },
        "avg": 0.3449130716423194
      },
      "learned": {
        "per_layer": {
          "4": 0.31185250480969745,
          "8": 0.3099167247613271,
          "12": 0.41439520567655563,
          "16": 0.33257966736952466,
          "20": 0.3849489390850067,
          "24": 0.5501216500997543
        },
        "avg": 0.383969115300311
      }
    },
    "64": {
      "raw_qk": {
        "per_layer": {
          "4": 0.37279334167639416,
          "8": 0.4341067870457967,
          "12": 0.295872134466966,
          "16": 0.37213313827912015,
          "20": 0.4122797225912412,
          "24": 0.5643380333979925
        },
        "avg": 0.40858719290958506
      },
      "learned": {
        "per_layer": {
          "4": 0.3371334026257197,
          "8": 0.3503660187125206,
          "12": 0.4602071891228358,
          "16": 0.3917141556739807,
          "20": 0.45263657718896866,
          "24": 0.5895731498797735
        },
        "avg": 0.43027174886729985
      }
    },
    "128": {
      "raw_qk": {
        "per_layer": {
          "4": 0.4215788046518962,
          "8": 0.5176494866609573,
          "12": 0.4037476380666097,
          "16": 0.47523776690165204,
          "20": 0.49859579155842465,
          "24": 0.6136594414710999
        },
        "avg": 0.48841148821844
      },
      "learned": {
        "per_layer": {
          "4": 0.37442514797051746,
          "8": 0.40738723675409955,
          "12": 0.5206492592891058,
          "16": 0.4700211783250173,
          "20": 0.5400509238243103,
          "24": 0.6421113312244415
        },
        "avg": 0.4924408462312486
      }
    },
    "256": {
      "raw_qk": {
        "per_layer": {
          "4": 0.4964489738146464,
          "8": 0.6146760632594427,
          "12": 0.5339744637409846,
          "16": 0.5874835352102915,
          "20": 0.602066790064176,
          "24": 0.6779818137486776
        },
        "avg": 0.5854386066397032
      },
      "learned": {
        "per_layer": {
          "4": 0.4301423355937004,
          "8": 0.48568252722422284,
          "12": 0.599395309885343,
          "16": 0.5693490405877432,
          "20": 0.6461250483989716,
          "24": 0.709612175822258
        },
        "avg": 0.5733844062520398
      }
    }
  },
  "learned_over_raw_K128": 1.0082499247253711
}