| { |
| "model": "Qwen/Qwen3-4B-Instruct-2507", |
| "ckpt": "/tmp/checkpoints_packed_d256/search_step_1000.pt", |
| "by_K": { |
| "16": { |
| "raw_qk": { |
| "per_layer": { |
| "4": 0.31979141881068546, |
| "8": 0.3186202198266983, |
| "12": 0.14174955089886984, |
| "16": 0.20945234596729279, |
| "20": 0.2801314915219943, |
| "24": 0.49455158909161884 |
| }, |
| "avg": 0.2940494360195266 |
| }, |
| "learned": { |
| "per_layer": { |
| "4": 0.2933313399553299, |
| "8": 0.29298429439465207, |
| "12": 0.39142733812332153, |
| "16": 0.2983766943216324, |
| "20": 0.3438563868403435, |
| "24": 0.525188535451889 |
| }, |
| "avg": 0.35752743151452804 |
| } |
| }, |
| "32": { |
| "raw_qk": { |
| "per_layer": { |
| "4": 0.341281255086263, |
| "8": 0.36785539239645004, |
| "12": 0.21019610886772475, |
| "16": 0.28323352212707203, |
| "20": 0.3406280030806859, |
| "24": 0.5262841482957205 |
| }, |
| "avg": 0.3449130716423194 |
| }, |
| "learned": { |
| "per_layer": { |
| "4": 0.3145047202706337, |
| "8": 0.3253612567981084, |
| "12": 0.4279355009396871, |
| "16": 0.344377006093661, |
| "20": 0.3977118283510208, |
| "24": 0.556083157658577 |
| }, |
| "avg": 0.39432891168528134 |
| } |
| }, |
| "64": { |
| "raw_qk": { |
| "per_layer": { |
| "4": 0.37279334167639416, |
| "8": 0.4341067870457967, |
| "12": 0.295872134466966, |
| "16": 0.37213313827912015, |
| "20": 0.4122797225912412, |
| "24": 0.5643380333979925 |
| }, |
| "avg": 0.40858719290958506 |
| }, |
| "learned": { |
| "per_layer": { |
| "4": 0.34352220843235654, |
| "8": 0.37063097457091015, |
| "12": 0.4752006282409032, |
| "16": 0.4067305897672971, |
| "20": 0.46815096338589984, |
| "24": 0.5977186262607574 |
| }, |
| "avg": 0.44365899844302076 |
| } |
| }, |
| "128": { |
| "raw_qk": { |
| "per_layer": { |
| "4": 0.4215788046518962, |
| "8": 0.5176494866609573, |
| "12": 0.4037476380666097, |
| "16": 0.47523776690165204, |
| "20": 0.49859579155842465, |
| "24": 0.6136594414710999 |
| }, |
| "avg": 0.48841148821844 |
| }, |
| "learned": { |
| "per_layer": { |
| "4": 0.3849342664082845, |
| "8": 0.43360541264216107, |
| "12": 0.5375104447205862, |
| "16": 0.488710917532444, |
| "20": 0.5576231330633163, |
| "24": 0.6525773753722509 |
| }, |
| "avg": 0.5091602582898406 |
| } |
| }, |
| "256": { |
| "raw_qk": { |
| "per_layer": { |
| "4": 0.4964489738146464, |
| "8": 0.6146760632594427, |
| "12": 0.5339744637409846, |
| "16": 0.5874835352102915, |
| "20": 0.602066790064176, |
| "24": 0.6779818137486776 |
| }, |
| "avg": 0.5854386066397032 |
| }, |
| "learned": { |
| "per_layer": { |
| "4": 0.44419410079717636, |
| "8": 0.5183951507012049, |
| "12": 0.6179872999588648, |
| "16": 0.5908408363660177, |
| "20": 0.6638876696427664, |
| "24": 0.7216567993164062 |
| }, |
| "avg": 0.592826976130406 |
| } |
| } |
| }, |
| "learned_over_raw_K128": 1.0424821499328059 |
| } |
