| { |
| "model": "Qwen/Qwen3-4B-Instruct-2507", |
| "ckpt": "/tmp/checkpoints_packed_d64/search_step_1000.pt", |
| "by_K": { |
| "16": { |
| "raw_qk": { |
| "per_layer": { |
| "4": 0.31979141881068546, |
| "8": 0.3186202198266983, |
| "12": 0.14174955089886984, |
| "16": 0.20945234596729279, |
| "20": 0.2801314915219943, |
| "24": 0.49455158909161884 |
| }, |
| "avg": 0.2940494360195266 |
| }, |
| "learned": { |
| "per_layer": { |
| "4": 0.29329264909029007, |
| "8": 0.28124504536390305, |
| "12": 0.37904051691293716, |
| "16": 0.2890646557013194, |
| "20": 0.3339161326487859, |
| "24": 0.5209685415029526 |
| }, |
| "avg": 0.34958792353669804 |
| } |
| }, |
| "32": { |
| "raw_qk": { |
| "per_layer": { |
| "4": 0.341281255086263, |
| "8": 0.36785539239645004, |
| "12": 0.21019610886772475, |
| "16": 0.28323352212707203, |
| "20": 0.3406280030806859, |
| "24": 0.5262841482957205 |
| }, |
| "avg": 0.3449130716423194 |
| }, |
| "learned": { |
| "per_layer": { |
| "4": 0.31185250480969745, |
| "8": 0.3099167247613271, |
| "12": 0.41439520567655563, |
| "16": 0.33257966736952466, |
| "20": 0.3849489390850067, |
| "24": 0.5501216500997543 |
| }, |
| "avg": 0.383969115300311 |
| } |
| }, |
| "64": { |
| "raw_qk": { |
| "per_layer": { |
| "4": 0.37279334167639416, |
| "8": 0.4341067870457967, |
| "12": 0.295872134466966, |
| "16": 0.37213313827912015, |
| "20": 0.4122797225912412, |
| "24": 0.5643380333979925 |
| }, |
| "avg": 0.40858719290958506 |
| }, |
| "learned": { |
| "per_layer": { |
| "4": 0.3371334026257197, |
| "8": 0.3503660187125206, |
| "12": 0.4602071891228358, |
| "16": 0.3917141556739807, |
| "20": 0.45263657718896866, |
| "24": 0.5895731498797735 |
| }, |
| "avg": 0.43027174886729985 |
| } |
| }, |
| "128": { |
| "raw_qk": { |
| "per_layer": { |
| "4": 0.4215788046518962, |
| "8": 0.5176494866609573, |
| "12": 0.4037476380666097, |
| "16": 0.47523776690165204, |
| "20": 0.49859579155842465, |
| "24": 0.6136594414710999 |
| }, |
| "avg": 0.48841148821844 |
| }, |
| "learned": { |
| "per_layer": { |
| "4": 0.37442514797051746, |
| "8": 0.40738723675409955, |
| "12": 0.5206492592891058, |
| "16": 0.4700211783250173, |
| "20": 0.5400509238243103, |
| "24": 0.6421113312244415 |
| }, |
| "avg": 0.4924408462312486 |
| } |
| }, |
| "256": { |
| "raw_qk": { |
| "per_layer": { |
| "4": 0.4964489738146464, |
| "8": 0.6146760632594427, |
| "12": 0.5339744637409846, |
| "16": 0.5874835352102915, |
| "20": 0.602066790064176, |
| "24": 0.6779818137486776 |
| }, |
| "avg": 0.5854386066397032 |
| }, |
| "learned": { |
| "per_layer": { |
| "4": 0.4301423355937004, |
| "8": 0.48568252722422284, |
| "12": 0.599395309885343, |
| "16": 0.5693490405877432, |
| "20": 0.6461250483989716, |
| "24": 0.709612175822258 |
| }, |
| "avg": 0.5733844062520398 |
| } |
| } |
| }, |
| "learned_over_raw_K128": 1.0082499247253711 |
| } |
