| { |
| "model": "Qwen/Qwen3-4B-Instruct-2507", |
| "ckpt": "/tmp/checkpoints_all36_d128_block/search_step_800.pt", |
| "by_K": { |
| "128": { |
| "raw_qk": { |
| "per_layer": { |
| "0": 0.9221310317516327, |
| "1": 0.9183070361614227, |
| "2": 0.9386252164840698, |
| "3": 0.9387637376785278, |
| "4": 0.9440614581108093, |
| "5": 0.96429443359375, |
| "6": 0.9560344219207764, |
| "7": 0.9818542897701263, |
| "8": 0.9707900285720825, |
| "9": 0.9587394297122955, |
| "10": 0.9742867350578308, |
| "11": 0.9763059914112091, |
| "12": 0.9611693322658539, |
| "13": 0.9707654714584351, |
| "14": 0.9733296930789948, |
| "15": 0.9675203263759613, |
| "16": 0.9556995928287506, |
| "17": 0.9593449831008911, |
| "18": 0.9652450382709503, |
| "19": 0.9612957835197449, |
| "20": 0.9591256380081177, |
| "21": 0.9657162129878998, |
| "22": 0.962736189365387, |
| "23": 0.9788879752159119, |
| "24": 0.9710511267185211, |
| "25": 0.9860879778862, |
| "26": 0.9775076508522034, |
| "27": 0.977691113948822, |
| "28": 0.9794201850891113, |
| "29": 0.9821525514125824, |
| "30": 0.9875719547271729, |
| "31": 0.9842455685138702, |
| "32": 0.9791028797626495, |
| "33": 0.9772593379020691, |
| "34": 0.9759000241756439, |
| "35": 0.9796911776065826 |
| }, |
| "avg": 0.9661864332026906 |
| }, |
| "learned": { |
| "per_layer": { |
| "0": 0.7756268978118896, |
| "1": 0.8533660173416138, |
| "2": 0.8985396027565002, |
| "3": 0.9248833358287811, |
| "4": 0.9358105957508087, |
| "5": 0.9497621357440948, |
| "6": 0.9386124014854431, |
| "7": 0.9828340709209442, |
| "8": 0.9707432389259338, |
| "9": 0.9764380753040314, |
| "10": 0.9706515073776245, |
| "11": 0.9758569598197937, |
| "12": 0.9699209034442902, |
| "13": 0.9717125296592712, |
| "14": 0.9731432497501373, |
| "15": 0.9723271131515503, |
| "16": 0.9620000123977661, |
| "17": 0.9667199552059174, |
| "18": 0.972541332244873, |
| "19": 0.9684668481349945, |
| "20": 0.9755434393882751, |
| "21": 0.9796880185604095, |
| "22": 0.9698434174060822, |
| "23": 0.9845690429210663, |
| "24": 0.9784398972988129, |
| "25": 0.98859041929245, |
| "26": 0.9856358468532562, |
| "27": 0.9832888841629028, |
| "28": 0.9848624765872955, |
| "29": 0.9874634146690369, |
| "30": 0.9862565696239471, |
| "31": 0.9839431643486023, |
| "32": 0.9798043966293335, |
| "33": 0.9695849716663361, |
| "34": 0.9600768983364105, |
| "35": 0.9647442996501923 |
| }, |
| "avg": 0.9603414427902963 |
| } |
| }, |
| "256": { |
| "raw_qk": { |
| "per_layer": { |
| "0": 0.9736533463001251, |
| "1": 0.9831015169620514, |
| "2": 0.985590249300003, |
| "3": 0.9856403172016144, |
| "4": 0.9863723814487457, |
| "5": 0.9927175343036652, |
| "6": 0.9899796843528748, |
| "7": 0.9961064755916595, |
| "8": 0.9944310188293457, |
| "9": 0.9924624264240265, |
| "10": 0.9945598244667053, |
| "11": 0.9956617951393127, |
| "12": 0.9914700388908386, |
| "13": 0.9947893917560577, |
| "14": 0.9956192076206207, |
| "15": 0.9947386384010315, |
| "16": 0.9918601512908936, |
| "17": 0.9932802319526672, |
| "18": 0.994801938533783, |
| "19": 0.9929636716842651, |
| "20": 0.9928486049175262, |
| "21": 0.9940518736839294, |
| "22": 0.9925838708877563, |
| "23": 0.9956205189228058, |
| "24": 0.9944405257701874, |
| "25": 0.99702388048172, |
| "26": 0.9963724911212921, |
| "27": 0.9952751994132996, |
| "28": 0.9952269494533539, |
| "29": 0.9970046579837799, |
| "30": 0.9976439476013184, |
| "31": 0.9967932999134064, |
| "32": 0.9949435591697693, |
| "33": 0.9954463541507721, |
| "34": 0.9949491024017334, |
| "35": 0.9950292408466339 |
| }, |
| "avg": 0.9929181643658214 |
| }, |
| "learned": { |
| "per_layer": { |
| "0": 0.9241782128810883, |
| "1": 0.9612297713756561, |
| "2": 0.9658167064189911, |
| "3": 0.9773095548152924, |
| "4": 0.9823991358280182, |
| "5": 0.9867597222328186, |
| "6": 0.9807394444942474, |
| "7": 0.9962039291858673, |
| "8": 0.9934562742710114, |
| "9": 0.9951532781124115, |
| "10": 0.9924466609954834, |
| "11": 0.9950800836086273, |
| "12": 0.992906391620636, |
| "13": 0.9934229552745819, |
| "14": 0.9937788546085358, |
| "15": 0.9944615662097931, |
| "16": 0.9920341670513153, |
| "17": 0.9936630129814148, |
| "18": 0.995515376329422, |
| "19": 0.993456095457077, |
| "20": 0.9959358274936676, |
| "21": 0.9966034889221191, |
| "22": 0.9939581751823425, |
| "23": 0.9973926544189453, |
| "24": 0.9962430894374847, |
| "25": 0.9975726306438446, |
| "26": 0.9972138106822968, |
| "27": 0.9966393709182739, |
| "28": 0.9968178272247314, |
| "29": 0.9975622594356537, |
| "30": 0.9967368543148041, |
| "31": 0.9965087473392487, |
| "32": 0.9954238533973694, |
| "33": 0.9919829368591309, |
| "34": 0.9898959100246429, |
| "35": 0.9893121719360352 |
| }, |
| "avg": 0.9896058556106355 |
| } |
| } |
| }, |
| "learned_over_raw_K128": 0.9939504528199392 |
| } |
