| { |
| "model": "Qwen/Qwen3-4B-Instruct-2507", |
| "ckpt": "/tmp/checkpoints_all36_d128_block/search_step_500.pt", |
| "by_K": { |
| "128": { |
| "raw_qk": { |
| "per_layer": { |
| "0": 0.9221310317516327, |
| "1": 0.9183070361614227, |
| "2": 0.9386252164840698, |
| "3": 0.9387637376785278, |
| "4": 0.9440614581108093, |
| "5": 0.96429443359375, |
| "6": 0.9560344219207764, |
| "7": 0.9818542897701263, |
| "8": 0.9707900285720825, |
| "9": 0.9587394297122955, |
| "10": 0.9742867350578308, |
| "11": 0.9763059914112091, |
| "12": 0.9611693322658539, |
| "13": 0.9707654714584351, |
| "14": 0.9733296930789948, |
| "15": 0.9675203263759613, |
| "16": 0.9556995928287506, |
| "17": 0.9593449831008911, |
| "18": 0.9652450382709503, |
| "19": 0.9612957835197449, |
| "20": 0.9591256380081177, |
| "21": 0.9657162129878998, |
| "22": 0.962736189365387, |
| "23": 0.9788879752159119, |
| "24": 0.9710511267185211, |
| "25": 0.9860879778862, |
| "26": 0.9775076508522034, |
| "27": 0.977691113948822, |
| "28": 0.9794201850891113, |
| "29": 0.9821525514125824, |
| "30": 0.9875719547271729, |
| "31": 0.9842455685138702, |
| "32": 0.9791028797626495, |
| "33": 0.9772593379020691, |
| "34": 0.9759000241756439, |
| "35": 0.9796911776065826 |
| }, |
| "avg": 0.9661864332026906 |
| }, |
| "learned": { |
| "per_layer": { |
| "0": 0.7799814641475677, |
| "1": 0.8509384989738464, |
| "2": 0.898685872554779, |
| "3": 0.923816055059433, |
| "4": 0.9334639310836792, |
| "5": 0.9474231600761414, |
| "6": 0.9364679157733917, |
| "7": 0.9821601510047913, |
| "8": 0.9702183306217194, |
| "9": 0.9760044515132904, |
| "10": 0.9704114198684692, |
| "11": 0.9754650890827179, |
| "12": 0.9693757593631744, |
| "13": 0.9711220562458038, |
| "14": 0.9729296863079071, |
| "15": 0.971889466047287, |
| "16": 0.9616761803627014, |
| "17": 0.9660459458827972, |
| "18": 0.972119152545929, |
| "19": 0.9680445790290833, |
| "20": 0.9750770926475525, |
| "21": 0.9793232381343842, |
| "22": 0.9695925712585449, |
| "23": 0.9843278527259827, |
| "24": 0.9780710339546204, |
| "25": 0.9884319007396698, |
| "26": 0.9854008555412292, |
| "27": 0.9830437302589417, |
| "28": 0.9846695363521576, |
| "29": 0.9871246516704559, |
| "30": 0.9859895706176758, |
| "31": 0.9835683107376099, |
| "32": 0.979343831539154, |
| "33": 0.9698410034179688, |
| "34": 0.9603682458400726, |
| "35": 0.9667042791843414 |
| }, |
| "avg": 0.9599754686156908 |
| } |
| }, |
| "256": { |
| "raw_qk": { |
| "per_layer": { |
| "0": 0.9736533463001251, |
| "1": 0.9831015169620514, |
| "2": 0.985590249300003, |
| "3": 0.9856403172016144, |
| "4": 0.9863723814487457, |
| "5": 0.9927175343036652, |
| "6": 0.9899796843528748, |
| "7": 0.9961064755916595, |
| "8": 0.9944310188293457, |
| "9": 0.9924624264240265, |
| "10": 0.9945598244667053, |
| "11": 0.9956617951393127, |
| "12": 0.9914700388908386, |
| "13": 0.9947893917560577, |
| "14": 0.9956192076206207, |
| "15": 0.9947386384010315, |
| "16": 0.9918601512908936, |
| "17": 0.9932802319526672, |
| "18": 0.994801938533783, |
| "19": 0.9929636716842651, |
| "20": 0.9928486049175262, |
| "21": 0.9940518736839294, |
| "22": 0.9925838708877563, |
| "23": 0.9956205189228058, |
| "24": 0.9944405257701874, |
| "25": 0.99702388048172, |
| "26": 0.9963724911212921, |
| "27": 0.9952751994132996, |
| "28": 0.9952269494533539, |
| "29": 0.9970046579837799, |
| "30": 0.9976439476013184, |
| "31": 0.9967932999134064, |
| "32": 0.9949435591697693, |
| "33": 0.9954463541507721, |
| "34": 0.9949491024017334, |
| "35": 0.9950292408466339 |
| }, |
| "avg": 0.9929181643658214 |
| }, |
| "learned": { |
| "per_layer": { |
| "0": 0.9298504292964935, |
| "1": 0.9594626426696777, |
| "2": 0.9668116569519043, |
| "3": 0.97734135389328, |
| "4": 0.9817334413528442, |
| "5": 0.9862845540046692, |
| "6": 0.9803015291690826, |
| "7": 0.9960329234600067, |
| "8": 0.9933978915214539, |
| "9": 0.9950797855854034, |
| "10": 0.9924003481864929, |
| "11": 0.9950767755508423, |
| "12": 0.9927493929862976, |
| "13": 0.9933296144008636, |
| "14": 0.9937319457530975, |
| "15": 0.9942990839481354, |
| "16": 0.9920076131820679, |
| "17": 0.9934728741645813, |
| "18": 0.9954265654087067, |
| "19": 0.9932436048984528, |
| "20": 0.9957763850688934, |
| "21": 0.9964823722839355, |
| "22": 0.993859052658081, |
| "23": 0.9973229467868805, |
| "24": 0.9960962235927582, |
| "25": 0.9975778460502625, |
| "26": 0.997130274772644, |
| "27": 0.9965287148952484, |
| "28": 0.9967555701732635, |
| "29": 0.9974750280380249, |
| "30": 0.996643602848053, |
| "31": 0.9963894486427307, |
| "32": 0.9952475130558014, |
| "33": 0.9919288456439972, |
| "34": 0.9904468953609467, |
| "35": 0.9905306994915009 |
| }, |
| "avg": 0.9896729290485382 |
| } |
| } |
| }, |
| "learned_over_raw_K128": 0.9935716706697983 |
| } |
