File size: 5,716 Bytes
242d60a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 | {
"model": "Qwen/Qwen3-4B-Instruct-2507",
"ckpt": "/tmp/checkpoints_all36_d128_block/search_step_800.pt",
"by_K": {
"128": {
"raw_qk": {
"per_layer": {
"0": 0.9221310317516327,
"1": 0.9183070361614227,
"2": 0.9386252164840698,
"3": 0.9387637376785278,
"4": 0.9440614581108093,
"5": 0.96429443359375,
"6": 0.9560344219207764,
"7": 0.9818542897701263,
"8": 0.9707900285720825,
"9": 0.9587394297122955,
"10": 0.9742867350578308,
"11": 0.9763059914112091,
"12": 0.9611693322658539,
"13": 0.9707654714584351,
"14": 0.9733296930789948,
"15": 0.9675203263759613,
"16": 0.9556995928287506,
"17": 0.9593449831008911,
"18": 0.9652450382709503,
"19": 0.9612957835197449,
"20": 0.9591256380081177,
"21": 0.9657162129878998,
"22": 0.962736189365387,
"23": 0.9788879752159119,
"24": 0.9710511267185211,
"25": 0.9860879778862,
"26": 0.9775076508522034,
"27": 0.977691113948822,
"28": 0.9794201850891113,
"29": 0.9821525514125824,
"30": 0.9875719547271729,
"31": 0.9842455685138702,
"32": 0.9791028797626495,
"33": 0.9772593379020691,
"34": 0.9759000241756439,
"35": 0.9796911776065826
},
"avg": 0.9661864332026906
},
"learned": {
"per_layer": {
"0": 0.7756268978118896,
"1": 0.8533660173416138,
"2": 0.8985396027565002,
"3": 0.9248833358287811,
"4": 0.9358105957508087,
"5": 0.9497621357440948,
"6": 0.9386124014854431,
"7": 0.9828340709209442,
"8": 0.9707432389259338,
"9": 0.9764380753040314,
"10": 0.9706515073776245,
"11": 0.9758569598197937,
"12": 0.9699209034442902,
"13": 0.9717125296592712,
"14": 0.9731432497501373,
"15": 0.9723271131515503,
"16": 0.9620000123977661,
"17": 0.9667199552059174,
"18": 0.972541332244873,
"19": 0.9684668481349945,
"20": 0.9755434393882751,
"21": 0.9796880185604095,
"22": 0.9698434174060822,
"23": 0.9845690429210663,
"24": 0.9784398972988129,
"25": 0.98859041929245,
"26": 0.9856358468532562,
"27": 0.9832888841629028,
"28": 0.9848624765872955,
"29": 0.9874634146690369,
"30": 0.9862565696239471,
"31": 0.9839431643486023,
"32": 0.9798043966293335,
"33": 0.9695849716663361,
"34": 0.9600768983364105,
"35": 0.9647442996501923
},
"avg": 0.9603414427902963
}
},
"256": {
"raw_qk": {
"per_layer": {
"0": 0.9736533463001251,
"1": 0.9831015169620514,
"2": 0.985590249300003,
"3": 0.9856403172016144,
"4": 0.9863723814487457,
"5": 0.9927175343036652,
"6": 0.9899796843528748,
"7": 0.9961064755916595,
"8": 0.9944310188293457,
"9": 0.9924624264240265,
"10": 0.9945598244667053,
"11": 0.9956617951393127,
"12": 0.9914700388908386,
"13": 0.9947893917560577,
"14": 0.9956192076206207,
"15": 0.9947386384010315,
"16": 0.9918601512908936,
"17": 0.9932802319526672,
"18": 0.994801938533783,
"19": 0.9929636716842651,
"20": 0.9928486049175262,
"21": 0.9940518736839294,
"22": 0.9925838708877563,
"23": 0.9956205189228058,
"24": 0.9944405257701874,
"25": 0.99702388048172,
"26": 0.9963724911212921,
"27": 0.9952751994132996,
"28": 0.9952269494533539,
"29": 0.9970046579837799,
"30": 0.9976439476013184,
"31": 0.9967932999134064,
"32": 0.9949435591697693,
"33": 0.9954463541507721,
"34": 0.9949491024017334,
"35": 0.9950292408466339
},
"avg": 0.9929181643658214
},
"learned": {
"per_layer": {
"0": 0.9241782128810883,
"1": 0.9612297713756561,
"2": 0.9658167064189911,
"3": 0.9773095548152924,
"4": 0.9823991358280182,
"5": 0.9867597222328186,
"6": 0.9807394444942474,
"7": 0.9962039291858673,
"8": 0.9934562742710114,
"9": 0.9951532781124115,
"10": 0.9924466609954834,
"11": 0.9950800836086273,
"12": 0.992906391620636,
"13": 0.9934229552745819,
"14": 0.9937788546085358,
"15": 0.9944615662097931,
"16": 0.9920341670513153,
"17": 0.9936630129814148,
"18": 0.995515376329422,
"19": 0.993456095457077,
"20": 0.9959358274936676,
"21": 0.9966034889221191,
"22": 0.9939581751823425,
"23": 0.9973926544189453,
"24": 0.9962430894374847,
"25": 0.9975726306438446,
"26": 0.9972138106822968,
"27": 0.9966393709182739,
"28": 0.9968178272247314,
"29": 0.9975622594356537,
"30": 0.9967368543148041,
"31": 0.9965087473392487,
"32": 0.9954238533973694,
"33": 0.9919829368591309,
"34": 0.9898959100246429,
"35": 0.9893121719360352
},
"avg": 0.9896058556106355
}
}
},
"learned_over_raw_K128": 0.9939504528199392
} |