Hugging Face's logo

datasysdev
/
ann-sparseattention

Transformers
GGUF
sparse-attention
approximate-nearest-neighbors
faiss
qwen3
long-context
conversational
Model card Files
xet
 Community
datasysdev commited on
Commit
cb2df3b
·
verified ·
1 Parent(s): 96b4858

Upload checkpoints_all36_d128_block/search_step_500.compare_retrieval.json

Browse files
Files changed (1) hide show
  1. checkpoints_all36_d128_block/search_step_500.compare_retrieval.json +175 -0
checkpoints_all36_d128_block/search_step_500.compare_retrieval.json ADDED
@@ -0,0 +1,175 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "Qwen/Qwen3-4B-Instruct-2507",
3
+ "ckpt": "/tmp/checkpoints_all36_d128_block/search_step_500.pt",
4
+ "by_K": {
5
+ "128": {
6
+ "raw_qk": {
7
+ "per_layer": {
8
+ "0": 0.9221310317516327,
9
+ "1": 0.9183070361614227,
10
+ "2": 0.9386252164840698,
11
+ "3": 0.9387637376785278,
12
+ "4": 0.9440614581108093,
13
+ "5": 0.96429443359375,
14
+ "6": 0.9560344219207764,
15
+ "7": 0.9818542897701263,
16
+ "8": 0.9707900285720825,
17
+ "9": 0.9587394297122955,
18
+ "10": 0.9742867350578308,
19
+ "11": 0.9763059914112091,
20
+ "12": 0.9611693322658539,
21
+ "13": 0.9707654714584351,
22
+ "14": 0.9733296930789948,
23
+ "15": 0.9675203263759613,
24
+ "16": 0.9556995928287506,
25
+ "17": 0.9593449831008911,
26
+ "18": 0.9652450382709503,
27
+ "19": 0.9612957835197449,
28
+ "20": 0.9591256380081177,
29
+ "21": 0.9657162129878998,
30
+ "22": 0.962736189365387,
31
+ "23": 0.9788879752159119,
32
+ "24": 0.9710511267185211,
33
+ "25": 0.9860879778862,
34
+ "26": 0.9775076508522034,
35
+ "27": 0.977691113948822,
36
+ "28": 0.9794201850891113,
37
+ "29": 0.9821525514125824,
38
+ "30": 0.9875719547271729,
39
+ "31": 0.9842455685138702,
40
+ "32": 0.9791028797626495,
41
+ "33": 0.9772593379020691,
42
+ "34": 0.9759000241756439,
43
+ "35": 0.9796911776065826
44
+ },
45
+ "avg": 0.9661864332026906
46
+ },
47
+ "learned": {
48
+ "per_layer": {
49
+ "0": 0.7799814641475677,
50
+ "1": 0.8509384989738464,
51
+ "2": 0.898685872554779,
52
+ "3": 0.923816055059433,
53
+ "4": 0.9334639310836792,
54
+ "5": 0.9474231600761414,
55
+ "6": 0.9364679157733917,
56
+ "7": 0.9821601510047913,
57
+ "8": 0.9702183306217194,
58
+ "9": 0.9760044515132904,
59
+ "10": 0.9704114198684692,
60
+ "11": 0.9754650890827179,
61
+ "12": 0.9693757593631744,
62
+ "13": 0.9711220562458038,
63
+ "14": 0.9729296863079071,
64
+ "15": 0.971889466047287,
65
+ "16": 0.9616761803627014,
66
+ "17": 0.9660459458827972,
67
+ "18": 0.972119152545929,
68
+ "19": 0.9680445790290833,
69
+ "20": 0.9750770926475525,
70
+ "21": 0.9793232381343842,
71
+ "22": 0.9695925712585449,
72
+ "23": 0.9843278527259827,
73
+ "24": 0.9780710339546204,
74
+ "25": 0.9884319007396698,
75
+ "26": 0.9854008555412292,
76
+ "27": 0.9830437302589417,
77
+ "28": 0.9846695363521576,
78
+ "29": 0.9871246516704559,
79
+ "30": 0.9859895706176758,
80
+ "31": 0.9835683107376099,
81
+ "32": 0.979343831539154,
82
+ "33": 0.9698410034179688,
83
+ "34": 0.9603682458400726,
84
+ "35": 0.9667042791843414
85
+ },
86
+ "avg": 0.9599754686156908
87
+ }
88
+ },
89
+ "256": {
90
+ "raw_qk": {
91
+ "per_layer": {
92
+ "0": 0.9736533463001251,
93
+ "1": 0.9831015169620514,
94
+ "2": 0.985590249300003,
95
+ "3": 0.9856403172016144,
96
+ "4": 0.9863723814487457,
97
+ "5": 0.9927175343036652,
98
+ "6": 0.9899796843528748,
99
+ "7": 0.9961064755916595,
100
+ "8": 0.9944310188293457,
101
+ "9": 0.9924624264240265,
102
+ "10": 0.9945598244667053,
103
+ "11": 0.9956617951393127,
104
+ "12": 0.9914700388908386,
105
+ "13": 0.9947893917560577,
106
+ "14": 0.9956192076206207,
107
+ "15": 0.9947386384010315,
108
+ "16": 0.9918601512908936,
109
+ "17": 0.9932802319526672,
110
+ "18": 0.994801938533783,
111
+ "19": 0.9929636716842651,
112
+ "20": 0.9928486049175262,
113
+ "21": 0.9940518736839294,
114
+ "22": 0.9925838708877563,
115
+ "23": 0.9956205189228058,
116
+ "24": 0.9944405257701874,
117
+ "25": 0.99702388048172,
118
+ "26": 0.9963724911212921,
119
+ "27": 0.9952751994132996,
120
+ "28": 0.9952269494533539,
121
+ "29": 0.9970046579837799,
122
+ "30": 0.9976439476013184,
123
+ "31": 0.9967932999134064,
124
+ "32": 0.9949435591697693,
125
+ "33": 0.9954463541507721,
126
+ "34": 0.9949491024017334,
127
+ "35": 0.9950292408466339
128
+ },
129
+ "avg": 0.9929181643658214
130
+ },
131
+ "learned": {
132
+ "per_layer": {
133
+ "0": 0.9298504292964935,
134
+ "1": 0.9594626426696777,
135
+ "2": 0.9668116569519043,
136
+ "3": 0.97734135389328,
137
+ "4": 0.9817334413528442,
138
+ "5": 0.9862845540046692,
139
+ "6": 0.9803015291690826,
140
+ "7": 0.9960329234600067,
141
+ "8": 0.9933978915214539,
142
+ "9": 0.9950797855854034,
143
+ "10": 0.9924003481864929,
144
+ "11": 0.9950767755508423,
145
+ "12": 0.9927493929862976,
146
+ "13": 0.9933296144008636,
147
+ "14": 0.9937319457530975,
148
+ "15": 0.9942990839481354,
149
+ "16": 0.9920076131820679,
150
+ "17": 0.9934728741645813,
151
+ "18": 0.9954265654087067,
152
+ "19": 0.9932436048984528,
153
+ "20": 0.9957763850688934,
154
+ "21": 0.9964823722839355,
155
+ "22": 0.993859052658081,
156
+ "23": 0.9973229467868805,
157
+ "24": 0.9960962235927582,
158
+ "25": 0.9975778460502625,
159
+ "26": 0.997130274772644,
160
+ "27": 0.9965287148952484,
161
+ "28": 0.9967555701732635,
162
+ "29": 0.9974750280380249,
163
+ "30": 0.996643602848053,
164
+ "31": 0.9963894486427307,
165
+ "32": 0.9952475130558014,
166
+ "33": 0.9919288456439972,
167
+ "34": 0.9904468953609467,
168
+ "35": 0.9905306994915009
169
+ },
170
+ "avg": 0.9896729290485382
171
+ }
172
+ }
173
+ },
174
+ "learned_over_raw_K128": 0.9935716706697983
175
+ }