Hugging Face's logo

datasysdev
/
ann-sparseattention

Transformers
GGUF
sparse-attention
approximate-nearest-neighbors
faiss
qwen3
long-context
conversational
Model card Files
xet
 Community
datasysdev commited on
Commit
242d60a
·
verified ·
1 Parent(s): ef9a166

Upload checkpoints_all36_d128_block/search_step_800.compare_retrieval.json

Browse files
Files changed (1) hide show
  1. checkpoints_all36_d128_block/search_step_800.compare_retrieval.json +175 -0
checkpoints_all36_d128_block/search_step_800.compare_retrieval.json ADDED
@@ -0,0 +1,175 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "Qwen/Qwen3-4B-Instruct-2507",
3
+ "ckpt": "/tmp/checkpoints_all36_d128_block/search_step_800.pt",
4
+ "by_K": {
5
+ "128": {
6
+ "raw_qk": {
7
+ "per_layer": {
8
+ "0": 0.9221310317516327,
9
+ "1": 0.9183070361614227,
10
+ "2": 0.9386252164840698,
11
+ "3": 0.9387637376785278,
12
+ "4": 0.9440614581108093,
13
+ "5": 0.96429443359375,
14
+ "6": 0.9560344219207764,
15
+ "7": 0.9818542897701263,
16
+ "8": 0.9707900285720825,
17
+ "9": 0.9587394297122955,
18
+ "10": 0.9742867350578308,
19
+ "11": 0.9763059914112091,
20
+ "12": 0.9611693322658539,
21
+ "13": 0.9707654714584351,
22
+ "14": 0.9733296930789948,
23
+ "15": 0.9675203263759613,
24
+ "16": 0.9556995928287506,
25
+ "17": 0.9593449831008911,
26
+ "18": 0.9652450382709503,
27
+ "19": 0.9612957835197449,
28
+ "20": 0.9591256380081177,
29
+ "21": 0.9657162129878998,
30
+ "22": 0.962736189365387,
31
+ "23": 0.9788879752159119,
32
+ "24": 0.9710511267185211,
33
+ "25": 0.9860879778862,
34
+ "26": 0.9775076508522034,
35
+ "27": 0.977691113948822,
36
+ "28": 0.9794201850891113,
37
+ "29": 0.9821525514125824,
38
+ "30": 0.9875719547271729,
39
+ "31": 0.9842455685138702,
40
+ "32": 0.9791028797626495,
41
+ "33": 0.9772593379020691,
42
+ "34": 0.9759000241756439,
43
+ "35": 0.9796911776065826
44
+ },
45
+ "avg": 0.9661864332026906
46
+ },
47
+ "learned": {
48
+ "per_layer": {
49
+ "0": 0.7756268978118896,
50
+ "1": 0.8533660173416138,
51
+ "2": 0.8985396027565002,
52
+ "3": 0.9248833358287811,
53
+ "4": 0.9358105957508087,
54
+ "5": 0.9497621357440948,
55
+ "6": 0.9386124014854431,
56
+ "7": 0.9828340709209442,
57
+ "8": 0.9707432389259338,
58
+ "9": 0.9764380753040314,
59
+ "10": 0.9706515073776245,
60
+ "11": 0.9758569598197937,
61
+ "12": 0.9699209034442902,
62
+ "13": 0.9717125296592712,
63
+ "14": 0.9731432497501373,
64
+ "15": 0.9723271131515503,
65
+ "16": 0.9620000123977661,
66
+ "17": 0.9667199552059174,
67
+ "18": 0.972541332244873,
68
+ "19": 0.9684668481349945,
69
+ "20": 0.9755434393882751,
70
+ "21": 0.9796880185604095,
71
+ "22": 0.9698434174060822,
72
+ "23": 0.9845690429210663,
73
+ "24": 0.9784398972988129,
74
+ "25": 0.98859041929245,
75
+ "26": 0.9856358468532562,
76
+ "27": 0.9832888841629028,
77
+ "28": 0.9848624765872955,
78
+ "29": 0.9874634146690369,
79
+ "30": 0.9862565696239471,
80
+ "31": 0.9839431643486023,
81
+ "32": 0.9798043966293335,
82
+ "33": 0.9695849716663361,
83
+ "34": 0.9600768983364105,
84
+ "35": 0.9647442996501923
85
+ },
86
+ "avg": 0.9603414427902963
87
+ }
88
+ },
89
+ "256": {
90
+ "raw_qk": {
91
+ "per_layer": {
92
+ "0": 0.9736533463001251,
93
+ "1": 0.9831015169620514,
94
+ "2": 0.985590249300003,
95
+ "3": 0.9856403172016144,
96
+ "4": 0.9863723814487457,
97
+ "5": 0.9927175343036652,
98
+ "6": 0.9899796843528748,
99
+ "7": 0.9961064755916595,
100
+ "8": 0.9944310188293457,
101
+ "9": 0.9924624264240265,
102
+ "10": 0.9945598244667053,
103
+ "11": 0.9956617951393127,
104
+ "12": 0.9914700388908386,
105
+ "13": 0.9947893917560577,
106
+ "14": 0.9956192076206207,
107
+ "15": 0.9947386384010315,
108
+ "16": 0.9918601512908936,
109
+ "17": 0.9932802319526672,
110
+ "18": 0.994801938533783,
111
+ "19": 0.9929636716842651,
112
+ "20": 0.9928486049175262,
113
+ "21": 0.9940518736839294,
114
+ "22": 0.9925838708877563,
115
+ "23": 0.9956205189228058,
116
+ "24": 0.9944405257701874,
117
+ "25": 0.99702388048172,
118
+ "26": 0.9963724911212921,
119
+ "27": 0.9952751994132996,
120
+ "28": 0.9952269494533539,
121
+ "29": 0.9970046579837799,
122
+ "30": 0.9976439476013184,
123
+ "31": 0.9967932999134064,
124
+ "32": 0.9949435591697693,
125
+ "33": 0.9954463541507721,
126
+ "34": 0.9949491024017334,
127
+ "35": 0.9950292408466339
128
+ },
129
+ "avg": 0.9929181643658214
130
+ },
131
+ "learned": {
132
+ "per_layer": {
133
+ "0": 0.9241782128810883,
134
+ "1": 0.9612297713756561,
135
+ "2": 0.9658167064189911,
136
+ "3": 0.9773095548152924,
137
+ "4": 0.9823991358280182,
138
+ "5": 0.9867597222328186,
139
+ "6": 0.9807394444942474,
140
+ "7": 0.9962039291858673,
141
+ "8": 0.9934562742710114,
142
+ "9": 0.9951532781124115,
143
+ "10": 0.9924466609954834,
144
+ "11": 0.9950800836086273,
145
+ "12": 0.992906391620636,
146
+ "13": 0.9934229552745819,
147
+ "14": 0.9937788546085358,
148
+ "15": 0.9944615662097931,
149
+ "16": 0.9920341670513153,
150
+ "17": 0.9936630129814148,
151
+ "18": 0.995515376329422,
152
+ "19": 0.993456095457077,
153
+ "20": 0.9959358274936676,
154
+ "21": 0.9966034889221191,
155
+ "22": 0.9939581751823425,
156
+ "23": 0.9973926544189453,
157
+ "24": 0.9962430894374847,
158
+ "25": 0.9975726306438446,
159
+ "26": 0.9972138106822968,
160
+ "27": 0.9966393709182739,
161
+ "28": 0.9968178272247314,
162
+ "29": 0.9975622594356537,
163
+ "30": 0.9967368543148041,
164
+ "31": 0.9965087473392487,
165
+ "32": 0.9954238533973694,
166
+ "33": 0.9919829368591309,
167
+ "34": 0.9898959100246429,
168
+ "35": 0.9893121719360352
169
+ },
170
+ "avg": 0.9896058556106355
171
+ }
172
+ }
173
+ },
174
+ "learned_over_raw_K128": 0.9939504528199392
175
+ }