Hugging Face's logo

harshithsaiv
/
kv-cache-compression

English
quantization
kv-cache
llm-inference
cuda
triton
memory-efficient
mitral
llama
inference-optimization
4-bit precision
mixed-precision
Model card Files
xet
 Community
harshithsaiv commited on
Commit
cf6fec9
·
1 Parent(s): 8eabcbc

Updated bit allocation for mistral

Browse files
Files changed (1) hide show
  1. results/mistral-7b/bit_allocation.json +67 -67
results/mistral-7b/bit_allocation.json CHANGED
@@ -1,58 +1,58 @@
1
  {
2
  "0": {
3
  "0": 8,
4
- "1": 8,
5
- "2": 8,
6
- "3": 8,
7
- "4": 8,
8
- "5": 8,
9
- "6": 8,
10
- "7": 8
11
  },
12
  "1": {
13
  "0": 8,
14
- "1": 8,
15
  "2": 8,
16
- "3": 8,
17
- "4": 8,
18
  "5": 8,
19
- "6": 8,
20
- "7": 8
21
  },
22
  "2": {
23
  "0": 8,
24
  "1": 8,
25
  "2": 8,
26
- "3": 8,
27
- "4": 8,
28
  "5": 8,
29
  "6": 8,
30
  "7": 8
31
  },
32
  "3": {
33
- "0": 8,
34
  "1": 8,
35
  "2": 8,
36
  "3": 8,
37
- "4": 8,
38
  "5": 8,
39
  "6": 8,
40
  "7": 8
41
  },
42
  "4": {
43
- "0": 8,
44
  "1": 8,
45
  "2": 8,
46
  "3": 8,
47
  "4": 8,
48
  "5": 8,
49
- "6": 8,
50
  "7": 8
51
  },
52
  "5": {
53
  "0": 8,
54
  "1": 8,
55
- "2": 8,
56
  "3": 8,
57
  "4": 8,
58
  "5": 8,
@@ -77,23 +77,23 @@
77
  "4": 8,
78
  "5": 8,
79
  "6": 8,
80
- "7": 8
81
  },
82
  "8": {
83
  "0": 8,
84
  "1": 8,
85
  "2": 8,
86
  "3": 8,
87
- "4": 8,
88
  "5": 8,
89
  "6": 8,
90
  "7": 8
91
  },
92
  "9": {
93
- "0": 8,
94
- "1": 8,
95
  "2": 8,
96
- "3": 8,
97
  "4": 8,
98
  "5": 8,
99
  "6": 8,
@@ -101,18 +101,18 @@
101
  },
102
  "10": {
103
  "0": 8,
104
- "1": 8,
105
  "2": 8,
106
  "3": 8,
107
- "4": 8,
108
  "5": 8,
109
  "6": 8,
110
  "7": 8
111
  },
112
  "11": {
113
- "0": 8,
114
  "1": 8,
115
- "2": 8,
116
  "3": 8,
117
  "4": 8,
118
  "5": 8,
@@ -127,7 +127,7 @@
127
  "4": 8,
128
  "5": 8,
129
  "6": 8,
130
- "7": 8
131
  },
132
  "13": {
133
  "0": 8,
@@ -141,13 +141,13 @@
141
  },
142
  "14": {
143
  "0": 8,
144
- "1": 8,
145
  "2": 8,
146
  "3": 8,
147
- "4": 8,
148
  "5": 8,
149
  "6": 8,
150
- "7": 8
151
  },
152
  "15": {
153
  "0": 8,
@@ -177,7 +177,7 @@
177
  "4": 8,
178
  "5": 8,
179
  "6": 8,
180
- "7": 8
181
  },
182
  "18": {
183
  "0": 8,
@@ -210,30 +210,30 @@
210
  "7": 8
211
  },
212
  "21": {
213
- "0": 8,
214
  "1": 8,
215
  "2": 8,
216
- "3": 8,
217
  "4": 8,
218
  "5": 8,
219
  "6": 8,
220
  "7": 8
221
  },
222
  "22": {
223
- "0": 8,
224
- "1": 8,
225
  "2": 8,
226
  "3": 8,
227
- "4": 8,
228
  "5": 8,
229
  "6": 8,
230
- "7": 8
231
  },
232
  "23": {
233
  "0": 8,
234
  "1": 8,
235
- "2": 8,
236
- "3": 8,
237
  "4": 8,
238
  "5": 8,
239
  "6": 8,
@@ -241,12 +241,12 @@
241
  },
242
  "24": {
243
  "0": 8,
244
- "1": 8,
245
  "2": 8,
246
- "3": 8,
247
- "4": 8,
248
  "5": 8,
249
- "6": 8,
250
  "7": 8
251
  },
252
  "25": {
@@ -254,17 +254,17 @@
254
  "1": 8,
255
  "2": 8,
256
  "3": 8,
257
- "4": 8,
258
  "5": 8,
259
- "6": 8,
260
  "7": 8
261
  },
262
  "26": {
263
  "0": 8,
264
- "1": 8,
265
  "2": 8,
266
- "3": 8,
267
- "4": 8,
268
  "5": 8,
269
  "6": 8,
270
  "7": 8
@@ -274,49 +274,49 @@
274
  "1": 8,
275
  "2": 8,
276
  "3": 8,
277
- "4": 8,
278
  "5": 8,
279
- "6": 8,
280
- "7": 8
281
  },
282
  "28": {
283
  "0": 8,
284
  "1": 8,
285
  "2": 8,
286
  "3": 8,
287
- "4": 8,
288
- "5": 8,
289
- "6": 8,
290
- "7": 8
291
  },
292
  "29": {
293
  "0": 8,
294
  "1": 8,
295
- "2": 8,
296
  "3": 8,
297
  "4": 8,
298
  "5": 8,
299
- "6": 8,
300
  "7": 8
301
  },
302
  "30": {
303
- "0": 8,
304
- "1": 8,
305
  "2": 8,
306
- "3": 8,
307
- "4": 8,
308
  "5": 8,
309
- "6": 8,
310
  "7": 8
311
  },
312
  "31": {
313
- "0": 8,
314
  "1": 8,
315
  "2": 8,
316
  "3": 8,
317
  "4": 8,
318
  "5": 8,
319
- "6": 8,
320
- "7": 8
321
  }
322
  }
 
1
  {
2
  "0": {
3
  "0": 8,
4
+ "1": 4,
5
+ "2": 4,
6
+ "3": 4,
7
+ "4": 4,
8
+ "5": 4,
9
+ "6": 4,
10
+ "7": 4
11
  },
12
  "1": {
13
  "0": 8,
14
+ "1": 4,
15
  "2": 8,
16
+ "3": 4,
17
+ "4": 4,
18
  "5": 8,
19
+ "6": 4,
20
+ "7": 4
21
  },
22
  "2": {
23
  "0": 8,
24
  "1": 8,
25
  "2": 8,
26
+ "3": 4,
27
+ "4": 4,
28
  "5": 8,
29
  "6": 8,
30
  "7": 8
31
  },
32
  "3": {
33
+ "0": 4,
34
  "1": 8,
35
  "2": 8,
36
  "3": 8,
37
+ "4": 4,
38
  "5": 8,
39
  "6": 8,
40
  "7": 8
41
  },
42
  "4": {
43
+ "0": 4,
44
  "1": 8,
45
  "2": 8,
46
  "3": 8,
47
  "4": 8,
48
  "5": 8,
49
+ "6": 4,
50
  "7": 8
51
  },
52
  "5": {
53
  "0": 8,
54
  "1": 8,
55
+ "2": 4,
56
  "3": 8,
57
  "4": 8,
58
  "5": 8,
 
77
  "4": 8,
78
  "5": 8,
79
  "6": 8,
80
+ "7": 4
81
  },
82
  "8": {
83
  "0": 8,
84
  "1": 8,
85
  "2": 8,
86
  "3": 8,
87
+ "4": 4,
88
  "5": 8,
89
  "6": 8,
90
  "7": 8
91
  },
92
  "9": {
93
+ "0": 4,
94
+ "1": 4,
95
  "2": 8,
96
+ "3": 4,
97
  "4": 8,
98
  "5": 8,
99
  "6": 8,
 
101
  },
102
  "10": {
103
  "0": 8,
104
+ "1": 4,
105
  "2": 8,
106
  "3": 8,
107
+ "4": 4,
108
  "5": 8,
109
  "6": 8,
110
  "7": 8
111
  },
112
  "11": {
113
+ "0": 4,
114
  "1": 8,
115
+ "2": 4,
116
  "3": 8,
117
  "4": 8,
118
  "5": 8,
 
127
  "4": 8,
128
  "5": 8,
129
  "6": 8,
130
+ "7": 4
131
  },
132
  "13": {
133
  "0": 8,
 
141
  },
142
  "14": {
143
  "0": 8,
144
+ "1": 4,
145
  "2": 8,
146
  "3": 8,
147
+ "4": 4,
148
  "5": 8,
149
  "6": 8,
150
+ "7": 4
151
  },
152
  "15": {
153
  "0": 8,
 
177
  "4": 8,
178
  "5": 8,
179
  "6": 8,
180
+ "7": 4
181
  },
182
  "18": {
183
  "0": 8,
 
210
  "7": 8
211
  },
212
  "21": {
213
+ "0": 4,
214
  "1": 8,
215
  "2": 8,
216
+ "3": 4,
217
  "4": 8,
218
  "5": 8,
219
  "6": 8,
220
  "7": 8
221
  },
222
  "22": {
223
+ "0": 4,
224
+ "1": 4,
225
  "2": 8,
226
  "3": 8,
227
+ "4": 4,
228
  "5": 8,
229
  "6": 8,
230
+ "7": 4
231
  },
232
  "23": {
233
  "0": 8,
234
  "1": 8,
235
+ "2": 4,
236
+ "3": 4,
237
  "4": 8,
238
  "5": 8,
239
  "6": 8,
 
241
  },
242
  "24": {
243
  "0": 8,
244
+ "1": 4,
245
  "2": 8,
246
+ "3": 4,
247
+ "4": 4,
248
  "5": 8,
249
+ "6": 4,
250
  "7": 8
251
  },
252
  "25": {
 
254
  "1": 8,
255
  "2": 8,
256
  "3": 8,
257
+ "4": 4,
258
  "5": 8,
259
+ "6": 4,
260
  "7": 8
261
  },
262
  "26": {
263
  "0": 8,
264
+ "1": 4,
265
  "2": 8,
266
+ "3": 4,
267
+ "4": 4,
268
  "5": 8,
269
  "6": 8,
270
  "7": 8
 
274
  "1": 8,
275
  "2": 8,
276
  "3": 8,
277
+ "4": 4,
278
  "5": 8,
279
+ "6": 4,
280
+ "7": 4
281
  },
282
  "28": {
283
  "0": 8,
284
  "1": 8,
285
  "2": 8,
286
  "3": 8,
287
+ "4": 4,
288
+ "5": 4,
289
+ "6": 4,
290
+ "7": 4
291
  },
292
  "29": {
293
  "0": 8,
294
  "1": 8,
295
+ "2": 4,
296
  "3": 8,
297
  "4": 8,
298
  "5": 8,
299
+ "6": 4,
300
  "7": 8
301
  },
302
  "30": {
303
+ "0": 4,
304
+ "1": 4,
305
  "2": 8,
306
+ "3": 4,
307
+ "4": 4,
308
  "5": 8,
309
+ "6": 4,
310
  "7": 8
311
  },
312
  "31": {
313
+ "0": 4,
314
  "1": 8,
315
  "2": 8,
316
  "3": 8,
317
  "4": 8,
318
  "5": 8,
319
+ "6": 4,
320
+ "7": 4
321
  }
322
  }