File size: 6,687 Bytes
36a6b96 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 |
{
"prefill": {
"memory": {
"unit": "MB",
"max_ram": 3377.37728,
"max_vram": 6058.672128,
"max_reserved": 5576.327168,
"max_allocated": 5355.380736
},
"latency": {
"unit": "s",
"mean": 0.24580386203672833,
"stdev": 0.00022219677637899774,
"values": [
0.24691427612304687,
0.24614707946777345,
0.24578355407714844,
0.24561459350585937,
0.2458961944580078,
0.24560025024414062,
0.2456575927734375,
0.24577023315429689,
0.24575999450683594,
0.24567091369628907,
0.2458787841796875,
0.24583171081542968,
0.24586341857910157,
0.24571495056152343,
0.24590336608886718,
0.24583168029785157,
0.24580096435546875,
0.24568524169921874,
0.24596377563476562,
0.24577023315429689,
0.24546406555175782,
0.24596377563476562,
0.24557772827148439,
0.24591360473632812,
0.24575999450683594,
0.2457108154296875,
0.24578866577148437,
0.2458992614746094,
0.24572927856445312,
0.2458675231933594,
0.24559309387207032,
0.2457733154296875,
0.24587980651855468,
0.24561048889160156,
0.24586239624023437,
0.2460037078857422,
0.24556236267089843,
0.24560127258300782,
0.24583680725097656,
0.2457057342529297,
0.24579583740234376
]
},
"throughput": {
"unit": "tokens/s",
"value": 1952.7764780533748
},
"energy": null,
"efficiency": null
},
"decode": {
"memory": {
"unit": "MB",
"max_ram": 3377.37728,
"max_vram": 7197.425664,
"max_reserved": 6712.983552,
"max_allocated": 6292.99456
},
"latency": {
"unit": "s",
"mean": 14.316073944091789,
"stdev": 0,
"values": [
14.316073944091789
]
},
"throughput": {
"unit": "tokens/s",
"value": 207.45911285445072
},
"energy": null,
"efficiency": null
},
"per_token": {
"memory": null,
"latency": {
"unit": "s",
"mean": 0.14460680751607868,
"stdev": 0.002633980375935467,
"values": [
0.14012416076660156,
0.14033407592773436,
0.14042930603027343,
0.14048460388183595,
0.14043341064453124,
0.14061260986328125,
0.14064947509765624,
0.14090342712402343,
0.14090138244628905,
0.14114303588867189,
0.14110208129882812,
0.14144717407226562,
0.14132838439941406,
0.14151271057128906,
0.14150758361816407,
0.14181170654296876,
0.14139903259277345,
0.14184550476074217,
0.14165708923339843,
0.14212300109863282,
0.14184141540527342,
0.14210560607910155,
0.14203392028808592,
0.1424148406982422,
0.1421895751953125,
0.1424486389160156,
0.14228172302246095,
0.14273228454589842,
0.14244146728515625,
0.14266983032226563,
0.14270361328125,
0.1431470031738281,
0.14275686645507812,
0.14305381774902343,
0.14302105712890625,
0.14357913208007814,
0.14316645812988282,
0.14344499206542968,
0.14331596374511718,
0.1438586883544922,
0.1435484161376953,
0.14384640502929688,
0.14373989868164064,
0.14430105590820314,
0.14400717163085938,
0.1441095733642578,
0.1440245819091797,
0.14447718811035157,
0.14414540100097656,
0.14444032287597655,
0.14435224914550782,
0.1449502716064453,
0.14458265686035157,
0.14494207763671876,
0.14483045959472657,
0.14541413879394532,
0.14501580810546874,
0.14532608032226563,
0.14521241760253906,
0.14594866943359375,
0.14552268981933594,
0.14566502380371094,
0.14550834655761719,
0.14620364379882814,
0.14571417236328124,
0.14605722045898437,
0.14588825988769533,
0.14663066101074218,
0.14609100341796874,
0.14640538024902344,
0.1462906951904297,
0.14695526123046876,
0.1464596405029297,
0.14691635131835937,
0.14672076416015625,
0.14758912658691406,
0.1468211212158203,
0.1471631317138672,
0.14699417114257812,
0.14770176696777343,
0.14713037109375,
0.14749798583984375,
0.14733413696289063,
0.1480693817138672,
0.14749900817871095,
0.1478656005859375,
0.14795468139648438,
0.14866021728515624,
0.148168701171875,
0.14849331665039062,
0.1483356170654297,
0.14925926208496093,
0.14847488403320314,
0.14881587219238282,
0.14865408325195312,
0.1494906921386719,
0.1488107452392578,
0.14924082946777345,
0.1488486328125
]
},
"throughput": {
"unit": "tokens/s",
"value": 207.45911285445072
},
"energy": null,
"efficiency": null
}
} |