File size: 6,547 Bytes
45696ce |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 |
{
"prefill": {
"memory": {
"unit": "MB",
"max_ram": 3576.500224,
"max_vram": 11024.728064,
"max_reserved": 10531.897344,
"max_allocated": 9458.51392
},
"latency": {
"unit": "s",
"mean": 0.26589927994577517,
"stdev": 0.004717404774457072,
"values": [
0.2945403137207031,
0.2661632080078125,
0.2650818176269531,
0.26499276733398436,
0.2650798034667969,
0.2648647766113281,
0.2649354248046875,
0.26499072265625,
0.2650746765136719,
0.2650808410644531,
0.2649825134277344,
0.26509210205078126,
0.2649774169921875,
0.2652323913574219,
0.2649415588378906,
0.2650306701660156,
0.26549758911132815,
0.26528564453125,
0.26555596923828123,
0.2653173828125,
0.26431591796875,
0.2648350830078125,
0.26553549194335935,
0.26530612182617186,
0.26522317504882814,
0.2651463623046875,
0.26528460693359374,
0.26509005737304686,
0.2648668212890625,
0.26478387451171875,
0.2652835693359375,
0.26543923950195314,
0.2648074340820312,
0.2652200927734375,
0.2651146240234375,
0.26522113037109374,
0.264890380859375,
0.265091064453125
]
},
"throughput": {
"unit": "tokens/s",
"value": 300.8658015783811
},
"energy": null,
"efficiency": null
},
"decode": {
"memory": {
"unit": "MB",
"max_ram": 3577.966592,
"max_vram": 11127.488512,
"max_reserved": 10634.657792,
"max_allocated": 9518.949888
},
"latency": {
"unit": "s",
"mean": 23.714270278930663,
"stdev": 0,
"values": [
23.714270278930663
]
},
"throughput": {
"unit": "tokens/s",
"value": 20.873507562228934
},
"energy": null,
"efficiency": null
},
"per_token": {
"memory": null,
"latency": {
"unit": "s",
"mean": 0.23953808362556225,
"stdev": 0.0011223838576208927,
"values": [
0.24126976013183593,
0.24184934997558594,
0.241944580078125,
0.24157696533203124,
0.24142233276367187,
0.24156364440917968,
0.24111923217773437,
0.24104243469238282,
0.24155647277832032,
0.2407731170654297,
0.24124211120605468,
0.24097996520996093,
0.24022732543945313,
0.24134962463378906,
0.24109158325195312,
0.24052633666992188,
0.24072601318359374,
0.24031642150878907,
0.2408704071044922,
0.24035328674316406,
0.24031436157226563,
0.2403256378173828,
0.2404638671875,
0.2411847686767578,
0.23977984619140624,
0.24041165161132813,
0.23975117492675782,
0.24032357788085937,
0.24015565490722657,
0.24032666015625,
0.23952691650390626,
0.23980543518066405,
0.2395371551513672,
0.24026419067382812,
0.24166502380371094,
0.2394122314453125,
0.24026419067382812,
0.2393456573486328,
0.23959449768066407,
0.23952793884277343,
0.23927090454101563,
0.23895552062988282,
0.23910604858398438,
0.23962112426757812,
0.2393866271972656,
0.23915008544921876,
0.23919512939453125,
0.2399006652832031,
0.23890739440917969,
0.23990170288085938,
0.23840255737304689,
0.23848141479492188,
0.2383585205078125,
0.23940914916992187,
0.23918080139160156,
0.23795199584960938,
0.23889100646972655,
0.23912037658691407,
0.23861759948730468,
0.23844248962402342,
0.23861759948730468,
0.2423388214111328,
0.23819981384277344,
0.23846400451660157,
0.23878758239746095,
0.2382704620361328,
0.23828684997558594,
0.23864218139648438,
0.23916134643554687,
0.23734579467773437,
0.23761715698242186,
0.23818240356445314,
0.2382940216064453,
0.238097412109375,
0.23909068298339844,
0.2390118408203125,
0.23936819458007813,
0.23946751403808594,
0.23883673095703126,
0.23915110778808593,
0.23877529907226563,
0.23924327087402344,
0.23862168884277343,
0.23861351013183593,
0.2389811248779297,
0.2389422149658203,
0.2384701385498047,
0.2389063720703125,
0.2385039367675781,
0.23919821166992186,
0.2386544647216797,
0.23751475524902343,
0.23883775329589843,
0.2382458953857422,
0.23910195922851563,
0.2384701385498047,
0.2383462371826172,
0.2390118408203125,
0.23856742858886718
]
},
"throughput": {
"unit": "tokens/s",
"value": 20.873507562228934
},
"energy": null,
"efficiency": null
}
} |