File size: 6,432 Bytes
c59043f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 |
{
"prefill": {
"memory": {
"unit": "MB",
"max_ram": 2261.991424,
"max_vram": 6025.117696,
"max_reserved": 5540.675584,
"max_allocated": 5394.580992
},
"latency": {
"unit": "s",
"mean": 0.28051526896158857,
"stdev": 0.0010279945260692365,
"values": [
0.2865542297363281,
0.2806671447753906,
0.2802841491699219,
0.2804653930664063,
0.2802114562988281,
0.2804633483886719,
0.28021658325195314,
0.2804223937988281,
0.2802063293457031,
0.2805329895019531,
0.28019915771484377,
0.28044082641601564,
0.2802318725585938,
0.28040499877929687,
0.2802206726074219,
0.280469482421875,
0.28022171020507813,
0.2803619689941406,
0.2803384399414062,
0.2805390625,
0.2801838073730469,
0.2803732604980469,
0.2802288513183594,
0.28042547607421875,
0.2802565002441406,
0.28036505126953126,
0.2802288513183594,
0.28041632080078127,
0.2801919860839844,
0.280416259765625,
0.2802309265136719,
0.28041012573242186,
0.2803465576171875,
0.28036294555664065,
0.2801705017089844,
0.28049005126953125
]
},
"throughput": {
"unit": "tokens/s",
"value": 2281.5157348444955
},
"energy": null,
"efficiency": null
},
"decode": {
"memory": {
"unit": "MB",
"max_ram": 2261.991424,
"max_vram": 7392.4608,
"max_reserved": 6908.018688,
"max_allocated": 6645.015552
},
"latency": {
"unit": "s",
"mean": 15.30309118652344,
"stdev": 0,
"values": [
15.30309118652344
]
},
"throughput": {
"unit": "tokens/s",
"value": 258.7712476997684
},
"energy": null,
"efficiency": null
},
"per_token": {
"memory": null,
"latency": {
"unit": "s",
"mean": 0.15457667865175193,
"stdev": 0.0035198829598372676,
"values": [
0.14853631591796876,
0.1485629425048828,
0.14871142578125,
0.14902067565917967,
0.14889369201660158,
0.149138427734375,
0.14918450927734375,
0.14940774536132811,
0.1493780517578125,
0.1495572509765625,
0.14969036865234375,
0.14998118591308593,
0.14973338317871093,
0.15008358764648438,
0.14999449157714845,
0.15047987365722656,
0.1502412872314453,
0.1506007080078125,
0.15072767639160156,
0.15112704467773438,
0.15085055541992187,
0.15127040100097655,
0.15110552978515626,
0.15161856079101563,
0.15133798217773436,
0.15164723205566405,
0.151625732421875,
0.15236813354492187,
0.15195341491699219,
0.15232614135742187,
0.15218482971191405,
0.15276133728027344,
0.15240089416503907,
0.15273983764648438,
0.15265689086914064,
0.15338700866699218,
0.15289138793945312,
0.15316275024414064,
0.15308082580566407,
0.15389286804199218,
0.153491455078125,
0.15368704223632812,
0.153523193359375,
0.15430963134765624,
0.15378125,
0.15412428283691407,
0.1539727325439453,
0.1546598358154297,
0.1542451171875,
0.15469056701660155,
0.15449600219726561,
0.15528857421875,
0.15475302124023438,
0.1551595458984375,
0.1550049285888672,
0.15575039672851562,
0.15514828491210939,
0.1556326446533203,
0.15556915283203124,
0.15636582946777344,
0.15577293395996095,
0.1561333770751953,
0.15592037963867186,
0.1568194580078125,
0.15625625610351562,
0.1566033935546875,
0.15639141845703125,
0.15731610107421876,
0.156579833984375,
0.1570672607421875,
0.15684402465820313,
0.1577379913330078,
0.15710514831542968,
0.15760794067382813,
0.15734579467773438,
0.15827763366699218,
0.15748812866210937,
0.15787930297851563,
0.1578219451904297,
0.15878656005859376,
0.15806874084472655,
0.15856640625,
0.158350341796875,
0.15942041015625,
0.15851519775390624,
0.1590292510986328,
0.15878656005859376,
0.15979519653320312,
0.15904768371582031,
0.15951052856445314,
0.15930572509765625,
0.16033587646484376,
0.15943577575683593,
0.16,
0.15970303344726564,
0.16082124328613281,
0.15998976135253906,
0.1604679718017578,
0.16022015380859375
]
},
"throughput": {
"unit": "tokens/s",
"value": 258.7712476997684
},
"energy": null,
"efficiency": null
}
} |