File size: 6,033 Bytes
1d8dc5f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 |
{
"prefill": {
"memory": {
"unit": "MB",
"max_ram": 3610.86976,
"max_vram": 13706.985472,
"max_reserved": 13214.154752,
"max_allocated": 11027.453952
},
"latency": {
"unit": "s",
"mean": 0.4016479968261719,
"stdev": 0.00450307941805889,
"values": [
0.4229984130859375,
0.4061747131347656,
0.4007014465332031,
0.4005621643066406,
0.4003113098144531,
0.4004822998046875,
0.4004290466308594,
0.4008570861816406,
0.40047308349609373,
0.40001739501953126,
0.4006717529296875,
0.4008058776855469,
0.40065023803710936,
0.4005826416015625,
0.40037786865234376,
0.40050381469726565,
0.40064306640625,
0.40063385009765623,
0.40045669555664065,
0.39976141357421874,
0.40033383178710935,
0.4008027648925781,
0.40047308349609373,
0.4009308166503906,
0.40056524658203124
]
},
"throughput": {
"unit": "tokens/s",
"value": 995.8969126220113
},
"energy": null,
"efficiency": null
},
"decode": {
"memory": {
"unit": "MB",
"max_ram": 3630.583808,
"max_vram": 13736.3456,
"max_reserved": 13243.51488,
"max_allocated": 11166.721536
},
"latency": {
"unit": "s",
"mean": 28.560530487060543,
"stdev": 0,
"values": [
28.560530487060543
]
},
"throughput": {
"unit": "tokens/s",
"value": 86.65805423751874
},
"energy": null,
"efficiency": null
},
"per_token": {
"memory": null,
"latency": {
"unit": "s",
"mean": 0.2884902069400055,
"stdev": 0.005574553256331573,
"values": [
0.30573773193359377,
0.3014727783203125,
0.29969100952148436,
0.29973300170898437,
0.29858203125,
0.298608642578125,
0.29833624267578124,
0.2978570251464844,
0.29740133666992186,
0.29682278442382815,
0.29672549438476564,
0.2960639953613281,
0.2946611328125,
0.2954721374511719,
0.2954915771484375,
0.2945587158203125,
0.29500518798828124,
0.2944102478027344,
0.29504205322265625,
0.29280154418945314,
0.2928322448730469,
0.29225778198242186,
0.2919669799804688,
0.29329202270507815,
0.2921922607421875,
0.29252301025390626,
0.2918021240234375,
0.29138430786132813,
0.29127166748046873,
0.2904176635742188,
0.2911313781738281,
0.2917529602050781,
0.2908026733398438,
0.29039410400390625,
0.2901688232421875,
0.28997222900390623,
0.28976434326171874,
0.2902865905761719,
0.28897689819335937,
0.28997222900390623,
0.2897592468261719,
0.28981964111328123,
0.28855502319335935,
0.2880440368652344,
0.2889666442871094,
0.28792318725585936,
0.28774911499023437,
0.2884403076171875,
0.2875832214355469,
0.28695040893554685,
0.2873180236816406,
0.2875217895507812,
0.2857482299804687,
0.28675787353515625,
0.2872637329101563,
0.28659710693359375,
0.2864322509765625,
0.28636672973632815,
0.28585574340820313,
0.28576461791992186,
0.2865428466796875,
0.2852802429199219,
0.2851758117675781,
0.284943359375,
0.28499661254882813,
0.28454400634765625,
0.28392242431640624,
0.2840729675292969,
0.28438323974609375,
0.28357427978515626,
0.28338687133789064,
0.28420913696289063,
0.28362240600585936,
0.2831769714355469,
0.2835486755371094,
0.28436376953125,
0.28338381958007813,
0.2831790161132812,
0.28350567626953127,
0.28333056640625,
0.28241510009765625,
0.2824325256347656,
0.2824560546875,
0.2829588623046875,
0.2829619140625,
0.28229632568359375,
0.2825932922363281,
0.28196762084960936,
0.2830182495117187,
0.28210791015625,
0.2823055419921875,
0.28156927490234374,
0.2821396484375,
0.2817843322753906,
0.28104806518554687,
0.28229119873046876,
0.2816358337402344,
0.281427978515625,
0.2809241638183594
]
},
"throughput": {
"unit": "tokens/s",
"value": 86.65805423751874
},
"energy": null,
"efficiency": null
}
} |