File size: 6,359 Bytes
bc0155d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 |
{
"prefill": {
"memory": {
"unit": "MB",
"max_ram": 3525.599232,
"max_vram": 11714.691072,
"max_reserved": 11221.860352,
"max_allocated": 9991.724544
},
"latency": {
"unit": "s",
"mean": 0.3010050461713006,
"stdev": 0.005007823178304763,
"values": [
0.32892837524414065,
0.3053179016113281,
0.3002685546875,
0.2994657287597656,
0.3013898315429687,
0.29939508056640624,
0.29945855712890623,
0.3003228149414062,
0.29954458618164065,
0.29985791015625,
0.3006095275878906,
0.3026851806640625,
0.2997637023925781,
0.300037109375,
0.2996305847167969,
0.29964801025390625,
0.3008983154296875,
0.2997923889160156,
0.2989916076660156,
0.29947698974609377,
0.2995968017578125,
0.2994698181152344,
0.29934283447265625,
0.299536376953125,
0.29968487548828127,
0.29946783447265624,
0.30125567626953126,
0.30218035888671874,
0.2998056945800781,
0.29990194702148437,
0.2995118103027344,
0.29968487548828127,
0.29941860961914063,
0.299831298828125
]
},
"throughput": {
"unit": "tokens/s",
"value": 531.5525504809801
},
"energy": null,
"efficiency": null
},
"decode": {
"memory": {
"unit": "MB",
"max_ram": 3536.162816,
"max_vram": 12268.3392,
"max_reserved": 11775.50848,
"max_allocated": 10078.902272
},
"latency": {
"unit": "s",
"mean": 25.19557006835937,
"stdev": 0,
"values": [
25.19557006835937
]
},
"throughput": {
"unit": "tokens/s",
"value": 39.292621572521725
},
"energy": null,
"efficiency": null
},
"per_token": {
"memory": null,
"latency": {
"unit": "s",
"mean": 0.25450070776120576,
"stdev": 0.0028173351708351006,
"values": [
0.2622341003417969,
0.2604031982421875,
0.2598922119140625,
0.259346435546875,
0.26023934936523435,
0.259125244140625,
0.259641357421875,
0.2600816650390625,
0.2582261657714844,
0.2586593322753906,
0.2587289733886719,
0.2589450378417969,
0.2583848876953125,
0.2578052978515625,
0.25718887329101564,
0.2568939514160156,
0.25893478393554686,
0.2582753295898437,
0.25697381591796875,
0.2581626892089844,
0.256964599609375,
0.256321533203125,
0.25642291259765626,
0.25666763305664064,
0.25534873962402344,
0.25577267456054686,
0.25576243591308595,
0.25529037475585936,
0.25610955810546876,
0.2554173431396484,
0.2555606994628906,
0.25545318603515627,
0.2541045684814453,
0.2553528289794922,
0.2549780426025391,
0.2547394561767578,
0.2549903411865234,
0.2602659912109375,
0.2590689392089844,
0.25401242065429686,
0.2539376678466797,
0.254271484375,
0.2540543975830078,
0.25435136413574216,
0.2539878387451172,
0.25366937255859373,
0.25358233642578126,
0.25377484130859373,
0.2529126434326172,
0.2540585021972656,
0.25315840148925783,
0.2530897979736328,
0.2536099853515625,
0.2528061370849609,
0.25310617065429686,
0.25320346069335936,
0.2535393218994141,
0.2527958984375,
0.253264892578125,
0.25287986755371095,
0.2530160675048828,
0.2529689636230469,
0.25187532043457034,
0.25355877685546874,
0.25151078796386717,
0.2520494079589844,
0.25327001953125,
0.25154150390625,
0.2519019470214844,
0.25225625610351565,
0.25124659729003906,
0.25213848876953127,
0.25257369995117185,
0.2515240936279297,
0.2524241943359375,
0.2529085388183594,
0.2525675506591797,
0.25248051452636716,
0.2522879943847656,
0.2524682312011719,
0.2532095947265625,
0.25251429748535154,
0.2515650634765625,
0.25252351379394533,
0.25155378723144534,
0.2517022705078125,
0.25210067749023435,
0.25160902404785157,
0.25191014099121095,
0.25165721130371094,
0.25126296997070313,
0.25138584899902344,
0.250998779296875,
0.2516172790527344,
0.2514114532470703,
0.25093426513671874,
0.25146060180664065,
0.251789306640625,
0.25099366760253905
]
},
"throughput": {
"unit": "tokens/s",
"value": 39.292621572521725
},
"energy": null,
"efficiency": null
}
} |