| { | |
| "prefill": { | |
| "memory": { | |
| "unit": "MB", | |
| "max_ram": 3387.24864, | |
| "max_vram": 6035.603456, | |
| "max_reserved": 5551.161344, | |
| "max_allocated": 5323.650048 | |
| }, | |
| "latency": { | |
| "unit": "s", | |
| "mean": 0.22253290303548182, | |
| "stdev": 0.00020647421006239362, | |
| "values": [ | |
| 0.22377577209472657, | |
| 0.22296064758300782, | |
| 0.22252543640136718, | |
| 0.22253669738769533, | |
| 0.22248652648925782, | |
| 0.2225284423828125, | |
| 0.22248141479492187, | |
| 0.2225428466796875, | |
| 0.22239334106445313, | |
| 0.22253977966308594, | |
| 0.222455810546875, | |
| 0.22255616760253907, | |
| 0.22238105773925781, | |
| 0.22249063110351563, | |
| 0.2224568328857422, | |
| 0.22251930236816406, | |
| 0.22246604919433594, | |
| 0.22251417541503907, | |
| 0.22261555480957032, | |
| 0.22251417541503907, | |
| 0.22248141479492187, | |
| 0.22251315307617187, | |
| 0.22248550415039062, | |
| 0.22254591369628907, | |
| 0.22247935485839843, | |
| 0.22253773498535157, | |
| 0.2224680938720703, | |
| 0.22244044494628906, | |
| 0.22249063110351563, | |
| 0.2225305633544922, | |
| 0.222376953125, | |
| 0.2225530548095703, | |
| 0.22244761657714843, | |
| 0.22253260803222658, | |
| 0.22256640625, | |
| 0.22257049560546874, | |
| 0.22244761657714843, | |
| 0.22257562255859376, | |
| 0.22245890808105467, | |
| 0.22250390625, | |
| 0.22244248962402344, | |
| 0.22244557189941405, | |
| 0.22242816162109375, | |
| 0.22251519775390624, | |
| 0.22240255737304687 | |
| ] | |
| }, | |
| "throughput": { | |
| "unit": "tokens/s", | |
| "value": 1437.9895989986592 | |
| }, | |
| "energy": null, | |
| "efficiency": null | |
| }, | |
| "decode": { | |
| "memory": { | |
| "unit": "MB", | |
| "max_ram": 3387.24864, | |
| "max_vram": 6788.481024, | |
| "max_reserved": 6304.038912, | |
| "max_allocated": 5949.916672 | |
| }, | |
| "latency": { | |
| "unit": "s", | |
| "mean": 14.074660812377926, | |
| "stdev": 0, | |
| "values": [ | |
| 14.074660812377926 | |
| ] | |
| }, | |
| "throughput": { | |
| "unit": "tokens/s", | |
| "value": 140.67834574448102 | |
| }, | |
| "energy": null, | |
| "efficiency": null | |
| }, | |
| "per_token": { | |
| "memory": null, | |
| "latency": { | |
| "unit": "s", | |
| "mean": 0.14216829103412046, | |
| "stdev": 0.0018006602354060954, | |
| "values": [ | |
| 0.1389486083984375, | |
| 0.1390417938232422, | |
| 0.13908787536621095, | |
| 0.13919027709960938, | |
| 0.13930598449707032, | |
| 0.13928755187988281, | |
| 0.13942373657226562, | |
| 0.13951078796386718, | |
| 0.13948109436035155, | |
| 0.1396531219482422, | |
| 0.13970329284667968, | |
| 0.13980569458007813, | |
| 0.1397073974609375, | |
| 0.1398835144042969, | |
| 0.1398824920654297, | |
| 0.14009957885742189, | |
| 0.14002584838867188, | |
| 0.1401589813232422, | |
| 0.14024191284179688, | |
| 0.1404938201904297, | |
| 0.14032077026367187, | |
| 0.14050201416015626, | |
| 0.1405470733642578, | |
| 0.1406914825439453, | |
| 0.1405839080810547, | |
| 0.1407088623046875, | |
| 0.14071501159667968, | |
| 0.14100787353515626, | |
| 0.1407825927734375, | |
| 0.14108160400390626, | |
| 0.1408931884765625, | |
| 0.14117068481445313, | |
| 0.14100480651855468, | |
| 0.1412393035888672, | |
| 0.14122904968261718, | |
| 0.14163046264648438, | |
| 0.14134375, | |
| 0.14147378540039063, | |
| 0.14139903259277345, | |
| 0.14174105834960937, | |
| 0.14149221801757814, | |
| 0.14176051330566405, | |
| 0.14176972961425782, | |
| 0.14206259155273437, | |
| 0.1417205810546875, | |
| 0.14199603271484376, | |
| 0.14184857177734375, | |
| 0.1422223358154297, | |
| 0.14199090576171874, | |
| 0.14227558898925782, | |
| 0.1420963897705078, | |
| 0.1425745849609375, | |
| 0.14217318725585937, | |
| 0.14249676513671874, | |
| 0.1423574981689453, | |
| 0.1427343292236328, | |
| 0.14243431091308595, | |
| 0.1426677703857422, | |
| 0.14258688354492188, | |
| 0.1431593017578125, | |
| 0.14274662780761718, | |
| 0.1429759979248047, | |
| 0.14276608276367186, | |
| 0.14324838256835937, | |
| 0.14285618591308594, | |
| 0.14316339111328125, | |
| 0.14306201171875, | |
| 0.14356069946289063, | |
| 0.14316543579101562, | |
| 0.14338150024414062, | |
| 0.1433139190673828, | |
| 0.1437962188720703, | |
| 0.1434634246826172, | |
| 0.14380645751953125, | |
| 0.14355558776855468, | |
| 0.14418739318847656, | |
| 0.14368666076660155, | |
| 0.14406553649902343, | |
| 0.14379930114746095, | |
| 0.14422732543945313, | |
| 0.14389759826660156, | |
| 0.14445362854003907, | |
| 0.14404403686523437, | |
| 0.14465843200683592, | |
| 0.14417510986328125, | |
| 0.1444065246582031, | |
| 0.14419865417480468, | |
| 0.14534246826171876, | |
| 0.1443747863769531, | |
| 0.1446614990234375, | |
| 0.14449971008300783, | |
| 0.14513868713378905, | |
| 0.14458674621582032, | |
| 0.14490623474121095, | |
| 0.14474240112304687, | |
| 0.14531173706054687, | |
| 0.14485504150390624, | |
| 0.14515199279785157, | |
| 0.14500761413574217 | |
| ] | |
| }, | |
| "throughput": { | |
| "unit": "tokens/s", | |
| "value": 140.67834574448102 | |
| }, | |
| "energy": null, | |
| "efficiency": null | |
| } | |
| } |