| { | |
| "prefill": { | |
| "memory": { | |
| "unit": "MB", | |
| "max_ram": 3443.720192, | |
| "max_vram": 6025.117696, | |
| "max_reserved": 5540.675584, | |
| "max_allocated": 5388.153344 | |
| }, | |
| "latency": { | |
| "unit": "s", | |
| "mean": 0.2740576312090899, | |
| "stdev": 0.0010989146381610653, | |
| "values": [ | |
| 0.28063232421875, | |
| 0.2741053466796875, | |
| 0.27385037231445314, | |
| 0.27369778442382814, | |
| 0.27390872192382815, | |
| 0.27380020141601563, | |
| 0.2738739318847656, | |
| 0.2738114624023438, | |
| 0.2738688049316406, | |
| 0.2738595886230469, | |
| 0.27389132690429685, | |
| 0.273786865234375, | |
| 0.27388314819335935, | |
| 0.27380224609375, | |
| 0.2739517517089844, | |
| 0.2738011474609375, | |
| 0.27385855102539064, | |
| 0.273807373046875, | |
| 0.2739568786621094, | |
| 0.2738319396972656, | |
| 0.27399679565429685, | |
| 0.27385549926757813, | |
| 0.2739507141113281, | |
| 0.2738472900390625, | |
| 0.2739578857421875, | |
| 0.2738472900390625, | |
| 0.2740223999023437, | |
| 0.27382785034179685, | |
| 0.27391180419921873, | |
| 0.27378176879882815, | |
| 0.2739169311523437, | |
| 0.27378994750976565, | |
| 0.273944580078125, | |
| 0.2737991638183594, | |
| 0.27390054321289065, | |
| 0.2737684631347656, | |
| 0.27403366088867187 | |
| ] | |
| }, | |
| "throughput": { | |
| "unit": "tokens/s", | |
| "value": 2043.3658334175445 | |
| }, | |
| "energy": null, | |
| "efficiency": null | |
| }, | |
| "decode": { | |
| "memory": { | |
| "unit": "MB", | |
| "max_ram": 3447.865344, | |
| "max_vram": 7134.511104, | |
| "max_reserved": 6650.068992, | |
| "max_allocated": 6483.66848 | |
| }, | |
| "latency": { | |
| "unit": "s", | |
| "mean": 14.578696182250976, | |
| "stdev": 0, | |
| "values": [ | |
| 14.578696182250976 | |
| ] | |
| }, | |
| "throughput": { | |
| "unit": "tokens/s", | |
| "value": 237.67557514632273 | |
| }, | |
| "energy": null, | |
| "efficiency": null | |
| }, | |
| "per_token": { | |
| "memory": null, | |
| "latency": { | |
| "unit": "s", | |
| "mean": 0.1472595573964745, | |
| "stdev": 0.0031237285892666312, | |
| "values": [ | |
| 0.1419008026123047, | |
| 0.14224179077148438, | |
| 0.14238514709472655, | |
| 0.14256536865234376, | |
| 0.14229913330078126, | |
| 0.14246298217773437, | |
| 0.14251315307617188, | |
| 0.14276914978027344, | |
| 0.14279986572265624, | |
| 0.14289202880859375, | |
| 0.14289715576171874, | |
| 0.14330880737304688, | |
| 0.14314802551269531, | |
| 0.14349005126953124, | |
| 0.14334976196289062, | |
| 0.1436846008300781, | |
| 0.14350335693359376, | |
| 0.14384844970703126, | |
| 0.14376243591308593, | |
| 0.14424575805664064, | |
| 0.14396723937988282, | |
| 0.1442734069824219, | |
| 0.14417510986328125, | |
| 0.14462669372558593, | |
| 0.14453248596191406, | |
| 0.14464921569824218, | |
| 0.1446051788330078, | |
| 0.14509056091308595, | |
| 0.14469223022460936, | |
| 0.1450557403564453, | |
| 0.14497074890136719, | |
| 0.14546739196777345, | |
| 0.1451304931640625, | |
| 0.1454776306152344, | |
| 0.14529330444335936, | |
| 0.14594355773925782, | |
| 0.1455503387451172, | |
| 0.1457960968017578, | |
| 0.14573670959472657, | |
| 0.14628659057617188, | |
| 0.14597427368164062, | |
| 0.1463009338378906, | |
| 0.14620672607421875, | |
| 0.1467893829345703, | |
| 0.14639718627929688, | |
| 0.1467310028076172, | |
| 0.14675558471679687, | |
| 0.14733106994628906, | |
| 0.14694400024414062, | |
| 0.147272705078125, | |
| 0.1471805419921875, | |
| 0.1478481903076172, | |
| 0.14733517456054687, | |
| 0.14773043823242188, | |
| 0.1476259765625, | |
| 0.14830181884765625, | |
| 0.1478604736328125, | |
| 0.14818508911132813, | |
| 0.14819941711425783, | |
| 0.1488773193359375, | |
| 0.14832127380371093, | |
| 0.1486499786376953, | |
| 0.14849740600585937, | |
| 0.14922547912597656, | |
| 0.14875135803222655, | |
| 0.14909542846679688, | |
| 0.1489582061767578, | |
| 0.14968421936035156, | |
| 0.14915072631835938, | |
| 0.1494978485107422, | |
| 0.1493237762451172, | |
| 0.1501644744873047, | |
| 0.14953471374511718, | |
| 0.14998527526855468, | |
| 0.14990028381347656, | |
| 0.15070719909667968, | |
| 0.14999142456054687, | |
| 0.15040409851074218, | |
| 0.15021157836914062, | |
| 0.15108096313476563, | |
| 0.1504102325439453, | |
| 0.15087820434570312, | |
| 0.15066316223144532, | |
| 0.1515888671875, | |
| 0.150935546875, | |
| 0.15129087829589843, | |
| 0.1512499237060547, | |
| 0.15167181396484375, | |
| 0.1510123596191406, | |
| 0.15173324584960937, | |
| 0.1517701110839844, | |
| 0.15265895080566405, | |
| 0.15192063903808595, | |
| 0.15193702697753905, | |
| 0.15167999267578125, | |
| 0.15261184692382812, | |
| 0.15188172912597656, | |
| 0.15233331298828126, | |
| 0.15209677124023438 | |
| ] | |
| }, | |
| "throughput": { | |
| "unit": "tokens/s", | |
| "value": 237.67557514632273 | |
| }, | |
| "energy": null, | |
| "efficiency": null | |
| } | |
| } |