| { | |
| "prefill": { | |
| "memory": { | |
| "unit": "MB", | |
| "max_ram": 3520.331776, | |
| "max_vram": 6002.049024, | |
| "max_reserved": 5519.704064, | |
| "max_allocated": 5308.375552 | |
| }, | |
| "latency": { | |
| "unit": "s", | |
| "mean": 0.17722602710389262, | |
| "stdev": 0.0003684363070144512, | |
| "values": [ | |
| 0.17861517333984375, | |
| 0.17795379638671874, | |
| 0.1778217010498047, | |
| 0.1769144287109375, | |
| 0.1771356201171875, | |
| 0.17695539855957032, | |
| 0.17753497314453126, | |
| 0.17694309997558594, | |
| 0.17741209411621095, | |
| 0.17667686462402343, | |
| 0.17742745971679688, | |
| 0.17694003295898436, | |
| 0.17742950439453126, | |
| 0.17683456420898438, | |
| 0.17743359375, | |
| 0.17688986206054688, | |
| 0.17742124938964843, | |
| 0.1767392578125, | |
| 0.17742233276367186, | |
| 0.17670758056640626, | |
| 0.17770803833007812, | |
| 0.17685810852050782, | |
| 0.17736090087890624, | |
| 0.17717042541503905, | |
| 0.1776558074951172, | |
| 0.17685504150390624, | |
| 0.1772881622314453, | |
| 0.17695846557617187, | |
| 0.1773107147216797, | |
| 0.17689088439941406, | |
| 0.1773690948486328, | |
| 0.17696461486816406, | |
| 0.1777223663330078, | |
| 0.1769738311767578, | |
| 0.1773506622314453, | |
| 0.17685606384277344, | |
| 0.17758822631835938, | |
| 0.1770250244140625, | |
| 0.17728102111816407, | |
| 0.17687449645996095, | |
| 0.17724928283691407, | |
| 0.17697587585449218, | |
| 0.17725337219238282, | |
| 0.17689701843261718, | |
| 0.17727999877929687, | |
| 0.1769144287109375, | |
| 0.17742437744140624, | |
| 0.17705471801757813, | |
| 0.1775431671142578, | |
| 0.17717042541503905, | |
| 0.17785133361816408, | |
| 0.17692466735839843, | |
| 0.17750834655761719, | |
| 0.17665638732910155, | |
| 0.17751962280273437, | |
| 0.177154052734375, | |
| 0.1772359313964844 | |
| ] | |
| }, | |
| "throughput": { | |
| "unit": "tokens/s", | |
| "value": 1354.202900792378 | |
| }, | |
| "energy": null, | |
| "efficiency": null | |
| }, | |
| "decode": { | |
| "memory": { | |
| "unit": "MB", | |
| "max_ram": 3520.331776, | |
| "max_vram": 6524.239872, | |
| "max_reserved": 6039.79776, | |
| "max_allocated": 5799.174656 | |
| }, | |
| "latency": { | |
| "unit": "s", | |
| "mean": 13.740053451538087, | |
| "stdev": 0, | |
| "values": [ | |
| 13.740053451538087 | |
| ] | |
| }, | |
| "throughput": { | |
| "unit": "tokens/s", | |
| "value": 108.07818217284783 | |
| }, | |
| "energy": null, | |
| "efficiency": null | |
| }, | |
| "per_token": { | |
| "memory": null, | |
| "latency": { | |
| "unit": "s", | |
| "mean": 0.1387884187024049, | |
| "stdev": 0.0013267860731352012, | |
| "values": [ | |
| 0.13642649841308593, | |
| 0.13647360229492186, | |
| 0.13654937744140624, | |
| 0.13651148986816405, | |
| 0.1366138916015625, | |
| 0.13663848876953125, | |
| 0.13677769470214843, | |
| 0.1366947784423828, | |
| 0.13682073974609374, | |
| 0.13688217163085936, | |
| 0.13693235778808593, | |
| 0.13699789428710937, | |
| 0.13699685668945313, | |
| 0.13709721374511719, | |
| 0.13711871337890624, | |
| 0.13721804809570312, | |
| 0.1373737030029297, | |
| 0.1373245391845703, | |
| 0.13739007568359374, | |
| 0.13745050048828125, | |
| 0.13745869445800782, | |
| 0.1375283203125, | |
| 0.13755084228515624, | |
| 0.13769215393066406, | |
| 0.1376645050048828, | |
| 0.13773823547363281, | |
| 0.13773414611816406, | |
| 0.13794610595703125, | |
| 0.13788569641113282, | |
| 0.13792562866210936, | |
| 0.13789082336425781, | |
| 0.1381273651123047, | |
| 0.1379809265136719, | |
| 0.13817855834960938, | |
| 0.13799935913085937, | |
| 0.138429443359375, | |
| 0.13814271545410156, | |
| 0.1382645721435547, | |
| 0.1381898193359375, | |
| 0.13846424865722656, | |
| 0.1383741455078125, | |
| 0.13847450256347657, | |
| 0.13849087524414064, | |
| 0.13870285034179688, | |
| 0.13860453796386718, | |
| 0.13863935852050782, | |
| 0.13871615600585938, | |
| 0.13895167541503906, | |
| 0.138682373046875, | |
| 0.13885952758789064, | |
| 0.13884416198730468, | |
| 0.13912985229492186, | |
| 0.1389475860595703, | |
| 0.13897421264648438, | |
| 0.13886361694335939, | |
| 0.1391575012207031, | |
| 0.13888819885253906, | |
| 0.139219970703125, | |
| 0.1390766143798828, | |
| 0.13939712524414063, | |
| 0.1390766143798828, | |
| 0.13926194763183594, | |
| 0.1392025604248047, | |
| 0.13958041381835937, | |
| 0.13937254333496094, | |
| 0.13950770568847656, | |
| 0.13935206604003905, | |
| 0.13972685241699218, | |
| 0.13947698974609374, | |
| 0.1397012481689453, | |
| 0.13956813049316405, | |
| 0.13995928955078124, | |
| 0.1400432586669922, | |
| 0.14021530151367187, | |
| 0.13979852294921874, | |
| 0.1405470733642578, | |
| 0.13980979919433595, | |
| 0.14013644409179687, | |
| 0.13998489379882811, | |
| 0.14032485961914062, | |
| 0.14011903381347657, | |
| 0.14025932312011719, | |
| 0.14017330932617186, | |
| 0.14061669921875, | |
| 0.1402388458251953, | |
| 0.14044671630859376, | |
| 0.14032485961914062, | |
| 0.14084915161132813, | |
| 0.14041702270507814, | |
| 0.14065151977539062, | |
| 0.14049484252929687, | |
| 0.14099763488769532, | |
| 0.14055731201171875, | |
| 0.1408501739501953, | |
| 0.14065254211425782, | |
| 0.14113177490234374, | |
| 0.14070375061035156, | |
| 0.141011962890625, | |
| 0.14083482360839844 | |
| ] | |
| }, | |
| "throughput": { | |
| "unit": "tokens/s", | |
| "value": 108.07818217284785 | |
| }, | |
| "energy": null, | |
| "efficiency": null | |
| } | |
| } |