| { | |
| "prefill": { | |
| "memory": { | |
| "unit": "MB", | |
| "max_ram": 3482.791936, | |
| "max_vram": 5991.563264, | |
| "max_reserved": 5507.121152, | |
| "max_allocated": 5293.886976 | |
| }, | |
| "latency": { | |
| "unit": "s", | |
| "mean": 0.16423192008596946, | |
| "stdev": 0.00019889040182647096, | |
| "values": [ | |
| 0.16545770263671875, | |
| 0.16465306091308593, | |
| 0.16468992614746095, | |
| 0.1641175079345703, | |
| 0.16409703063964845, | |
| 0.16402227783203124, | |
| 0.16426803588867187, | |
| 0.16409088134765626, | |
| 0.16417587280273438, | |
| 0.16401817321777343, | |
| 0.16415744018554687, | |
| 0.1639772186279297, | |
| 0.16430899047851563, | |
| 0.1642670135498047, | |
| 0.16420556640625, | |
| 0.164173828125, | |
| 0.16428749084472657, | |
| 0.16417485046386718, | |
| 0.16405810546875, | |
| 0.16431922912597657, | |
| 0.1642854461669922, | |
| 0.1642117156982422, | |
| 0.16413594055175781, | |
| 0.16416050720214845, | |
| 0.16413900756835936, | |
| 0.1642239990234375, | |
| 0.1640396728515625, | |
| 0.1641502685546875, | |
| 0.16422808837890626, | |
| 0.16410726928710936, | |
| 0.16409599304199218, | |
| 0.1642045440673828, | |
| 0.1640273895263672, | |
| 0.1641318359375, | |
| 0.16424960327148438, | |
| 0.16427008056640624, | |
| 0.1641994171142578, | |
| 0.16424858093261718, | |
| 0.1642854461669922, | |
| 0.16418406677246095, | |
| 0.1642434539794922, | |
| 0.1641932830810547, | |
| 0.16427008056640624, | |
| 0.16417689514160155, | |
| 0.16429977416992186, | |
| 0.1642608642578125, | |
| 0.16433766174316405, | |
| 0.164168701171875, | |
| 0.16428749084472657, | |
| 0.16426495361328125, | |
| 0.16429466247558594, | |
| 0.16426188659667967, | |
| 0.16419635009765626, | |
| 0.16415437316894532, | |
| 0.16411648559570313, | |
| 0.1641430969238281, | |
| 0.1642229766845703, | |
| 0.1643550720214844, | |
| 0.16430899047851563, | |
| 0.16421682739257812, | |
| 0.164274169921875 | |
| ] | |
| }, | |
| "throughput": { | |
| "unit": "tokens/s", | |
| "value": 974.2320488991775 | |
| }, | |
| "energy": null, | |
| "efficiency": null | |
| }, | |
| "decode": { | |
| "memory": { | |
| "unit": "MB", | |
| "max_ram": 3486.969856, | |
| "max_vram": 6394.216448, | |
| "max_reserved": 5909.774336, | |
| "max_allocated": 5607.824384 | |
| }, | |
| "latency": { | |
| "unit": "s", | |
| "mean": 13.628475357055665, | |
| "stdev": 0, | |
| "values": [ | |
| 13.628475357055665 | |
| ] | |
| }, | |
| "throughput": { | |
| "unit": "tokens/s", | |
| "value": 72.64202150737736 | |
| }, | |
| "energy": null, | |
| "efficiency": null | |
| }, | |
| "per_token": { | |
| "memory": null, | |
| "latency": { | |
| "unit": "s", | |
| "mean": 0.13766136724298653, | |
| "stdev": 0.0010905201659738928, | |
| "values": [ | |
| 0.1359298553466797, | |
| 0.13591552734375, | |
| 0.13594009399414062, | |
| 0.1359667205810547, | |
| 0.13600460815429688, | |
| 0.13601895141601564, | |
| 0.13605580139160156, | |
| 0.13604658508300782, | |
| 0.13616741943359376, | |
| 0.13620632934570312, | |
| 0.13625753784179687, | |
| 0.13625447082519532, | |
| 0.136195068359375, | |
| 0.13614387512207032, | |
| 0.13626264953613282, | |
| 0.13613772583007813, | |
| 0.13635993957519532, | |
| 0.13644082641601563, | |
| 0.1364346923828125, | |
| 0.13646847534179687, | |
| 0.13652479553222657, | |
| 0.13652070617675782, | |
| 0.13655039978027345, | |
| 0.13660263061523437, | |
| 0.13669888305664063, | |
| 0.1367736358642578, | |
| 0.13674803161621094, | |
| 0.13675418090820313, | |
| 0.13672857666015625, | |
| 0.13689036560058593, | |
| 0.13687397766113282, | |
| 0.13697433471679688, | |
| 0.1370214385986328, | |
| 0.13715968322753908, | |
| 0.13709925842285156, | |
| 0.13714533996582032, | |
| 0.1371463623046875, | |
| 0.13719244384765625, | |
| 0.13717607116699218, | |
| 0.13731327819824218, | |
| 0.1372692413330078, | |
| 0.13724057006835938, | |
| 0.13735423278808595, | |
| 0.13743923950195314, | |
| 0.1373849639892578, | |
| 0.13742898559570313, | |
| 0.13749760437011718, | |
| 0.13763481140136719, | |
| 0.13762355041503907, | |
| 0.13764198303222655, | |
| 0.13764198303222655, | |
| 0.13776690673828124, | |
| 0.13771366882324218, | |
| 0.1378979797363281, | |
| 0.13785189819335938, | |
| 0.1379737548828125, | |
| 0.137923583984375, | |
| 0.13800653076171876, | |
| 0.13797171020507812, | |
| 0.1381898193359375, | |
| 0.13807923889160156, | |
| 0.13827072143554686, | |
| 0.13827687072753905, | |
| 0.1384796142578125, | |
| 0.13831065368652343, | |
| 0.13842124938964845, | |
| 0.13827378845214844, | |
| 0.13860453796386718, | |
| 0.13834854125976562, | |
| 0.138503173828125, | |
| 0.1383936004638672, | |
| 0.13875404357910157, | |
| 0.13848165893554687, | |
| 0.13869363403320312, | |
| 0.13860659790039062, | |
| 0.1388748779296875, | |
| 0.13853286743164062, | |
| 0.13878886413574218, | |
| 0.13861477661132812, | |
| 0.13890354919433595, | |
| 0.13877760314941406, | |
| 0.13896397399902344, | |
| 0.13888716125488282, | |
| 0.13914215087890625, | |
| 0.13898136901855468, | |
| 0.13904998779296876, | |
| 0.13895167541503906, | |
| 0.139293701171875, | |
| 0.13901107788085937, | |
| 0.1390970916748047, | |
| 0.1390909423828125, | |
| 0.13933363342285157, | |
| 0.1391513671875, | |
| 0.13932237243652343, | |
| 0.13918821716308594, | |
| 0.13949542236328125, | |
| 0.13923225402832032, | |
| 0.1394923553466797, | |
| 0.13924557495117187 | |
| ] | |
| }, | |
| "throughput": { | |
| "unit": "tokens/s", | |
| "value": 72.64202150737735 | |
| }, | |
| "energy": null, | |
| "efficiency": null | |
| } | |
| } |