| { | |
| "prefill": { | |
| "memory": { | |
| "unit": "MB", | |
| "max_ram": 3452.55936, | |
| "max_vram": 5985.271808, | |
| "max_reserved": 5500.829696, | |
| "max_allocated": 5277.432832 | |
| }, | |
| "latency": { | |
| "unit": "s", | |
| "mean": 0.14864961130478807, | |
| "stdev": 0.00020932099247040743, | |
| "values": [ | |
| 0.1501033935546875, | |
| 0.14921522521972655, | |
| 0.14923980712890625, | |
| 0.1486120910644531, | |
| 0.1485875244140625, | |
| 0.14857522583007812, | |
| 0.1486120910644531, | |
| 0.14858956909179688, | |
| 0.14857626342773436, | |
| 0.1485946807861328, | |
| 0.1486069793701172, | |
| 0.14863360595703126, | |
| 0.14859980773925782, | |
| 0.14859059143066405, | |
| 0.1485803527832031, | |
| 0.1486049346923828, | |
| 0.1485701141357422, | |
| 0.1487626190185547, | |
| 0.1486182403564453, | |
| 0.14862950134277345, | |
| 0.14853631591796876, | |
| 0.14859056091308595, | |
| 0.14861619567871093, | |
| 0.14861004638671876, | |
| 0.14859776306152345, | |
| 0.14856396484375, | |
| 0.1485875244140625, | |
| 0.1485865020751953, | |
| 0.148600830078125, | |
| 0.1486254119873047, | |
| 0.14860389709472657, | |
| 0.14863462829589844, | |
| 0.14862643432617187, | |
| 0.14873599243164062, | |
| 0.1486131134033203, | |
| 0.14856806945800782, | |
| 0.14859263610839843, | |
| 0.14857830810546874, | |
| 0.14856704711914062, | |
| 0.14861415100097655, | |
| 0.14859878540039062, | |
| 0.14859980773925782, | |
| 0.1486018524169922, | |
| 0.14859980773925782, | |
| 0.14864076232910156, | |
| 0.14862028503417968, | |
| 0.14859878540039062, | |
| 0.14859674072265625, | |
| 0.14863871765136719, | |
| 0.14862232971191405, | |
| 0.148600830078125, | |
| 0.1486684112548828, | |
| 0.14858854675292968, | |
| 0.14857113647460937, | |
| 0.14853018188476563, | |
| 0.14864178466796876, | |
| 0.1486182403564453, | |
| 0.14864582824707032, | |
| 0.14861106872558594, | |
| 0.14861619567871093, | |
| 0.14862950134277345, | |
| 0.148642822265625, | |
| 0.14871859741210938, | |
| 0.14860389709472657, | |
| 0.1486069793701172, | |
| 0.1486192626953125, | |
| 0.14860389709472657, | |
| 0.1485865020751953 | |
| ] | |
| }, | |
| "throughput": { | |
| "unit": "tokens/s", | |
| "value": 538.1783329118141 | |
| }, | |
| "energy": null, | |
| "efficiency": null | |
| }, | |
| "decode": { | |
| "memory": { | |
| "unit": "MB", | |
| "max_ram": 3452.55936, | |
| "max_vram": 6153.043968, | |
| "max_reserved": 5668.601856, | |
| "max_allocated": 5461.697536 | |
| }, | |
| "latency": { | |
| "unit": "s", | |
| "mean": 13.483782165527336, | |
| "stdev": 0, | |
| "values": [ | |
| 13.483782165527336 | |
| ] | |
| }, | |
| "throughput": { | |
| "unit": "tokens/s", | |
| "value": 36.71076808593942 | |
| }, | |
| "energy": null, | |
| "efficiency": null | |
| }, | |
| "per_token": { | |
| "memory": null, | |
| "latency": { | |
| "unit": "s", | |
| "mean": 0.13619981985381147, | |
| "stdev": 0.000452258433880561, | |
| "values": [ | |
| 0.1354967041015625, | |
| 0.13573324584960939, | |
| 0.1356011505126953, | |
| 0.13553663635253907, | |
| 0.13552537536621093, | |
| 0.13557145690917968, | |
| 0.13561138916015625, | |
| 0.13565440368652343, | |
| 0.13559808349609376, | |
| 0.13561549377441406, | |
| 0.13564723205566406, | |
| 0.13561549377441406, | |
| 0.13562777709960938, | |
| 0.13564723205566406, | |
| 0.1356759033203125, | |
| 0.13565338134765625, | |
| 0.13569126892089844, | |
| 0.1357127685546875, | |
| 0.13578956604003906, | |
| 0.1358008270263672, | |
| 0.13584281921386718, | |
| 0.1357445068359375, | |
| 0.13585305786132812, | |
| 0.13578034973144532, | |
| 0.13579673767089845, | |
| 0.13576089477539063, | |
| 0.13585101318359374, | |
| 0.1358551025390625, | |
| 0.13593190002441408, | |
| 0.13583258056640626, | |
| 0.13583973693847656, | |
| 0.1358561248779297, | |
| 0.13588275146484374, | |
| 0.1358233642578125, | |
| 0.13594419860839843, | |
| 0.135878662109375, | |
| 0.135947265625, | |
| 0.13588275146484374, | |
| 0.13594522094726563, | |
| 0.13590322875976563, | |
| 0.13600460815429688, | |
| 0.13590118408203125, | |
| 0.1360230407714844, | |
| 0.13603532409667968, | |
| 0.13604454040527345, | |
| 0.13600973510742187, | |
| 0.13606297302246093, | |
| 0.13602508544921876, | |
| 0.1360885772705078, | |
| 0.13615513610839844, | |
| 0.1361459197998047, | |
| 0.13616128540039063, | |
| 0.13614898681640625, | |
| 0.13620223999023437, | |
| 0.13623193359375, | |
| 0.13624114990234376, | |
| 0.13626162719726562, | |
| 0.13628416442871094, | |
| 0.13632511901855468, | |
| 0.13637017822265626, | |
| 0.13639474487304687, | |
| 0.13638758850097657, | |
| 0.1363783721923828, | |
| 0.1364602813720703, | |
| 0.13640396118164064, | |
| 0.13648486328125, | |
| 0.13642547607421876, | |
| 0.13651968383789062, | |
| 0.13646234130859375, | |
| 0.13649305725097657, | |
| 0.1365708770751953, | |
| 0.13669888305664063, | |
| 0.13656474304199218, | |
| 0.1365606384277344, | |
| 0.13659341430664063, | |
| 0.1367060546875, | |
| 0.13661900329589843, | |
| 0.13661900329589843, | |
| 0.13664767456054688, | |
| 0.13676646423339844, | |
| 0.13667225646972656, | |
| 0.13671218872070312, | |
| 0.13672550964355468, | |
| 0.13682073974609374, | |
| 0.13676133728027343, | |
| 0.1367316436767578, | |
| 0.1367736358642578, | |
| 0.1369169921875, | |
| 0.1368299560546875, | |
| 0.1367930908203125, | |
| 0.13684326171875, | |
| 0.13699276733398438, | |
| 0.13689138793945313, | |
| 0.13688934326171875, | |
| 0.13688217163085936, | |
| 0.13704806518554688, | |
| 0.1370408935546875, | |
| 0.13696000671386718, | |
| 0.137059326171875 | |
| ] | |
| }, | |
| "throughput": { | |
| "unit": "tokens/s", | |
| "value": 36.71076808593942 | |
| }, | |
| "energy": null, | |
| "efficiency": null | |
| } | |
| } |