benchmark_report.json

{
    "prefill": {
        "memory": {
            "unit": "MB",
            "max_ram": 1476.665344,
            "max_vram": 15674.114048,
            "max_reserved": 15189.671936,
            "max_allocated": 15039.57248
        },
        "latency": {
            "unit": "s",
            "mean": 0.040603640750171685,
            "stdev": 0.00010272403794894722,
            "values": [
                0.04150588989257813,
                0.041099262237548825,
                0.04108083343505859,
                0.041046016693115236,
                0.041059326171875,
                0.04103680038452148,
                0.041040897369384766,
                0.04108486557006836,
                0.04070195388793945,
                0.040600574493408204,
                0.04060979080200195,
                0.040619007110595705,
                0.04062617492675781,
                0.04057907104492187,
                0.04058931350708008,
                0.040612831115722656,
                0.040643585205078124,
                0.04059033584594727,
                0.04063641738891602,
                0.04063948822021484,
                0.040600574493408204,
                0.04063129425048828,
                0.040622081756591794,
                0.04060876846313476,
                0.04061491012573242,
                0.04060774230957031,
                0.040592384338378903,
                0.04062412643432617,
                0.04067225646972656,
                0.040635391235351564,
                0.04062819290161133,
                0.04060569763183594,
                0.04060467147827149,
                0.040622081756591794,
                0.04063436889648438,
                0.040602622985839845,
                0.04059033584594727,
                0.04055449676513672,
                0.040612865447998046,
                0.04063846588134765,
                0.04060467147827149,
                0.040599552154541016,
                0.04057088088989258,
                0.040578048706054685,
                0.040578048706054685,
                0.040584190368652344,
                0.040671230316162106,
                0.040592384338378903,
                0.04060671997070313,
                0.04064767837524414,
                0.04061798477172852,
                0.04059033584594727,
                0.040569854736328126,
                0.040548351287841795,
                0.04056883239746094,
                0.04058931350708008,
                0.04058931350708008,
                0.04056883239746094,
                0.040597503662109374,
                0.04053401565551758,
                0.040574977874755856,
                0.04057395172119141,
                0.04055859375,
                0.04056883239746094,
                0.04058726501464844,
                0.04060569763183594,
                0.04057088088989258,
                0.040567806243896484,
                0.04061183929443359,
                0.0406416015625,
                0.04059852981567383,
                0.040612865447998046,
                0.04062003326416016,
                0.04057292938232422,
                0.04057699203491211,
                0.0406743049621582,
                0.04055446243286133,
                0.04056268692016601,
                0.04059545516967773,
                0.04056268692016601,
                0.040591358184814456,
                0.04060467147827149,
                0.04057807922363281,
                0.04058726501464844,
                0.04063334274291992,
                0.04060979080200195,
                0.040635391235351564,
                0.04063129425048828,
                0.04056883239746094,
                0.040574977874755856,
                0.040630271911621094,
                0.040583168029785156,
                0.040597503662109374,
                0.040569854736328126,
                0.040592384338378903,
                0.040576000213623044,
                0.040594432830810545,
                0.04060569763183594,
                0.040546302795410154,
                0.04056268692016601,
                0.04057699203491211,
                0.04063334274291992,
                0.040632320404052735,
                0.04059648132324219,
                0.04056268692016601,
                0.040584190368652344,
                0.04056576156616211,
                0.04060774230957031,
                0.04058828735351563,
                0.040649665832519534,
                0.04056371307373047,
                0.04056979370117188,
                0.04055654525756836,
                0.040574977874755856,
                0.04055859375,
                0.040612865447998046,
                0.04056268692016601,
                0.040584190368652344,
                0.04058214569091797,
                0.040602622985839845,
                0.0405667839050293,
                0.040629249572753906,
                0.040581119537353515,
                0.04055551910400391,
                0.04056883239746094,
                0.0405852165222168,
                0.04058515167236328,
                0.040600574493408204,
                0.04056883239746094,
                0.04056371307373047,
                0.04059545516967773,
                0.04056268692016601,
                0.0405667839050293,
                0.04057088088989258,
                0.040548351287841795,
                0.04058828735351563,
                0.040578048706054685,
                0.040567806243896484,
                0.040583168029785156,
                0.04060467147827149,
                0.040546302795410154,
                0.0405667839050293,
                0.040586238861083986,
                0.04060160064697266,
                0.040627201080322264,
                0.04063948822021484,
                0.04062412643432617,
                0.040612865447998046,
                0.040612865447998046,
                0.040561664581298826,
                0.04056883239746094,
                0.04056268692016601,
                0.04058828735351563,
                0.04051148986816406,
                0.04062515258789062,
                0.040583168029785156,
                0.04059033584594727,
                0.040548351287841795,
                0.04057088088989258,
                0.04057190322875977,
                0.04057696151733398,
                0.04055551910400391,
                0.040635391235351564,
                0.04057292938232422,
                0.04056268692016601,
                0.04057292938232422,
                0.04055654525756836,
                0.04058726501464844,
                0.04057696151733398,
                0.0405852165222168,
                0.04055654525756836,
                0.04055859375,
                0.04058009719848633,
                0.04058726501464844,
                0.04056883239746094,
                0.040546302795410154,
                0.04052684783935547,
                0.040546302795410154,
                0.04058726501464844,
                0.04057190322875977,
                0.04058828735351563,
                0.04054832077026367,
                0.040567806243896484,
                0.04057702255249023,
                0.040567806243896484,
                0.04055859375,
                0.040546302795410154,
                0.040542209625244144,
                0.04056063842773437,
                0.04060979080200195,
                0.0405667839050293,
                0.04059340667724609,
                0.04059545516967773,
                0.040542209625244144,
                0.04054937744140625,
                0.0405401611328125,
                0.040553470611572266,
                0.04059648132324219,
                0.04059852981567383,
                0.040648704528808595,
                0.040632320404052735,
                0.04057907104492187,
                0.04060569763183594,
                0.04060569763183594,
                0.04060979080200195,
                0.04060671997070313,
                0.04057190322875977,
                0.04058931350708008,
                0.04053395080566406,
                0.040584190368652344,
                0.04057088088989258,
                0.04057699203491211,
                0.040594432830810545,
                0.04057907104492187,
                0.04057702255249023,
                0.040621055603027346,
                0.040594432830810545,
                0.040551422119140625,
                0.040548351287841795,
                0.04059648132324219,
                0.04055654525756836,
                0.04057699203491211,
                0.04053606414794922,
                0.040545280456542966,
                0.0405667839050293,
                0.04054425430297852,
                0.04057190322875977,
                0.04055039978027344,
                0.04053094482421875,
                0.040567806243896484,
                0.04059340667724609,
                0.04058310317993164,
                0.04063334274291992,
                0.040629249572753906,
                0.04058726501464844,
                0.040567806243896484,
                0.040567806243896484,
                0.04059545516967773,
                0.0405810546875,
                0.04056883239746094,
                0.040503231048583985,
                0.040548351287841795,
                0.040527809143066404,
                0.04055756759643555,
                0.04055039978027344,
                0.040545280456542966
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 788.1066675003692
        },
        "energy": null,
        "efficiency": null
    },
    "decode": {
        "memory": {
            "unit": "MB",
            "max_ram": 1487.515648,
            "max_vram": 15739.12576,
            "max_reserved": 15254.683648,
            "max_allocated": 15101.761024
        },
        "latency": {
            "unit": "s",
            "mean": 3.7061430625915524,
            "stdev": 0.0004980685782824457,
            "values": [
                3.705805835723877,
                3.705776123046874,
                3.7068472290039054
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 53.42481298105821
        },
        "energy": null,
        "efficiency": null
    },
    "per_token": {
        "memory": null,
        "latency": {
            "unit": "s",
            "mean": 0.03743578851102577,
            "stdev": 0.0001575827338807445,
            "values": [
                0.037220352172851565,
                0.03719680023193359,
                0.037154815673828126,
                0.037185535430908204,
                0.03719580841064453,
                0.03720291137695313,
                0.0371701774597168,
                0.037160961151123044,
                0.03719680023193359,
                0.03721011352539062,
                0.0372305908203125,
                0.03724288177490234,
                0.03720806503295898,
                0.03727462387084961,
                0.03731660842895508,
                0.03725721740722656,
                0.03722143936157227,
                0.03725203323364258,
                0.03722649765014648,
                0.037253120422363284,
                0.03725107192993164,
                0.037310462951660156,
                0.03722649765014648,
                0.03725823974609375,
                0.037253120422363284,
                0.03728793716430664,
                0.03727667236328125,
                0.03733401489257813,
                0.03727360153198242,
                0.03732889556884766,
                0.037340160369873046,
                0.03734732818603516,
                0.03732889556884766,
                0.03734630584716797,
                0.037343231201171875,
                0.037369857788085936,
                0.03735551834106445,
                0.03737702560424805,
                0.03736576080322265,
                0.03743436813354492,
                0.03738521575927734,
                0.03744153594970703,
                0.03738521575927734,
                0.03743129730224609,
                0.03743334579467773,
                0.037501953125,
                0.03741183853149414,
                0.03751628875732422,
                0.03740262222290039,
                0.037548030853271484,
                0.03744153594970703,
                0.03753881454467774,
                0.03740364837646484,
                0.037607425689697264,
                0.037438465118408204,
                0.03755724716186523,
                0.03743948745727539,
                0.03760537719726562,
                0.03747123336791992,
                0.03755724716186523,
                0.03745177459716797,
                0.03763507080078125,
                0.03745587158203125,
                0.037553150177001955,
                0.03747123336791992,
                0.037628929138183595,
                0.037512191772460936,
                0.037601280212402347,
                0.03750400161743164,
                0.037644287109375,
                0.03754291152954101,
                0.03761459350585938,
                0.03752447891235351,
                0.03756851196289063,
                0.0374958381652832,
                0.03754800033569336,
                0.03755417633056641,
                0.03759206390380859,
                0.037533695220947266,
                0.037634048461914066,
                0.03753062438964844,
                0.037572608947753904,
                0.03756748962402344,
                0.03757363128662109,
                0.037558273315429686,
                0.037563392639160156,
                0.037580799102783204,
                0.03767705535888672,
                0.03755929565429687,
                0.03762278366088867,
                0.0375623664855957,
                0.03762278366088867,
                0.03756851196289063,
                0.037648384094238284,
                0.03764121627807617,
                0.03772927856445313,
                0.03761356735229492,
                0.037618686676025394,
                0.03762483215332031,
                0.037149696350097655,
                0.03717836761474609,
                0.037175361633300784,
                0.03716089630126953,
                0.037166080474853515,
                0.03717836761474609,
                0.0371599349975586,
                0.03724390411376953,
                0.03721932983398438,
                0.03721318435668945,
                0.03719987106323242,
                0.03724800109863281,
                0.03720806503295898,
                0.03725107192993164,
                0.03725209426879883,
                0.03724800109863281,
                0.03721830368041992,
                0.0372592658996582,
                0.03723775863647461,
                0.037266433715820314,
                0.0372408332824707,
                0.03729715347290039,
                0.0372408332824707,
                0.037256191253662106,
                0.03726131057739258,
                0.0373043212890625,
                0.03727872085571289,
                0.03733196640014649,
                0.03728796768188476,
                0.03733808135986328,
                0.03734425735473633,
                0.037359615325927735,
                0.03730944061279297,
                0.037359615325927735,
                0.03734732818603516,
                0.037348350524902346,
                0.03739033508300781,
                0.037369857788085936,
                0.0373493766784668,
                0.03739340972900391,
                0.037354496002197264,
                0.03743539047241211,
                0.03738828659057617,
                0.03745792007446289,
                0.0374128646850586,
                0.03748659133911133,
                0.037416961669921874,
                0.03746099090576172,
                0.03741900634765625,
                0.0375551986694336,
                0.03743436813354492,
                0.03751833724975586,
                0.03741900634765625,
                0.03759820938110352,
                0.03745280075073242,
                0.037533695220947266,
                0.037473281860351565,
                0.03760332870483398,
                0.03745792007446289,
                0.037564414978027344,
                0.03747635269165039,
                0.03764019012451172,
                0.03746303939819336,
                0.037585918426513674,
                0.037473281860351565,
                0.037601280212402347,
                0.03751935958862305,
                0.03759206390380859,
                0.037515262603759765,
                0.03765657424926758,
                0.03748454284667969,
                0.0375838737487793,
                0.03751424026489258,
                0.03754598236083984,
                0.037533695220947266,
                0.037553150177001955,
                0.03751628875732422,
                0.03757363128662109,
                0.037531646728515625,
                0.0376545295715332,
                0.03754905700683594,
                0.03762278366088867,
                0.037548030853271484,
                0.03759718322753906,
                0.03759308624267578,
                0.03759308624267578,
                0.0375551986694336,
                0.037678081512451174,
                0.03755929565429687,
                0.03769139099121094,
                0.03758796691894531,
                0.037634048461914066,
                0.03760537719726562,
                0.03765555191040039,
                0.037615615844726565,
                0.037713920593261716,
                0.037615615844726565,
                0.03764121627807617,
                0.037618686676025394,
                0.03712819290161133,
                0.037184513092041016,
                0.037168128967285156,
                0.03717529678344727,
                0.037190654754638675,
                0.03719270324707031,
                0.03712921524047851,
                0.037146625518798826,
                0.03721420669555664,
                0.03722547149658203,
                0.037207038879394534,
                0.03724697494506836,
                0.03724288177490234,
                0.03741900634765625,
                0.0372305908203125,
                0.03721830368041992,
                0.03726438522338867,
                0.03726131057739258,
                0.037272575378417966,
                0.037233665466308595,
                0.03724288177490234,
                0.03729919815063477,
                0.03725721740722656,
                0.03727872085571289,
                0.03727360153198242,
                0.03733913421630859,
                0.03727667236328125,
                0.03733196640014649,
                0.03734527969360352,
                0.03731763076782227,
                0.0373309440612793,
                0.03734220886230469,
                0.037321727752685545,
                0.03733708953857422,
                0.03733606338500976,
                0.037341182708740234,
                0.03734220886230469,
                0.03740979385375977,
                0.037375999450683595,
                0.03741388702392578,
                0.03736678314208984,
                0.037416961669921874,
                0.03741900634765625,
                0.03743641662597656,
                0.037408767700195314,
                0.037515262603759765,
                0.037400577545166014,
                0.03750502395629883,
                0.03745075225830078,
                0.03757670211791992,
                0.03744153594970703,
                0.037558273315429686,
                0.03747123336791992,
                0.03767603302001953,
                0.03747225570678711,
                0.037566463470458986,
                0.03747840118408203,
                0.03760332870483398,
                0.03748556900024414,
                0.03758796691894531,
                0.03748659133911133,
                0.03763302230834961,
                0.03746201705932617,
                0.03757363128662109,
                0.037466110229492186,
                0.037615615844726565,
                0.0375654411315918,
                0.03760947036743164,
                0.0374835205078125,
                0.03766988754272461,
                0.0375203857421875,
                0.0376258544921875,
                0.037541889190673826,
                0.03754393768310547,
                0.037556224822998044,
                0.03755417633056641,
                0.037582847595214845,
                0.037579776763916016,
                0.037572608947753904,
                0.03763814544677734,
                0.03756851196289063,
                0.03764633560180664,
                0.037539840698242184,
                0.03759206390380859,
                0.03758796691894531,
                0.03760639953613281,
                0.037580799102783204,
                0.03772108840942383,
                0.037618686676025394,
                0.03765248107910156,
                0.037599231719970705,
                0.03763916778564453,
                0.03759513473510742,
                0.0376627197265625,
                0.03762073516845703,
                0.037700607299804685,
                0.0376258544921875,
                0.03765760040283203,
                0.037648384094238284
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 53.42481298105823
        },
        "energy": null,
        "efficiency": null
    }
}