benchmark_report.json

{
    "prefill": {
        "memory": {
            "unit": "MB",
            "max_ram": 1476.096,
            "max_vram": 15674.114048,
            "max_reserved": 15189.671936,
            "max_allocated": 15039.57248
        },
        "latency": {
            "unit": "s",
            "mean": 0.04060129179605625,
            "stdev": 9.672596162009366e-05,
            "values": [
                0.041347713470458985,
                0.04112588882446289,
                0.041048000335693356,
                0.04107161712646484,
                0.04105420684814453,
                0.041047039031982424,
                0.0409354248046875,
                0.04108697509765625,
                0.04071731185913086,
                0.04058313751220703,
                0.040600574493408204,
                0.04060876846313476,
                0.04061183929443359,
                0.04065280151367188,
                0.04060774230957031,
                0.04060467147827149,
                0.04065788650512695,
                0.040602622985839845,
                0.040671230316162106,
                0.04064255905151367,
                0.040586238861083986,
                0.04059340667724609,
                0.040615936279296876,
                0.04061491012573242,
                0.040619007110595705,
                0.040574977874755856,
                0.04060979080200195,
                0.040622081756591794,
                0.04063334274291992,
                0.040600574493408204,
                0.04058009719848633,
                0.04061183929443359,
                0.04056979370117188,
                0.04060467147827149,
                0.040637439727783206,
                0.04064761734008789,
                0.04058214569091797,
                0.04057292938232422,
                0.04061183929443359,
                0.04064665603637695,
                0.04059545516967773,
                0.040635391235351564,
                0.04059648132324219,
                0.04058726501464844,
                0.04059033584594727,
                0.040561664581298826,
                0.04064972686767578,
                0.04053606414794922,
                0.040583168029785156,
                0.04058214569091797,
                0.04063334274291992,
                0.040594432830810545,
                0.04059648132324219,
                0.04056268692016601,
                0.04055859375,
                0.040561664581298826,
                0.04059033584594727,
                0.04055859375,
                0.040591358184814456,
                0.0405032958984375,
                0.040574977874755856,
                0.04058214569091797,
                0.04057088088989258,
                0.040578048706054685,
                0.04059743881225586,
                0.04061798477172852,
                0.04056576156616211,
                0.04056063842773437,
                0.040583168029785156,
                0.040613887786865234,
                0.0406036491394043,
                0.04060361480712891,
                0.04062822341918945,
                0.04056576156616211,
                0.04060876846313476,
                0.040627201080322264,
                0.04058009719848633,
                0.04056371307373047,
                0.04057395172119141,
                0.04058009719848633,
                0.04057702255249023,
                0.04061695861816406,
                0.04060569763183594,
                0.04060671997070313,
                0.04057696151733398,
                0.0405852165222168,
                0.040613887786865234,
                0.04061798477172852,
                0.040574977874755856,
                0.04056979370117188,
                0.04062003326416016,
                0.04057286453247071,
                0.040592384338378903,
                0.040567806243896484,
                0.040583168029785156,
                0.04058726501464844,
                0.040576000213623044,
                0.040613887786865234,
                0.040597503662109374,
                0.04057907104492187,
                0.040545280456542966,
                0.04060460662841797,
                0.04062003326416016,
                0.040602622985839845,
                0.04060774230957031,
                0.040581119537353515,
                0.040574977874755856,
                0.04057907104492187,
                0.040653823852539066,
                0.040610816955566405,
                0.04060160064697266,
                0.04058828735351563,
                0.040542209625244144,
                0.04058726501464844,
                0.040548351287841795,
                0.040610816955566405,
                0.04054937744140625,
                0.04057702255249023,
                0.040591358184814456,
                0.04059648132324219,
                0.040581119537353515,
                0.040622081756591794,
                0.04060876846313476,
                0.040548351287841795,
                0.040576000213623044,
                0.04056268692016601,
                0.040574977874755856,
                0.0405667839050293,
                0.040599552154541016,
                0.040569854736328126,
                0.04058726501464844,
                0.04056268692016601,
                0.040592384338378903,
                0.04055859375,
                0.040581119537353515,
                0.040561664581298826,
                0.04055756759643555,
                0.040574977874755856,
                0.04058009719848633,
                0.040610816955566405,
                0.040548351287841795,
                0.04053807830810547,
                0.04056063842773437,
                0.040586238861083986,
                0.04063334274291992,
                0.04060774230957031,
                0.04062515258789062,
                0.0406036491394043,
                0.040632320404052735,
                0.040597503662109374,
                0.04059545516967773,
                0.040600574493408204,
                0.040584190368652344,
                0.04053094482421875,
                0.040612865447998046,
                0.04055654525756836,
                0.04058931350708008,
                0.04054937744140625,
                0.040567806243896484,
                0.04057292938232422,
                0.04058214569091797,
                0.04057702255249023,
                0.04064153671264648,
                0.0406036491394043,
                0.04054732894897461,
                0.040581119537353515,
                0.04058009719848633,
                0.04056883239746094,
                0.04060876846313476,
                0.04058214569091797,
                0.04057596969604492,
                0.0405667839050293,
                0.040610816955566405,
                0.04056576156616211,
                0.04053504180908203,
                0.04054118347167969,
                0.04053401565551758,
                0.04052377700805664,
                0.040548351287841795,
                0.04057395172119141,
                0.04058931350708008,
                0.04057702255249023,
                0.04056576156616211,
                0.04055756759643555,
                0.04057187271118164,
                0.04055859375,
                0.040548351287841795,
                0.040574977874755856,
                0.04052070236206055,
                0.04059545516967773,
                0.040569854736328126,
                0.04055449676513672,
                0.04060979080200195,
                0.04057702255249023,
                0.040528896331787106,
                0.04055244827270508,
                0.04055756759643555,
                0.04058009719848633,
                0.040619007110595705,
                0.040645633697509766,
                0.040621055603027346,
                0.04058726501464844,
                0.040600574493408204,
                0.04055039978027344,
                0.04062822341918945,
                0.04060467147827149,
                0.040599552154541016,
                0.040548351287841795,
                0.040569854736328126,
                0.04057292938232422,
                0.040610816955566405,
                0.040564735412597655,
                0.04056371307373047,
                0.04055756759643555,
                0.04058828735351563,
                0.040583168029785156,
                0.04059033584594727,
                0.04053913497924805,
                0.04057292938232422,
                0.040564735412597655,
                0.04054732894897461,
                0.040583168029785156,
                0.04061491012573242,
                0.04053094482421875,
                0.04053811264038086,
                0.04057395172119141,
                0.0405667839050293,
                0.04052275085449219,
                0.04059033584594727,
                0.04058828735351563,
                0.04058009719848633,
                0.04058726501464844,
                0.04062412643432617,
                0.040621055603027346,
                0.040602622985839845,
                0.040564735412597655,
                0.04057088088989258,
                0.04057190322875977,
                0.040586238861083986,
                0.0405401611328125,
                0.040497150421142575,
                0.04051251220703125,
                0.040525825500488284,
                0.04055756759643555,
                0.040529918670654294,
                0.04052787017822266
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 788.1522627589961
        },
        "energy": null,
        "efficiency": null
    },
    "decode": {
        "memory": {
            "unit": "MB",
            "max_ram": 1487.01184,
            "max_vram": 15739.12576,
            "max_reserved": 15254.683648,
            "max_allocated": 15101.761024
        },
        "latency": {
            "unit": "s",
            "mean": 3.7079623514811195,
            "stdev": 0.0005176731659298959,
            "values": [
                3.7086382026672364,
                3.707380710601807,
                3.707868141174314
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 53.39860042562468
        },
        "energy": null,
        "efficiency": null
    },
    "per_token": {
        "memory": null,
        "latency": {
            "unit": "s",
            "mean": 0.037454165166475956,
            "stdev": 0.00015713925002910255,
            "values": [
                0.037207038879394534,
                0.037171199798583986,
                0.03721113586425781,
                0.037184513092041016,
                0.03719372940063476,
                0.03722547149658203,
                0.03721625518798828,
                0.03726028823852539,
                0.03726028823852539,
                0.03727974319458008,
                0.037266433715820314,
                0.0372674560546875,
                0.037272575378417966,
                0.03726335906982422,
                0.03728793716430664,
                0.037275646209716795,
                0.03726028823852539,
                0.03727667236328125,
                0.03729817581176758,
                0.03728793716430664,
                0.03729612731933594,
                0.03729817581176758,
                0.037256191253662106,
                0.03727667236328125,
                0.03727155303955078,
                0.037303295135498044,
                0.03732889556884766,
                0.03736883163452148,
                0.037285888671875,
                0.03735859298706055,
                0.03734425735473633,
                0.03736883163452148,
                0.03738009643554688,
                0.03736883163452148,
                0.03738214492797851,
                0.03736064147949219,
                0.03738828659057617,
                0.03740467071533203,
                0.03735756683349609,
                0.03747020721435547,
                0.03737497711181641,
                0.03745075225830078,
                0.037395454406738284,
                0.03743743896484375,
                0.03741183853149414,
                0.037459968566894535,
                0.03744768142700195,
                0.03748863983154297,
                0.03743948745727539,
                0.037591041564941405,
                0.037482494354248046,
                0.037566463470458986,
                0.03745280075073242,
                0.03761356735229492,
                0.03747635269165039,
                0.03759820938110352,
                0.03749990463256836,
                0.03762176132202148,
                0.037495807647705076,
                0.037577728271484374,
                0.0374917106628418,
                0.03764121627807617,
                0.037512191772460936,
                0.03765350341796875,
                0.037498878479003905,
                0.03764019012451172,
                0.037525505065917966,
                0.037645313262939455,
                0.03755929565429687,
                0.037664768218994144,
                0.03758694458007812,
                0.037601280212402347,
                0.03751935958862305,
                0.037585918426513674,
                0.037574657440185545,
                0.03760025787353516,
                0.0376627197265625,
                0.03759001541137695,
                0.03761459350585938,
                0.03762790298461914,
                0.03757875061035156,
                0.03762688064575195,
                0.03760639953613281,
                0.037631999969482424,
                0.03759718322753906,
                0.03768320083618164,
                0.0375838737487793,
                0.03773952102661133,
                0.03763507080078125,
                0.037644287109375,
                0.03763507080078125,
                0.03765862274169922,
                0.03762790298461914,
                0.037651454925537106,
                0.037634048461914066,
                0.037751808166503906,
                0.03758694458007812,
                0.037634048461914066,
                0.03761663818359375,
                0.037198848724365234,
                0.037157886505126955,
                0.03719270324707031,
                0.03721011352539062,
                0.03719475173950195,
                0.037190654754638675,
                0.03719987106323242,
                0.03719680023193359,
                0.03722649765014648,
                0.037275646209716795,
                0.0372408332824707,
                0.0372408332824707,
                0.037217281341552735,
                0.03724800109863281,
                0.03722547149658203,
                0.03725721740722656,
                0.03727462387084961,
                0.037294078826904296,
                0.03725721740722656,
                0.03729612731933594,
                0.03724492645263672,
                0.03732889556884766,
                0.03727769470214844,
                0.03726847839355469,
                0.0372674560546875,
                0.03735039901733399,
                0.037275646209716795,
                0.0373196792602539,
                0.03731763076782227,
                0.03733401489257813,
                0.03729817581176758,
                0.0373831672668457,
                0.03732992172241211,
                0.037348350524902346,
                0.03733913421630859,
                0.03738828659057617,
                0.0373831672668457,
                0.03742310333251953,
                0.037395454406738284,
                0.03742822265625,
                0.037378047943115236,
                0.03745280075073242,
                0.037395454406738284,
                0.03749273681640625,
                0.03740979385375977,
                0.03750912094116211,
                0.03745382308959961,
                0.03747840118408203,
                0.03743641662597656,
                0.037591041564941405,
                0.03747225570678711,
                0.03755724716186523,
                0.03743334579467773,
                0.037572608947753904,
                0.03746099090576172,
                0.037531646728515625,
                0.037473281860351565,
                0.03763711929321289,
                0.03746815872192383,
                0.03759513473510742,
                0.03747532653808594,
                0.03768012619018555,
                0.03750604629516602,
                0.037548030853271484,
                0.03751833724975586,
                0.037599231719970705,
                0.03750400161743164,
                0.03767295837402344,
                0.03750092697143555,
                0.03769241714477539,
                0.0375470085144043,
                0.0376360969543457,
                0.037531646728515625,
                0.0375623664855957,
                0.037564414978027344,
                0.03756851196289063,
                0.037553150177001955,
                0.03760332870483398,
                0.03755724716186523,
                0.03762688064575195,
                0.03754393768310547,
                0.03760025787353516,
                0.037599231719970705,
                0.03757363128662109,
                0.037579776763916016,
                0.03762073516845703,
                0.03762073516845703,
                0.037719039916992186,
                0.037550079345703126,
                0.037610496520996094,
                0.037585918426513674,
                0.03763916778564453,
                0.03762176132202148,
                0.03763711929321289,
                0.03764223861694336,
                0.03772927856445313,
                0.03763097763061524,
                0.037689342498779296,
                0.03764223861694336,
                0.03719782257080078,
                0.03715584182739258,
                0.03719168090820312,
                0.03718048095703125,
                0.03720185470581055,
                0.03721625518798828,
                0.03715584182739258,
                0.037182464599609374,
                0.03722956848144531,
                0.037256191253662106,
                0.03722649765014648,
                0.0372592658996582,
                0.03724492645263672,
                0.03726438522338867,
                0.03723980712890625,
                0.03723980712890625,
                0.03723571014404297,
                0.03725823974609375,
                0.037275646209716795,
                0.03728179168701172,
                0.03723980712890625,
                0.03727974319458008,
                0.03729817581176758,
                0.03731455993652344,
                0.037285888671875,
                0.03731148910522461,
                0.03729919815063477,
                0.037341182708740234,
                0.03736883163452148,
                0.03735244750976562,
                0.03736064147949219,
                0.03737395095825195,
                0.03735244750976562,
                0.037362686157226564,
                0.037362686157226564,
                0.037362686157226564,
                0.037367809295654295,
                0.03738214492797851,
                0.03739340972900391,
                0.03741593551635742,
                0.03740262222290039,
                0.03747123336791992,
                0.03744768142700195,
                0.03744153594970703,
                0.03745382308959961,
                0.03747532653808594,
                0.037465087890625,
                0.03748863983154297,
                0.037454849243164064,
                0.03761971282958984,
                0.037498878479003905,
                0.037564414978027344,
                0.03745280075073242,
                0.03765760040283203,
                0.03746713638305664,
                0.037577728271484374,
                0.03750092697143555,
                0.037628929138183595,
                0.03750604629516602,
                0.037591041564941405,
                0.037515262603759765,
                0.03763097763061524,
                0.03746815872192383,
                0.037553150177001955,
                0.0374917106628418,
                0.037617664337158206,
                0.03752959823608398,
                0.03759513473510742,
                0.0375470085144043,
                0.03768012619018555,
                0.03750707244873047,
                0.0376258544921875,
                0.03753062438964844,
                0.03762176132202148,
                0.03758489608764649,
                0.03758182525634766,
                0.03755929565429687,
                0.03757567977905273,
                0.03755724716186523,
                0.03763916778564453,
                0.03754086303710937,
                0.03762073516845703,
                0.03757567977905273,
                0.037602302551269534,
                0.037602302551269534,
                0.03763814544677734,
                0.03760332870483398,
                0.03771084976196289,
                0.03762688064575195,
                0.037634048461914066,
                0.03759718322753906,
                0.03765862274169922,
                0.03762790298461914,
                0.0376627197265625,
                0.03763711929321289,
                0.03772313690185547,
                0.03762073516845703,
                0.03763302230834961,
                0.0376545295715332
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 53.39860042562468
        },
        "energy": null,
        "efficiency": null
    }
}