benchmark_report.json

{
    "prefill": {
        "memory": {
            "unit": "MB",
            "max_ram": 1476.89472,
            "max_vram": 15674.114048,
            "max_reserved": 15189.671936,
            "max_allocated": 15039.57248
        },
        "latency": {
            "unit": "s",
            "mean": 0.04062439397486242,
            "stdev": 9.737644067555968e-05,
            "values": [
                0.04141324615478516,
                0.04108697509765625,
                0.04112179183959961,
                0.041076736450195314,
                0.041078784942626956,
                0.041062400817871096,
                0.04100096130371094,
                0.04110752105712891,
                0.04077363204956055,
                0.04063436889648438,
                0.040668159484863284,
                0.04063641738891602,
                0.04062003326416016,
                0.04059545516967773,
                0.04063948822021484,
                0.04063948822021484,
                0.0406558723449707,
                0.04067737579345703,
                0.04064665603637695,
                0.040632320404052735,
                0.04062822341918945,
                0.04065484619140625,
                0.04063436889648438,
                0.040632320404052735,
                0.04062918472290039,
                0.040627201080322264,
                0.04066918563842774,
                0.040610816955566405,
                0.040643585205078124,
                0.040576000213623044,
                0.04063641738891602,
                0.04062003326416016,
                0.040645633697509766,
                0.040630271911621094,
                0.04063334274291992,
                0.04064460754394531,
                0.040619007110595705,
                0.040659969329833984,
                0.04064150238037109,
                0.040637439727783206,
                0.040600574493408204,
                0.040637439727783206,
                0.04057292938232422,
                0.04062515258789062,
                0.040637439727783206,
                0.04057702255249023,
                0.04064460754394531,
                0.04056268692016601,
                0.04063846588134765,
                0.0406640625,
                0.04063129425048828,
                0.040583168029785156,
                0.04060467147827149,
                0.04062412643432617,
                0.040599552154541016,
                0.040586238861083986,
                0.04059648132324219,
                0.04056063842773437,
                0.0405852165222168,
                0.04057190322875977,
                0.040594432830810545,
                0.04065075302124024,
                0.04060467147827149,
                0.04060467147827149,
                0.04062617492675781,
                0.040630271911621094,
                0.040592384338378903,
                0.04059852981567383,
                0.04061491012573242,
                0.04066099166870117,
                0.040594432830810545,
                0.04063948822021484,
                0.040586238861083986,
                0.040583168029785156,
                0.04059340667724609,
                0.04062310409545898,
                0.04055551910400391,
                0.04059340667724609,
                0.04059852981567383,
                0.040581119537353515,
                0.04061798477172852,
                0.04058828735351563,
                0.040643585205078124,
                0.04063436889648438,
                0.040635391235351564,
                0.04062102508544922,
                0.040648704528808595,
                0.04064767837524414,
                0.040619007110595705,
                0.040591358184814456,
                0.040621055603027346,
                0.04057702255249023,
                0.04060876846313476,
                0.04064767837524414,
                0.04065484619140625,
                0.04059033584594727,
                0.04059436798095703,
                0.040597503662109374,
                0.040581119537353515,
                0.040574943542480466,
                0.040612865447998046,
                0.04058828735351563,
                0.04061798477172852,
                0.04062515258789062,
                0.040619007110595705,
                0.04063129425048828,
                0.040592384338378903,
                0.04057696151733398,
                0.0406732177734375,
                0.04062310409545898,
                0.04063334274291992,
                0.04060979080200195,
                0.04060569763183594,
                0.040597503662109374,
                0.040619007110595705,
                0.040613887786865234,
                0.040586238861083986,
                0.040583168029785156,
                0.0406036491394043,
                0.040615936279296876,
                0.040621055603027346,
                0.04062515258789062,
                0.04064460754394531,
                0.040584190368652344,
                0.04061695861816406,
                0.04058726501464844,
                0.04058822250366211,
                0.04060160064697266,
                0.040635391235351564,
                0.04055756759643555,
                0.040576000213623044,
                0.040548351287841795,
                0.04062412643432617,
                0.040581119537353515,
                0.040613887786865234,
                0.04060569763183594,
                0.04060150527954102,
                0.04060876846313476,
                0.04058214569091797,
                0.04059648132324219,
                0.040574977874755856,
                0.04058828735351563,
                0.040597503662109374,
                0.04058931350708008,
                0.040630271911621094,
                0.040629249572753906,
                0.0406558723449707,
                0.04063436889648438,
                0.04063948822021484,
                0.040602622985839845,
                0.04060671997070313,
                0.04060774230957031,
                0.04059852981567383,
                0.04058214569091797,
                0.04063846588134765,
                0.040597503662109374,
                0.04060774230957031,
                0.040581119537353515,
                0.040583168029785156,
                0.040576000213623044,
                0.04060979080200195,
                0.04058009719848633,
                0.040648704528808595,
                0.04059545516967773,
                0.04058726501464844,
                0.04061491012573242,
                0.04060979080200195,
                0.04061382293701172,
                0.04062003326416016,
                0.04060569763183594,
                0.04059545516967773,
                0.04057907104492187,
                0.0405852165222168,
                0.0405667839050293,
                0.040578048706054685,
                0.04053299331665039,
                0.04056371307373047,
                0.04055244827270508,
                0.040613887786865234,
                0.04057190322875977,
                0.04057702255249023,
                0.04057907104492187,
                0.04058009719848633,
                0.04060774230957031,
                0.04060774230957031,
                0.04061491012573242,
                0.04056883239746094,
                0.04057088088989258,
                0.04057292938232422,
                0.040635391235351564,
                0.04061695861816406,
                0.040635391235351564,
                0.040586238861083986,
                0.04058009719848633,
                0.04054732894897461,
                0.04056883239746094,
                0.04057702255249023,
                0.040602622985839845,
                0.04060774230957031,
                0.04062515258789062,
                0.04065075302124024,
                0.04060467147827149,
                0.04062515258789062,
                0.040610816955566405,
                0.040602622985839845,
                0.04059545516967773,
                0.040599552154541016,
                0.040576000213623044,
                0.0406036491394043,
                0.04062412643432617,
                0.040599552154541016,
                0.04062003326416016,
                0.040643585205078124,
                0.040581119537353515,
                0.04060774230957031,
                0.040581119537353515,
                0.04058214569091797,
                0.04061491012573242,
                0.040600513458251955,
                0.04061183929443359,
                0.0406036491394043,
                0.04058931350708008,
                0.04055955123901367,
                0.04057907104492187,
                0.040578048706054685,
                0.04061695861816406,
                0.04059027099609375,
                0.04060160064697266,
                0.040591358184814456,
                0.040619007110595705,
                0.040635391235351564,
                0.04060979080200195,
                0.04064457702636719,
                0.040592384338378903,
                0.04063846588134765,
                0.040597503662109374,
                0.04058931350708008,
                0.04061491012573242,
                0.04061183929443359,
                0.040578048706054685,
                0.040561664581298826,
                0.040564735412597655,
                0.040584190368652344,
                0.04059033584594727,
                0.04054937744140625,
                0.04057088088989258
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 787.7040583005613
        },
        "energy": null,
        "efficiency": null
    },
    "decode": {
        "memory": {
            "unit": "MB",
            "max_ram": 1487.908864,
            "max_vram": 15739.12576,
            "max_reserved": 15254.683648,
            "max_allocated": 15101.761024
        },
        "latency": {
            "unit": "s",
            "mean": 3.709277858734131,
            "stdev": 0.0006353868436230146,
            "values": [
                3.708397563934327,
                3.7098741645812985,
                3.7095618476867673
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 53.37966244124177
        },
        "energy": null,
        "efficiency": null
    },
    "per_token": {
        "memory": null,
        "latency": {
            "unit": "s",
            "mean": 0.03746745311852656,
            "stdev": 0.00015769933796486534,
            "values": [
                0.03725721740722656,
                0.03718143844604492,
                0.03720294570922852,
                0.03719680023193359,
                0.03717324829101563,
                0.037223423004150394,
                0.03720908737182617,
                0.03718041610717773,
                0.03722854232788086,
                0.03728486251831055,
                0.03722137451171875,
                0.03728691101074219,
                0.03721932983398438,
                0.03729305648803711,
                0.03724697494506836,
                0.03724288177490234,
                0.03730022430419922,
                0.03727872085571289,
                0.03723980712890625,
                0.03729510498046875,
                0.03726131057739258,
                0.037308414459228514,
                0.03726335906982422,
                0.037282817840576174,
                0.03729100799560547,
                0.037353473663330077,
                0.0373043212890625,
                0.03734732818603516,
                0.0373125114440918,
                0.037343231201171875,
                0.037372928619384765,
                0.037422080993652344,
                0.03732992172241211,
                0.037381118774414065,
                0.03737702560424805,
                0.03737702560424805,
                0.03735859298706055,
                0.03745792007446289,
                0.037364734649658206,
                0.03746713638305664,
                0.03742105484008789,
                0.03745894241333008,
                0.037416961669921874,
                0.03749273681640625,
                0.03743334579467773,
                0.03753062438964844,
                0.037449726104736326,
                0.03750912094116211,
                0.037438465118408204,
                0.03755724716186523,
                0.03746918487548828,
                0.03758694458007812,
                0.03748966217041016,
                0.037645313262939455,
                0.037449726104736326,
                0.0375654411315918,
                0.037465087890625,
                0.03758899307250976,
                0.03748147201538086,
                0.03759820938110352,
                0.0375203857421875,
                0.03765964889526367,
                0.037477375030517575,
                0.037596160888671876,
                0.03750604629516602,
                0.03764019012451172,
                0.037558273315429686,
                0.037634048461914066,
                0.0375551986694336,
                0.037698558807373043,
                0.0375470085144043,
                0.037647361755371096,
                0.037564414978027344,
                0.03757670211791992,
                0.03753574371337891,
                0.0375623664855957,
                0.037577728271484374,
                0.03758899307250976,
                0.03755724716186523,
                0.03767500686645508,
                0.03755724716186523,
                0.03763302230834961,
                0.037585918426513674,
                0.037610496520996094,
                0.03757875061035156,
                0.03760639953613281,
                0.037601280212402347,
                0.037684223175048825,
                0.03761356735229492,
                0.037661697387695314,
                0.03759718322753906,
                0.037644287109375,
                0.037610496520996094,
                0.03765760040283203,
                0.03761356735229492,
                0.03773235321044922,
                0.03763507080078125,
                0.03765760040283203,
                0.037651454925537106,
                0.03723161697387695,
                0.037203968048095705,
                0.03718348693847656,
                0.037207038879394534,
                0.03723775863647461,
                0.03719168090820312,
                0.03719782257080078,
                0.03722444915771484,
                0.03722854232788086,
                0.03723161697387695,
                0.0372592658996582,
                0.037272575378417966,
                0.03725823974609375,
                0.03730022430419922,
                0.03729305648803711,
                0.03729612731933594,
                0.037302272796630856,
                0.03733913421630859,
                0.037266433715820314,
                0.03728691101074219,
                0.03727769470214844,
                0.037308414459228514,
                0.03726233673095703,
                0.03731763076782227,
                0.03730636978149414,
                0.03736166381835938,
                0.03733401489257813,
                0.03736166381835938,
                0.03732992172241211,
                0.03737190246582031,
                0.037340160369873046,
                0.03744870376586914,
                0.0374015998840332,
                0.037395454406738284,
                0.03738726425170898,
                0.03739136123657227,
                0.03738828659057617,
                0.03741491317749023,
                0.037384193420410154,
                0.03744768142700195,
                0.037443584442138675,
                0.03746303939819336,
                0.037389312744140625,
                0.037512191772460936,
                0.03742822265625,
                0.03750400161743164,
                0.03748863983154297,
                0.03751833724975586,
                0.03750604629516602,
                0.037612545013427735,
                0.037459968566894535,
                0.03757875061035156,
                0.03750604629516602,
                0.03766579055786133,
                0.03754086303710937,
                0.03757670211791992,
                0.03751935958862305,
                0.03761971282958984,
                0.03752140808105469,
                0.03763097763061524,
                0.03750092697143555,
                0.03764019012451172,
                0.0375551986694336,
                0.037628929138183595,
                0.03759001541137695,
                0.03763711929321289,
                0.037548030853271484,
                0.03768012619018555,
                0.03753472137451172,
                0.03767193603515625,
                0.03750502395629883,
                0.03764121627807617,
                0.037566463470458986,
                0.037580799102783204,
                0.03759718322753906,
                0.03760025787353516,
                0.037566463470458986,
                0.03757363128662109,
                0.03757363128662109,
                0.03769548797607422,
                0.037596160888671876,
                0.037612545013427735,
                0.037596160888671876,
                0.03762073516845703,
                0.03762688064575195,
                0.03764223861694336,
                0.03762688064575195,
                0.037718017578125,
                0.03760025787353516,
                0.03765043258666992,
                0.03759513473510742,
                0.03767091369628906,
                0.037634048461914066,
                0.03769139099121094,
                0.03769753646850586,
                0.03775283050537109,
                0.037664768218994144,
                0.03764940643310547,
                0.03761151885986328,
                0.0372305908203125,
                0.03719782257080078,
                0.03718860626220703,
                0.03723161697387695,
                0.0372408332824707,
                0.03719680023193359,
                0.037185535430908204,
                0.037207038879394534,
                0.03723161697387695,
                0.037233665466308595,
                0.037200897216796876,
                0.03726335906982422,
                0.03726335906982422,
                0.03728793716430664,
                0.037282817840576174,
                0.0372674560546875,
                0.03726028823852539,
                0.03729715347290039,
                0.03725209426879883,
                0.03725516891479492,
                0.03733708953857422,
                0.03730636978149414,
                0.037275646209716795,
                0.037338111877441404,
                0.03733401489257813,
                0.0373309440612793,
                0.03730022430419922,
                0.03743129730224609,
                0.03733401489257813,
                0.037348350524902346,
                0.037351425170898435,
                0.03738521575927734,
                0.03739648056030274,
                0.037370880126953124,
                0.03735859298706055,
                0.03749478530883789,
                0.03737395095825195,
                0.03740979385375977,
                0.03741183853149414,
                0.03742924880981445,
                0.03741798400878906,
                0.03744870376586914,
                0.03736883163452148,
                0.037459968566894535,
                0.03746815872192383,
                0.03750912094116211,
                0.0374917106628418,
                0.03758796691894531,
                0.03748966217041016,
                0.037574657440185545,
                0.03746099090576172,
                0.037599231719970705,
                0.03745894241333008,
                0.037617664337158206,
                0.03750092697143555,
                0.03757056045532227,
                0.03748556900024414,
                0.0376545295715332,
                0.03751424026489258,
                0.03761459350585938,
                0.037515262603759765,
                0.03760947036743164,
                0.037498878479003905,
                0.03760435104370117,
                0.037526527404785154,
                0.03764019012451172,
                0.037566463470458986,
                0.03768012619018555,
                0.03750400161743164,
                0.03767705535888672,
                0.037531646728515625,
                0.03765862274169922,
                0.03756748962402344,
                0.03758489608764649,
                0.037556224822998044,
                0.03758182525634766,
                0.037541889190673826,
                0.037599231719970705,
                0.03758694458007812,
                0.0377077751159668,
                0.03758796691894531,
                0.037601280212402347,
                0.037599231719970705,
                0.03762278366088867,
                0.03763814544677734,
                0.037647361755371096,
                0.037580799102783204,
                0.037776382446289065,
                0.03761151885986328,
                0.03766988754272461,
                0.037599231719970705,
                0.03768832015991211,
                0.03765043258666992,
                0.03778662490844727,
                0.03767910385131836,
                0.03772108840942383,
                0.03764019012451172,
                0.03765657424926758,
                0.03767910385131836
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 53.37966244124179
        },
        "energy": null,
        "efficiency": null
    }
}