benchmark_report.json

{
    "prefill": {
        "memory": {
            "unit": "MB",
            "max_ram": 1476.939776,
            "max_vram": 15674.114048,
            "max_reserved": 15189.671936,
            "max_allocated": 15039.57248
        },
        "latency": {
            "unit": "s",
            "mean": 0.04063008809283498,
            "stdev": 9.76172995398282e-05,
            "values": [
                0.04138979339599609,
                0.04110335922241211,
                0.04110335922241211,
                0.041083839416503905,
                0.04105625534057617,
                0.041199615478515625,
                0.04084940719604492,
                0.04111667251586914,
                0.040799232482910154,
                0.040612865447998046,
                0.04062617492675781,
                0.04060876846313476,
                0.04069375991821289,
                0.040612865447998046,
                0.04067020797729492,
                0.04064255905151367,
                0.04073164749145508,
                0.04062515258789062,
                0.04067839813232422,
                0.040662017822265625,
                0.04065689468383789,
                0.04063334274291992,
                0.04065280151367188,
                0.04067635345458984,
                0.040630271911621094,
                0.04060774230957031,
                0.040621055603027346,
                0.04060671997070313,
                0.04069375991821289,
                0.04062822341918945,
                0.04064665603637695,
                0.04063436889648438,
                0.040610816955566405,
                0.040629249572753906,
                0.040653823852539066,
                0.04065689468383789,
                0.040622081756591794,
                0.04060979080200195,
                0.04064460754394531,
                0.04066304016113281,
                0.04063641738891602,
                0.04064972686767578,
                0.04061798477172852,
                0.04061695861816406,
                0.040619007110595705,
                0.04062617492675781,
                0.04067020797729492,
                0.04057907104492187,
                0.0406640625,
                0.04065894317626953,
                0.0406578254699707,
                0.04061692810058594,
                0.04061491012573242,
                0.040581119537353515,
                0.04060569763183594,
                0.040583168029785156,
                0.040632320404052735,
                0.0406036491394043,
                0.04061491012573242,
                0.04054425430297852,
                0.04062003326416016,
                0.040599552154541016,
                0.0405852165222168,
                0.040592384338378903,
                0.040632320404052735,
                0.040643585205078124,
                0.040619007110595705,
                0.040576000213623044,
                0.040633281707763674,
                0.04063641738891602,
                0.040619968414306644,
                0.04057292938232422,
                0.040622081756591794,
                0.040599552154541016,
                0.04061798477172852,
                0.04061695861816406,
                0.04059033584594727,
                0.040612865447998046,
                0.04062819290161133,
                0.04060876846313476,
                0.04059545516967773,
                0.04059545516967773,
                0.040612865447998046,
                0.040597503662109374,
                0.040637439727783206,
                0.040648704528808595,
                0.040643585205078124,
                0.04062515258789062,
                0.04067225646972656,
                0.040622081756591794,
                0.040651775360107424,
                0.040602622985839845,
                0.04059033584594727,
                0.04060774230957031,
                0.04060979080200195,
                0.040629249572753906,
                0.0406036491394043,
                0.04064665603637695,
                0.0405401611328125,
                0.04056576156616211,
                0.040623039245605466,
                0.040627201080322264,
                0.040635391235351564,
                0.040613887786865234,
                0.04061695861816406,
                0.04060153579711914,
                0.040586238861083986,
                0.04063641738891602,
                0.040613887786865234,
                0.04064355087280273,
                0.040586238861083986,
                0.04060160064697266,
                0.04060774230957031,
                0.040637439727783206,
                0.040583168029785156,
                0.04061798477172852,
                0.04058518218994141,
                0.040621055603027346,
                0.04064051055908203,
                0.04061183929443359,
                0.0405667839050293,
                0.040612865447998046,
                0.04061798477172852,
                0.040570911407470704,
                0.04061798477172852,
                0.04058726501464844,
                0.04058620834350586,
                0.04057190322875977,
                0.040600574493408204,
                0.040586238861083986,
                0.04063840103149414,
                0.04058009719848633,
                0.04060876846313476,
                0.04059033584594727,
                0.040621055603027346,
                0.04062412643432617,
                0.040612865447998046,
                0.04060774230957031,
                0.040619007110595705,
                0.04060160064697266,
                0.0405852165222168,
                0.040599552154541016,
                0.04063641738891602,
                0.04059648132324219,
                0.040597503662109374,
                0.040613887786865234,
                0.040643585205078124,
                0.040667137145996096,
                0.040648704528808595,
                0.040591358184814456,
                0.04061695861816406,
                0.040578048706054685,
                0.04064972686767578,
                0.04059545516967773,
                0.04063948822021484,
                0.040569854736328126,
                0.040632320404052735,
                0.040551422119140625,
                0.04063436889648438,
                0.04058828735351563,
                0.04060467147827149,
                0.040594432830810545,
                0.0406640625,
                0.04063334274291992,
                0.040615936279296876,
                0.04061183929443359,
                0.04060160064697266,
                0.040594432830810545,
                0.040619007110595705,
                0.040648704528808595,
                0.04060569763183594,
                0.04056268692016601,
                0.040599552154541016,
                0.04057292938232422,
                0.04056576156616211,
                0.04056268692016601,
                0.04056371307373047,
                0.04054732894897461,
                0.04059033584594727,
                0.04061695861816406,
                0.04061183929443359,
                0.04061491012573242,
                0.04059033584594727,
                0.0405852165222168,
                0.040581119537353515,
                0.040592384338378903,
                0.04055551910400391,
                0.04056576156616211,
                0.04056576156616211,
                0.04060671997070313,
                0.040600574493408204,
                0.04060569763183594,
                0.040653823852539066,
                0.04059033584594727,
                0.04055756759643555,
                0.04057702255249023,
                0.04059852981567383,
                0.04060768127441406,
                0.04064255905151367,
                0.04065792083740234,
                0.04064255905151367,
                0.040615936279296876,
                0.04061183929443359,
                0.040665088653564455,
                0.04066099166870117,
                0.04061183929443359,
                0.040653823852539066,
                0.040615936279296876,
                0.04059545516967773,
                0.04058214569091797,
                0.040632320404052735,
                0.04060255813598633,
                0.04060979080200195,
                0.040574977874755856,
                0.0406036491394043,
                0.040643585205078124,
                0.040583168029785156,
                0.04059545516967773,
                0.04058726501464844,
                0.040594432830810545,
                0.04058828735351563,
                0.040618911743164066,
                0.040581119537353515,
                0.040553470611572266,
                0.040586143493652346,
                0.04060774230957031,
                0.04065075302124024,
                0.04056576156616211,
                0.04061798477172852,
                0.040612865447998046,
                0.040615936279296876,
                0.040613887786865234,
                0.04067942428588867,
                0.04064972686767578,
                0.040612865447998046,
                0.040610816955566405,
                0.04062617492675781,
                0.040630271911621094,
                0.04064767837524414,
                0.04057292938232422,
                0.0405667839050293,
                0.04055244827270508,
                0.04056576156616211,
                0.040594432830810545,
                0.04058009719848633,
                0.0405667839050293
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 787.5936652385237
        },
        "energy": null,
        "efficiency": null
    },
    "decode": {
        "memory": {
            "unit": "MB",
            "max_ram": 1488.171008,
            "max_vram": 15739.12576,
            "max_reserved": 15254.683648,
            "max_allocated": 15101.761024
        },
        "latency": {
            "unit": "s",
            "mean": 3.71086609395345,
            "stdev": 5.97791248185428e-05,
            "values": [
                3.710939136505127,
                3.7108664360046384,
                3.7107927093505846
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 53.35681616823217
        },
        "energy": null,
        "efficiency": null
    },
    "per_token": {
        "memory": null,
        "latency": {
            "unit": "s",
            "mean": 0.03748349589851969,
            "stdev": 0.00015874297262972117,
            "values": [
                0.03729715347290039,
                0.03720806503295898,
                0.03720294570922852,
                0.03721420669555664,
                0.03719782257080078,
                0.03722444915771484,
                0.037253120422363284,
                0.037256191253662106,
                0.03727974319458008,
                0.03727052688598633,
                0.03727155303955078,
                0.03725823974609375,
                0.03727052688598633,
                0.03731353759765625,
                0.03729100799560547,
                0.03726335906982422,
                0.037318656921386716,
                0.037294078826904296,
                0.03727667236328125,
                0.03733913421630859,
                0.0373043212890625,
                0.0373125114440918,
                0.03729919815063477,
                0.03727872085571289,
                0.03727667236328125,
                0.03737190246582031,
                0.03733401489257813,
                0.03740467071533203,
                0.037356544494628906,
                0.03736064147949219,
                0.03736064147949219,
                0.03744460678100586,
                0.03737395095825195,
                0.037384193420410154,
                0.037410816192626956,
                0.03739852905273437,
                0.03744460678100586,
                0.037466110229492186,
                0.03740262222290039,
                0.037501953125,
                0.037443584442138675,
                0.03747532653808594,
                0.03743436813354492,
                0.03746713638305664,
                0.03748044967651367,
                0.037539840698242184,
                0.03745280075073242,
                0.03751116943359375,
                0.03747430419921875,
                0.03758694458007812,
                0.03750297546386719,
                0.037564414978027344,
                0.037482494354248046,
                0.03763916778564453,
                0.037498878479003905,
                0.03761151885986328,
                0.03753062438964844,
                0.03763507080078125,
                0.03751321411132812,
                0.037596160888671876,
                0.037515262603759765,
                0.03769753646850586,
                0.03753267288208008,
                0.037647361755371096,
                0.03751731109619141,
                0.03766886520385742,
                0.037526527404785154,
                0.03766886520385742,
                0.03758694458007812,
                0.03770470428466797,
                0.03758899307250976,
                0.03763916778564453,
                0.0375838737487793,
                0.03760844802856445,
                0.03760332870483398,
                0.03761971282958984,
                0.037601280212402347,
                0.03761151885986328,
                0.03762790298461914,
                0.037644287109375,
                0.037572608947753904,
                0.037615615844726565,
                0.037602302551269534,
                0.037618686676025394,
                0.037618686676025394,
                0.037631999969482424,
                0.03761459350585938,
                0.03777536010742188,
                0.03766886520385742,
                0.03768320083618164,
                0.03765760040283203,
                0.03765760040283203,
                0.03769343948364258,
                0.0376995849609375,
                0.037663745880126956,
                0.037765121459960936,
                0.03765043258666992,
                0.03769139099121094,
                0.03763507080078125,
                0.03720806503295898,
                0.03717222213745117,
                0.037207038879394534,
                0.03724697494506836,
                0.03723161697387695,
                0.037217281341552735,
                0.03722444915771484,
                0.03721932983398438,
                0.03723161697387695,
                0.03730022430419922,
                0.03725721740722656,
                0.03727155303955078,
                0.03732889556884766,
                0.03729100799560547,
                0.03727360153198242,
                0.03725414276123047,
                0.037302272796630856,
                0.03734732818603516,
                0.03729612731933594,
                0.03731660842895508,
                0.03726131057739258,
                0.0373309440612793,
                0.03729305648803711,
                0.03732582473754883,
                0.03728998565673828,
                0.03738214492797851,
                0.037324798583984374,
                0.037378047943115236,
                0.03734425735473633,
                0.03738214492797851,
                0.0373493766784668,
                0.037408767700195314,
                0.03737190246582031,
                0.03740671920776367,
                0.03736678314208984,
                0.03738726425170898,
                0.0374466552734375,
                0.037449726104736326,
                0.03738726425170898,
                0.037493759155273435,
                0.037410816192626956,
                0.03750400161743164,
                0.03746713638305664,
                0.03746918487548828,
                0.03747020721435547,
                0.03752959823608398,
                0.03743231964111328,
                0.0375203857421875,
                0.03745177459716797,
                0.03760435104370117,
                0.037495807647705076,
                0.037602302551269534,
                0.03749068832397461,
                0.037607425689697264,
                0.03749990463256836,
                0.037601280212402347,
                0.03748556900024414,
                0.037678081512451174,
                0.0375203857421875,
                0.037594112396240234,
                0.037525505065917966,
                0.03769036865234375,
                0.037515262603759765,
                0.037618686676025394,
                0.037512191772460936,
                0.03768115234375,
                0.03750092697143555,
                0.03768524932861328,
                0.037569534301757815,
                0.03774054336547852,
                0.03757567977905273,
                0.037661697387695314,
                0.03756032180786133,
                0.03762073516845703,
                0.03756748962402344,
                0.03762483215332031,
                0.03758182525634766,
                0.03762176132202148,
                0.037585918426513674,
                0.03767705535888672,
                0.037602302551269534,
                0.03764223861694336,
                0.037612545013427735,
                0.03769651031494141,
                0.03763916778564453,
                0.037684223175048825,
                0.03758489608764649,
                0.037754878997802735,
                0.037574657440185545,
                0.03766579055786133,
                0.03764223861694336,
                0.03769241714477539,
                0.03767705535888672,
                0.037700607299804685,
                0.03768524932861328,
                0.03781631851196289,
                0.03766988754272461,
                0.0377262077331543,
                0.03766886520385742,
                0.03723775863647461,
                0.03718656158447266,
                0.037250049591064455,
                0.03724390411376953,
                0.03722444915771484,
                0.03726233673095703,
                0.037200897216796876,
                0.03725209426879883,
                0.037222400665283206,
                0.037282817840576174,
                0.03721830368041992,
                0.03726847839355469,
                0.03725107192993164,
                0.03729100799560547,
                0.03727974319458008,
                0.03727052688598633,
                0.037282817840576174,
                0.037292030334472655,
                0.03728076934814453,
                0.037318656921386716,
                0.03732582473754883,
                0.037335041046142575,
                0.03731455993652344,
                0.03734220886230469,
                0.037348350524902346,
                0.037318656921386716,
                0.037302272796630856,
                0.0373493766784668,
                0.037338111877441404,
                0.03737395095825195,
                0.03737190246582031,
                0.03741798400878906,
                0.03738521575927734,
                0.03740467071533203,
                0.03737395095825195,
                0.03740364837646484,
                0.03737190246582031,
                0.03741798400878906,
                0.037394432067871096,
                0.037495807647705076,
                0.03741183853149414,
                0.03748761749267578,
                0.037459968566894535,
                0.03748556900024414,
                0.03750092697143555,
                0.03752345657348633,
                0.03749478530883789,
                0.037539840698242184,
                0.03747840118408203,
                0.03760435104370117,
                0.03750502395629883,
                0.0376258544921875,
                0.03751935958862305,
                0.0376545295715332,
                0.03750297546386719,
                0.03758182525634766,
                0.03751116943359375,
                0.037628929138183595,
                0.03747430419921875,
                0.03763507080078125,
                0.037533695220947266,
                0.03763097763061524,
                0.03753062438964844,
                0.03759820938110352,
                0.0375623664855957,
                0.03763916778564453,
                0.037558273315429686,
                0.03766067123413086,
                0.03753779220581055,
                0.03771084976196289,
                0.03758182525634766,
                0.03766886520385742,
                0.037572608947753904,
                0.03761356735229492,
                0.03763507080078125,
                0.03762073516845703,
                0.037582847595214845,
                0.03763302230834961,
                0.037585918426513674,
                0.03767500686645508,
                0.03758489608764649,
                0.037585918426513674,
                0.037602302551269534,
                0.03762995147705078,
                0.03764326477050781,
                0.037628929138183595,
                0.03763302230834961,
                0.03772723388671875,
                0.03761356735229492,
                0.03769651031494141,
                0.037661697387695314,
                0.03767910385131836,
                0.03765350341796875,
                0.03765760040283203,
                0.037663745880126956,
                0.037720062255859374,
                0.03770265579223633,
                0.03772723388671875,
                0.03772108840942383
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 53.356816168232186
        },
        "energy": null,
        "efficiency": null
    }
}