benchmark_report.json

{
    "prefill": {
        "memory": {
            "unit": "MB",
            "max_ram": 1476.370432,
            "max_vram": 15674.114048,
            "max_reserved": 15189.671936,
            "max_allocated": 15039.57248
        },
        "latency": {
            "unit": "s",
            "mean": 0.04065064185227804,
            "stdev": 0.00010230905193306444,
            "values": [
                0.041487518310546874,
                0.04111769485473633,
                0.04118214416503906,
                0.041109504699707033,
                0.04109619140625,
                0.041133056640625,
                0.04107571029663086,
                0.0411514892578125,
                0.04080640029907227,
                0.04065689468383789,
                0.040659969329833984,
                0.04064665603637695,
                0.040667198181152345,
                0.04075212860107422,
                0.04067839813232422,
                0.04067225646972656,
                0.04067737579345703,
                0.040643585205078124,
                0.04069171142578125,
                0.04068454360961914,
                0.04066611099243164,
                0.04066918563842774,
                0.04065689468383789,
                0.0406558723449707,
                0.04067843246459961,
                0.04065484619140625,
                0.04063129425048828,
                0.04063948822021484,
                0.04067020797729492,
                0.040681472778320314,
                0.040630271911621094,
                0.04066611099243164,
                0.04063129425048828,
                0.04068454360961914,
                0.04065280151367188,
                0.0406743049621582,
                0.040622081756591794,
                0.04061695861816406,
                0.04065792083740234,
                0.04067020797729492,
                0.04065785598754883,
                0.04067532730102539,
                0.04062412643432617,
                0.040668159484863284,
                0.04062412643432617,
                0.040653759002685544,
                0.04060671997070313,
                0.04059852981567383,
                0.04067331314086914,
                0.04066304016113281,
                0.040662017822265625,
                0.04062003326416016,
                0.04063641738891602,
                0.04062617492675781,
                0.040627201080322264,
                0.040610816955566405,
                0.040651775360107424,
                0.040619007110595705,
                0.04065075302124024,
                0.040591358184814456,
                0.04067225646972656,
                0.040651775360107424,
                0.04060361480712891,
                0.04058726501464844,
                0.040630271911621094,
                0.0406743049621582,
                0.04060467147827149,
                0.040671230316162106,
                0.040635391235351564,
                0.040648704528808595,
                0.04063641738891602,
                0.04063334274291992,
                0.04064255905151367,
                0.04060876846313476,
                0.04062617492675781,
                0.040627201080322264,
                0.04061798477172852,
                0.040632320404052735,
                0.04068966293334961,
                0.04062515258789062,
                0.040619007110595705,
                0.040643585205078124,
                0.04064460754394531,
                0.04069068908691406,
                0.04065075302124024,
                0.04064051055908203,
                0.04063846588134765,
                0.04064761734008789,
                0.04061183929443359,
                0.040648704528808595,
                0.040653823852539066,
                0.040615936279296876,
                0.040612865447998046,
                0.04060160064697266,
                0.040612865447998046,
                0.04061695861816406,
                0.04062617492675781,
                0.040626144409179686,
                0.04059852981567383,
                0.04064153671264648,
                0.040645633697509766,
                0.04066099166870117,
                0.04064255905151367,
                0.040627201080322264,
                0.040645633697509766,
                0.0406558723449707,
                0.040627201080322264,
                0.04060671997070313,
                0.04060876846313476,
                0.04066611099243164,
                0.04062515258789062,
                0.040668159484863284,
                0.040599552154541016,
                0.040635391235351564,
                0.0405852165222168,
                0.04064255905151367,
                0.0406363525390625,
                0.040629249572753906,
                0.04060569763183594,
                0.040639457702636717,
                0.040591358184814456,
                0.04069683074951172,
                0.04065484619140625,
                0.040630271911621094,
                0.040621055603027346,
                0.04062412643432617,
                0.040635391235351564,
                0.04063126373291016,
                0.04069683074951172,
                0.04060467147827149,
                0.040632320404052735,
                0.040591358184814456,
                0.04063846588134765,
                0.040629249572753906,
                0.04057395172119141,
                0.040615936279296876,
                0.04061491012573242,
                0.04060979080200195,
                0.04063436889648438,
                0.04067225646972656,
                0.040615936279296876,
                0.040594432830810545,
                0.04062310409545898,
                0.04065689468383789,
                0.04064153671264648,
                0.04060979080200195,
                0.04064153671264648,
                0.040667137145996096,
                0.04063948822021484,
                0.04063536071777344,
                0.04062822341918945,
                0.040630271911621094,
                0.040621055603027346,
                0.04060876846313476,
                0.04065484619140625,
                0.040635391235351564,
                0.04059545516967773,
                0.04063641738891602,
                0.040668159484863284,
                0.04061491012573242,
                0.04062412643432617,
                0.04060671997070313,
                0.04063641738891602,
                0.040643585205078124,
                0.04061183929443359,
                0.04064153671264648,
                0.04061183929443359,
                0.04061695861816406,
                0.04060774230957031,
                0.04069887924194336,
                0.04062822341918945,
                0.040651775360107424,
                0.04064255905151367,
                0.04060160064697266,
                0.04059033584594727,
                0.040597503662109374,
                0.04060671997070313,
                0.04063948822021484,
                0.04062508773803711,
                0.040602622985839845,
                0.040600574493408204,
                0.04062515258789062,
                0.04063334274291992,
                0.040602622985839845,
                0.040643585205078124,
                0.0406036491394043,
                0.040553470611572266,
                0.04062515258789062,
                0.0405852165222168,
                0.04061695861816406,
                0.04059648132324219,
                0.040636318206787106,
                0.04065894317626953,
                0.040592384338378903,
                0.040594432830810545,
                0.040567806243896484,
                0.040629249572753906,
                0.040667137145996096,
                0.04060876846313476,
                0.0406640625,
                0.04064255905151367,
                0.04064665603637695,
                0.04066099166870117,
                0.04061798477172852,
                0.04067225646972656,
                0.04062412643432617,
                0.04062822341918945,
                0.04063334274291992,
                0.04062617492675781,
                0.04062003326416016,
                0.04063334274291992,
                0.04061491012573242,
                0.040612831115722656,
                0.04061695861816406,
                0.040630271911621094,
                0.04062819290161133,
                0.04060467147827149,
                0.040615936279296876,
                0.040584190368652344,
                0.04062822341918945,
                0.040584190368652344,
                0.04065075302124024,
                0.04058828735351563,
                0.04060979080200195,
                0.040586238861083986,
                0.040658912658691405,
                0.040615936279296876,
                0.040602657318115236,
                0.040597503662109374,
                0.04064460754394531,
                0.04060569763183594,
                0.040645633697509766,
                0.040668159484863284,
                0.04064051055908203,
                0.040639457702636717,
                0.04060671997070313,
                0.040621055603027346,
                0.04062617492675781,
                0.04065894317626953,
                0.04061798477172852,
                0.040553470611572266,
                0.04061798477172852,
                0.04057395172119141,
                0.04067327880859375,
                0.04055244827270508,
                0.040610816955566405
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 787.1954424799995
        },
        "energy": null,
        "efficiency": null
    },
    "decode": {
        "memory": {
            "unit": "MB",
            "max_ram": 1487.609856,
            "max_vram": 15739.12576,
            "max_reserved": 15254.683648,
            "max_allocated": 15101.761024
        },
        "latency": {
            "unit": "s",
            "mean": 3.71031038792928,
            "stdev": 0.0003956553636294078,
            "values": [
                3.710101493835448,
                3.709965293884277,
                3.7108643760681166
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 53.3648076031999
        },
        "energy": null,
        "efficiency": null
    },
    "per_token": {
        "memory": null,
        "latency": {
            "unit": "s",
            "mean": 0.03747788270635637,
            "stdev": 0.00015737209717694308,
            "values": [
                0.03731660842895508,
                0.03723263931274414,
                0.03724390411376953,
                0.037253120422363284,
                0.037149696350097655,
                0.03721113586425781,
                0.0373043212890625,
                0.03723263931274414,
                0.037223423004150394,
                0.037269504547119144,
                0.037275646209716795,
                0.037269504547119144,
                0.03726233673095703,
                0.03732275390625,
                0.03732070541381836,
                0.037282817840576174,
                0.03727667236328125,
                0.03727155303955078,
                0.03728902435302734,
                0.03730425643920898,
                0.03723980712890625,
                0.03729612731933594,
                0.03729919815063477,
                0.03730124664306641,
                0.03727155303955078,
                0.037379070281982424,
                0.03734527969360352,
                0.03736576080322265,
                0.037348350524902346,
                0.03734220886230469,
                0.037372928619384765,
                0.03741183853149414,
                0.037321727752685545,
                0.037384193420410154,
                0.037410816192626956,
                0.037397502899169925,
                0.037362686157226564,
                0.03742617416381836,
                0.03737497711181641,
                0.03750809478759766,
                0.03741798400878906,
                0.03747123336791992,
                0.037408767700195314,
                0.0374917106628418,
                0.037443584442138675,
                0.03750713729858399,
                0.03744147109985352,
                0.037525505065917966,
                0.03748147201538086,
                0.037602302551269534,
                0.03745177459716797,
                0.037599231719970705,
                0.03750297546386719,
                0.037663745880126956,
                0.037479423522949216,
                0.03762179183959961,
                0.03751625442504883,
                0.0376545295715332,
                0.03753881454467774,
                0.03757670211791992,
                0.037510143280029294,
                0.03769753646850586,
                0.03749273681640625,
                0.03762073516845703,
                0.037495807647705076,
                0.03763507080078125,
                0.03754905700683594,
                0.03765657424926758,
                0.03758489608764649,
                0.03771084976196289,
                0.0375470085144043,
                0.03766067123413086,
                0.037569534301757815,
                0.03761971282958984,
                0.03754393768310547,
                0.037579776763916016,
                0.03757363128662109,
                0.03762483215332031,
                0.03759001541137695,
                0.03767091369628906,
                0.037596160888671876,
                0.037634048461914066,
                0.037596160888671876,
                0.03763302230834961,
                0.03758899307250976,
                0.037645313262939455,
                0.03760947036743164,
                0.03775078582763672,
                0.03762278366088867,
                0.0376627197265625,
                0.037618686676025394,
                0.03767196655273437,
                0.03762377548217773,
                0.03769651031494141,
                0.03762483215332031,
                0.03775283050537109,
                0.03765350341796875,
                0.03767910385131836,
                0.03763916778564453,
                0.03725823974609375,
                0.037168128967285156,
                0.0372305908203125,
                0.03722137451171875,
                0.03721625518798828,
                0.03720601654052735,
                0.0372408332824707,
                0.03724185562133789,
                0.037236736297607424,
                0.03725823974609375,
                0.03728384017944336,
                0.03725721740722656,
                0.03726233673095703,
                0.03728486251831055,
                0.03727974319458008,
                0.0372674560546875,
                0.037275646209716795,
                0.03729305648803711,
                0.0372592658996582,
                0.037321727752685545,
                0.03727052688598633,
                0.037338111877441404,
                0.037314590454101564,
                0.03730326461791992,
                0.03729817581176758,
                0.037340160369873046,
                0.03733196640014649,
                0.037372928619384765,
                0.037362686157226564,
                0.037395454406738284,
                0.037362686157226564,
                0.03740671920776367,
                0.037338111877441404,
                0.03735859298706055,
                0.03739852905273437,
                0.037400577545166014,
                0.037392383575439454,
                0.03743334579467773,
                0.037381118774414065,
                0.03748761749267578,
                0.03741593551635742,
                0.037466110229492186,
                0.03742924880981445,
                0.03748761749267578,
                0.03739648056030274,
                0.03751321411132812,
                0.03743231964111328,
                0.037515262603759765,
                0.0374835205078125,
                0.03759107208251953,
                0.037459934234619144,
                0.03760025787353516,
                0.03748556900024414,
                0.037618686676025394,
                0.03748863983154297,
                0.03755212783813477,
                0.03748147201538086,
                0.037651454925537106,
                0.03752755355834961,
                0.03760435104370117,
                0.03749587249755859,
                0.03765958404541016,
                0.037498878479003905,
                0.03761151885986328,
                0.03751321411132812,
                0.03765964889526367,
                0.037550079345703126,
                0.0376545295715332,
                0.0375470085144043,
                0.037693504333496095,
                0.03755923080444336,
                0.03765248107910156,
                0.037561344146728515,
                0.03758796691894531,
                0.037553150177001955,
                0.03760844802856445,
                0.037564414978027344,
                0.037585918426513674,
                0.03757875061035156,
                0.0376995849609375,
                0.037694465637207034,
                0.03766988754272461,
                0.03759820938110352,
                0.03765862274169922,
                0.03761670303344727,
                0.037693374633789065,
                0.03764940643310547,
                0.037735424041748046,
                0.03761459350585938,
                0.0376360969543457,
                0.037612545013427735,
                0.03768627166748047,
                0.037618686676025394,
                0.03770163345336914,
                0.03768832015991211,
                0.037743614196777346,
                0.037617664337158206,
                0.037700607299804685,
                0.03764223861694336,
                0.03721932983398438,
                0.03719782257080078,
                0.037195777893066405,
                0.03725107192993164,
                0.03718348693847656,
                0.03720806503295898,
                0.03720601654052735,
                0.0372674560546875,
                0.03726540756225586,
                0.03726847839355469,
                0.0373043212890625,
                0.03725721740722656,
                0.03725823974609375,
                0.037302272796630856,
                0.037272575378417966,
                0.03726335906982422,
                0.03729919815063477,
                0.037308414459228514,
                0.03727667236328125,
                0.03732275390625,
                0.037321727752685545,
                0.037308414459228514,
                0.03731558227539063,
                0.0373196792602539,
                0.03730124664306641,
                0.03737702560424805,
                0.037338111877441404,
                0.037343231201171875,
                0.03738009643554688,
                0.037369857788085936,
                0.037386302947998044,
                0.037415870666503905,
                0.03736883163452148,
                0.037392383575439454,
                0.03740262222290039,
                0.03740979385375977,
                0.037392383575439454,
                0.037440513610839846,
                0.03739648056030274,
                0.03745792007446289,
                0.037438465118408204,
                0.03747225570678711,
                0.037425151824951174,
                0.037477375030517575,
                0.03746713638305664,
                0.0375245132446289,
                0.03748246383666992,
                0.03753267288208008,
                0.03751424026489258,
                0.03758182525634766,
                0.03750092697143555,
                0.03756851196289063,
                0.03751119995117187,
                0.03762684631347656,
                0.03748659133911133,
                0.03758694458007812,
                0.03750809478759766,
                0.03763814544677734,
                0.037548030853271484,
                0.03765350341796875,
                0.037544960021972655,
                0.03766988754272461,
                0.03751628875732422,
                0.037615615844726565,
                0.03752243041992188,
                0.037667839050292966,
                0.03755724716186523,
                0.03767295837402344,
                0.037548030853271484,
                0.03771289443969727,
                0.037572608947753904,
                0.03767910385131836,
                0.03760025787353516,
                0.03759206390380859,
                0.037591041564941405,
                0.037599231719970705,
                0.0375654411315918,
                0.03760435104370117,
                0.03760844802856445,
                0.037684223175048825,
                0.037612545013427735,
                0.037631999969482424,
                0.03758796691894531,
                0.03763814544677734,
                0.0376258544921875,
                0.037661697387695314,
                0.037601280212402347,
                0.03776204681396484,
                0.037645313262939455,
                0.03768115234375,
                0.037644287109375,
                0.03771084976196289,
                0.03769241714477539,
                0.03768832015991211,
                0.03767295837402344,
                0.03776716613769531,
                0.03763916778564453,
                0.03769753646850586,
                0.037667839050292966
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 53.364807603199885
        },
        "energy": null,
        "efficiency": null
    }
}