benchmark_report.json

{
    "prefill": {
        "memory": {
            "unit": "MB",
            "max_ram": 1476.288512,
            "max_vram": 15674.114048,
            "max_reserved": 15189.671936,
            "max_allocated": 15039.57248
        },
        "latency": {
            "unit": "s",
            "mean": 0.04063447015653783,
            "stdev": 0.00010095564261546599,
            "values": [
                0.041492897033691405,
                0.04113404846191406,
                0.04114329528808594,
                0.041093120574951174,
                0.04108390426635742,
                0.04109107208251953,
                0.04096716690063477,
                0.04110335922241211,
                0.040852481842041016,
                0.040612865447998046,
                0.04064972686767578,
                0.040597503662109374,
                0.04065792083740234,
                0.040645633697509766,
                0.04066099166870117,
                0.04061692810058594,
                0.040653823852539066,
                0.040637439727783206,
                0.04067023849487305,
                0.04066304016113281,
                0.040622081756591794,
                0.040622081756591794,
                0.04063641738891602,
                0.04062003326416016,
                0.040659969329833984,
                0.04062515258789062,
                0.04065075302124024,
                0.04057491302490234,
                0.04063334274291992,
                0.040643585205078124,
                0.040703998565673825,
                0.040619007110595705,
                0.04061993789672851,
                0.040645633697509766,
                0.040635391235351564,
                0.040680416107177736,
                0.04062515258789062,
                0.040635391235351564,
                0.04059545516967773,
                0.04067225646972656,
                0.04070809555053711,
                0.04068966293334961,
                0.040615936279296876,
                0.04065075302124024,
                0.040627201080322264,
                0.04063641738891602,
                0.04063436889648438,
                0.040613887786865234,
                0.040612865447998046,
                0.040671230316162106,
                0.04064255905151367,
                0.04062003326416016,
                0.04059743881225586,
                0.04057395172119141,
                0.040612865447998046,
                0.04061491012573242,
                0.040637439727783206,
                0.040622081756591794,
                0.040591358184814456,
                0.040545280456542966,
                0.04061183929443359,
                0.040652736663818356,
                0.04062412643432617,
                0.040615936279296876,
                0.04058726501464844,
                0.04062617492675781,
                0.04061183929443359,
                0.040592384338378903,
                0.040632320404052735,
                0.040643585205078124,
                0.04063846588134765,
                0.04062310409545898,
                0.04063436889648438,
                0.04059545516967773,
                0.040622081756591794,
                0.04062515258789062,
                0.0405852165222168,
                0.040629249572753906,
                0.04062003326416016,
                0.040629249572753906,
                0.04062003326416016,
                0.04060569763183594,
                0.040610816955566405,
                0.04060671997070313,
                0.040648704528808595,
                0.04066304016113281,
                0.04064972686767578,
                0.04064972686767578,
                0.04060569763183594,
                0.04064767837524414,
                0.04065792083740234,
                0.040651775360107424,
                0.04062310409545898,
                0.04058726501464844,
                0.04062822341918945,
                0.04063846588134765,
                0.04064153671264648,
                0.040637439727783206,
                0.04055756759643555,
                0.04060876846313476,
                0.040635391235351564,
                0.04063948822021484,
                0.04065792083740234,
                0.04062515258789062,
                0.04061183929443359,
                0.040599552154541016,
                0.040602622985839845,
                0.04058931350708008,
                0.04064153671264648,
                0.040632320404052735,
                0.040602622985839845,
                0.0405852165222168,
                0.04059340667724609,
                0.040653823852539066,
                0.04061183929443359,
                0.04060979080200195,
                0.04062617492675781,
                0.04058931350708008,
                0.04064051055908203,
                0.04061183929443359,
                0.04058009719848633,
                0.040602622985839845,
                0.040637439727783206,
                0.040629249572753906,
                0.040637439727783206,
                0.04060160064697266,
                0.040630271911621094,
                0.040586238861083986,
                0.04063129425048828,
                0.04063638305664063,
                0.04062412643432617,
                0.04060774230957031,
                0.04059545516967773,
                0.040600574493408204,
                0.04058931350708008,
                0.040635391235351564,
                0.04057692718505859,
                0.040591327667236325,
                0.04059033584594727,
                0.04062617492675781,
                0.04060979080200195,
                0.04058313751220703,
                0.040592384338378903,
                0.04063948822021484,
                0.04064255905151367,
                0.040613887786865234,
                0.04064665603637695,
                0.0406363525390625,
                0.040619007110595705,
                0.04061491012573242,
                0.040622081756591794,
                0.0405667839050293,
                0.04064972686767578,
                0.040576000213623044,
                0.04066099166870117,
                0.04059648132324219,
                0.04063948822021484,
                0.04056883239746094,
                0.04063641738891602,
                0.040597503662109374,
                0.040599552154541016,
                0.04057292938232422,
                0.040608673095703124,
                0.04064051055908203,
                0.04060160064697266,
                0.04059340667724609,
                0.04064153671264648,
                0.04059545516967773,
                0.040612865447998046,
                0.040622081756591794,
                0.0405852165222168,
                0.040591358184814456,
                0.04059852981567383,
                0.040600574493408204,
                0.04059852981567383,
                0.04066099166870117,
                0.04055654525756836,
                0.040581119537353515,
                0.040613887786865234,
                0.040578048706054685,
                0.04060979080200195,
                0.04060467147827149,
                0.04058726501464844,
                0.04061075210571289,
                0.04060467147827149,
                0.04060467147827149,
                0.04058214569091797,
                0.040578048706054685,
                0.04055551910400391,
                0.040621055603027346,
                0.0405852165222168,
                0.04063846588134765,
                0.04064255905151367,
                0.04058828735351563,
                0.04057088088989258,
                0.040546302795410154,
                0.040600574493408204,
                0.04062412643432617,
                0.04066304016113281,
                0.040627201080322264,
                0.04064255905151367,
                0.040630271911621094,
                0.040629249572753906,
                0.04061695861816406,
                0.04061695861816406,
                0.040602622985839845,
                0.04060873413085937,
                0.040621055603027346,
                0.040635391235351564,
                0.04060876846313476,
                0.0406036491394043,
                0.04060876846313476,
                0.04061695861816406,
                0.04058009719848633,
                0.040621055603027346,
                0.04062310409545898,
                0.040600574493408204,
                0.040600574493408204,
                0.04058726501464844,
                0.040621055603027346,
                0.040627201080322264,
                0.04057702255249023,
                0.0406036491394043,
                0.04057395172119141,
                0.04058726501464844,
                0.04060876846313476,
                0.04061491012573242,
                0.040576000213623044,
                0.040584190368652344,
                0.040599552154541016,
                0.04061798477172852,
                0.04061183929443359,
                0.040667137145996096,
                0.04061798477172852,
                0.040602622985839845,
                0.04062412643432617,
                0.040622081756591794,
                0.040637439727783206,
                0.04061798477172852,
                0.040599552154541016,
                0.040551422119140625,
                0.040591327667236325,
                0.04056876754760742,
                0.040594432830810545,
                0.04053606414794922,
                0.040602622985839845
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 787.5087303150525
        },
        "energy": null,
        "efficiency": null
    },
    "decode": {
        "memory": {
            "unit": "MB",
            "max_ram": 1487.204352,
            "max_vram": 15739.12576,
            "max_reserved": 15254.683648,
            "max_allocated": 15101.761024
        },
        "latency": {
            "unit": "s",
            "mean": 3.708370947519938,
            "stdev": 0.00019802779189991205,
            "values": [
                3.7080985488891582,
                3.708450828552247,
                3.7085634651184085
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 53.392716856553214
        },
        "energy": null,
        "efficiency": null
    },
    "per_token": {
        "memory": null,
        "latency": {
            "unit": "s",
            "mean": 0.03745829239919128,
            "stdev": 0.00015786611458293046,
            "values": [
                0.03724390411376953,
                0.037177345275878904,
                0.037190654754638675,
                0.037195777893066405,
                0.03715584182739258,
                0.037212158203125,
                0.037253120422363284,
                0.03718041610717773,
                0.03724390411376953,
                0.03727462387084961,
                0.03723775863647461,
                0.03725823974609375,
                0.03727360153198242,
                0.037303295135498044,
                0.03725516891479492,
                0.03725823974609375,
                0.037292030334472655,
                0.03728793716430664,
                0.03724595260620117,
                0.03731455993652344,
                0.037256191253662106,
                0.03729715347290039,
                0.03729100799560547,
                0.037266433715820314,
                0.037275646209716795,
                0.037356544494628906,
                0.03729919815063477,
                0.037340160369873046,
                0.03734220886230469,
                0.03733196640014649,
                0.037370880126953124,
                0.0374200325012207,
                0.03729919815063477,
                0.03736883163452148,
                0.03737702560424805,
                0.037392383575439454,
                0.03737395095825195,
                0.03741900634765625,
                0.037343231201171875,
                0.03745177459716797,
                0.03739648056030274,
                0.03746406555175781,
                0.03738828659057617,
                0.03746918487548828,
                0.0374128646850586,
                0.03745894241333008,
                0.03744870376586914,
                0.03753779220581055,
                0.037477375030517575,
                0.037566463470458986,
                0.03744563293457031,
                0.0375838737487793,
                0.03743129730224609,
                0.03764019012451172,
                0.03747020721435547,
                0.03760435104370117,
                0.03750604629516602,
                0.03760537719726562,
                0.03744255828857422,
                0.03759001541137695,
                0.03748147201538086,
                0.03766886520385742,
                0.037482494354248046,
                0.03757567977905273,
                0.037495807647705076,
                0.03762278366088867,
                0.03753267288208008,
                0.03764121627807617,
                0.03757363128662109,
                0.03768832015991211,
                0.03757567977905273,
                0.03763507080078125,
                0.03753676986694336,
                0.0375623664855957,
                0.03753062438964844,
                0.03753062438964844,
                0.03755417633056641,
                0.03759718322753906,
                0.0375470085144043,
                0.03764940643310547,
                0.03754905700683594,
                0.03759308624267578,
                0.037628929138183595,
                0.03758694458007812,
                0.037596160888671876,
                0.03761356735229492,
                0.03760435104370117,
                0.03770675277709961,
                0.03762995147705078,
                0.03764019012451172,
                0.03758489608764649,
                0.037664768218994144,
                0.03760537719726562,
                0.03767193603515625,
                0.03761356735229492,
                0.03772825622558594,
                0.03763302230834961,
                0.03767193603515625,
                0.03762688064575195,
                0.03716198348999023,
                0.03719987106323242,
                0.03719475173950195,
                0.037177345275878904,
                0.03721011352539062,
                0.03721113586425781,
                0.03720499038696289,
                0.03719372940063476,
                0.037238784790039066,
                0.03726233673095703,
                0.03727667236328125,
                0.037282817840576174,
                0.03722444915771484,
                0.03724288177490234,
                0.037250049591064455,
                0.037223423004150394,
                0.03725516891479492,
                0.03727769470214844,
                0.037272575378417966,
                0.03726335906982422,
                0.03723468780517578,
                0.0373043212890625,
                0.03721420669555664,
                0.03729612731933594,
                0.03732582473754883,
                0.03737497711181641,
                0.03732275390625,
                0.03734425735473633,
                0.03732787322998047,
                0.037372928619384765,
                0.037356544494628906,
                0.037392383575439454,
                0.037369857788085936,
                0.037362686157226564,
                0.03733913421630859,
                0.037375999450683595,
                0.03744460678100586,
                0.03740671920776367,
                0.037379070281982424,
                0.0374466552734375,
                0.0373831672668457,
                0.037443584442138675,
                0.037395454406738284,
                0.03747840118408203,
                0.03745894241333008,
                0.03748863983154297,
                0.03744768142700195,
                0.03749990463256836,
                0.03744870376586914,
                0.03758694458007812,
                0.037443584442138675,
                0.037533695220947266,
                0.03747840118408203,
                0.03759001541137695,
                0.03748761749267578,
                0.03757363128662109,
                0.037449726104736326,
                0.03764121627807617,
                0.037479423522949216,
                0.03758796691894531,
                0.03750502395629883,
                0.03767705535888672,
                0.03748556900024414,
                0.03758796691894531,
                0.03746918487548828,
                0.0376360969543457,
                0.037533695220947266,
                0.03768832015991211,
                0.037531646728515625,
                0.03768832015991211,
                0.037550079345703126,
                0.03765760040283203,
                0.03755929565429687,
                0.03762483215332031,
                0.03758899307250976,
                0.03760639953613281,
                0.03757567977905273,
                0.03758899307250976,
                0.03760435104370117,
                0.03766579055786133,
                0.03756032180786133,
                0.037579776763916016,
                0.03762176132202148,
                0.03760332870483398,
                0.0375838737487793,
                0.037667839050292966,
                0.037596160888671876,
                0.037705726623535156,
                0.03760332870483398,
                0.03764326477050781,
                0.037582847595214845,
                0.037719039916992186,
                0.03760025787353516,
                0.03768729782104492,
                0.037661697387695314,
                0.03774054336547852,
                0.037634048461914066,
                0.03769241714477539,
                0.03762995147705078,
                0.037217281341552735,
                0.037169151306152344,
                0.037177345275878904,
                0.037220352172851565,
                0.03719987106323242,
                0.03722956848144531,
                0.037223423004150394,
                0.0372592658996582,
                0.03719475173950195,
                0.037250049591064455,
                0.03720499038696289,
                0.03724697494506836,
                0.03721625518798828,
                0.03730636978149414,
                0.037233665466308595,
                0.03725516891479492,
                0.03724800109863281,
                0.037269504547119144,
                0.03724697494506836,
                0.037292030334472655,
                0.037272575378417966,
                0.037302272796630856,
                0.03727769470214844,
                0.037303295135498044,
                0.03728179168701172,
                0.037343231201171875,
                0.03729919815063477,
                0.03734425735473633,
                0.03731148910522461,
                0.037370880126953124,
                0.03737497711181641,
                0.03739648056030274,
                0.0373831672668457,
                0.037395454406738284,
                0.037364734649658206,
                0.03740879821777344,
                0.03738211059570312,
                0.03741798400878906,
                0.03738009643554688,
                0.03743436813354492,
                0.03745177459716797,
                0.03745177459716797,
                0.037389312744140625,
                0.03745075225830078,
                0.03742924880981445,
                0.0375470085144043,
                0.03745792007446289,
                0.03748761749267578,
                0.03749068832397461,
                0.03755110549926758,
                0.03745792007446289,
                0.03759206390380859,
                0.03745587158203125,
                0.0376360969543457,
                0.037482494354248046,
                0.03755110549926758,
                0.03750707244873047,
                0.03761151885986328,
                0.03746201705932617,
                0.03758899307250976,
                0.03749785614013672,
                0.037634048461914066,
                0.03749273681640625,
                0.03759820938110352,
                0.0375203857421875,
                0.03759513473510742,
                0.037553150177001955,
                0.03765350341796875,
                0.03755724716186523,
                0.037698558807373043,
                0.037525505065917966,
                0.037634048461914066,
                0.03755724716186523,
                0.037582847595214845,
                0.03757567977905273,
                0.037582847595214845,
                0.03756851196289063,
                0.037594112396240234,
                0.037556224822998044,
                0.03765555191040039,
                0.037558273315429686,
                0.037647361755371096,
                0.03754905700683594,
                0.037615615844726565,
                0.037607425689697264,
                0.037623809814453124,
                0.03764633560180664,
                0.03770265579223633,
                0.037599231719970705,
                0.03765555191040039,
                0.0375838737487793,
                0.037689342498779296,
                0.03759820938110352,
                0.03766579055786133,
                0.03766681671142578,
                0.03773952102661133,
                0.03765760040283203,
                0.03768012619018555,
                0.03768524932861328
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 53.39271685655323
        },
        "energy": null,
        "efficiency": null
    }
}