benchmark_report.json

{
    "prefill": {
        "memory": {
            "unit": "MB",
            "max_ram": 1476.21888,
            "max_vram": 15674.114048,
            "max_reserved": 15189.671936,
            "max_allocated": 15039.57248
        },
        "latency": {
            "unit": "s",
            "mean": 0.040593590511539115,
            "stdev": 9.838939445947689e-05,
            "values": [
                0.041398624420166015,
                0.04108185577392578,
                0.041082878112792966,
                0.04108902359008789,
                0.04100198364257813,
                0.04106444931030274,
                0.04095590209960937,
                0.041073665618896485,
                0.04070195388793945,
                0.04058214569091797,
                0.04058214569091797,
                0.040602622985839845,
                0.040610816955566405,
                0.04060160064697266,
                0.04063123321533203,
                0.040576000213623044,
                0.04061798477172852,
                0.040583168029785156,
                0.040621055603027346,
                0.040568801879882814,
                0.04058931350708008,
                0.040581119537353515,
                0.040612865447998046,
                0.04061695861816406,
                0.040561664581298826,
                0.04056060791015625,
                0.04060160064697266,
                0.04058214569091797,
                0.040629249572753906,
                0.04057702255249023,
                0.040586238861083986,
                0.040584190368652344,
                0.04057088088989258,
                0.04060675048828125,
                0.04063334274291992,
                0.04061695861816406,
                0.04056063842773437,
                0.040542209625244144,
                0.04058009719848633,
                0.04063641738891602,
                0.04058828735351563,
                0.040638496398925784,
                0.04055859375,
                0.04058214569091797,
                0.04057395172119141,
                0.040569854736328126,
                0.04060876846313476,
                0.04054937744140625,
                0.0406097297668457,
                0.04060160064697266,
                0.04062617492675781,
                0.04056268692016601,
                0.040574977874755856,
                0.040578048706054685,
                0.040584190368652344,
                0.04052787017822266,
                0.040574977874755856,
                0.04054732894897461,
                0.040610816955566405,
                0.04052479934692383,
                0.04058931350708008,
                0.04058214569091797,
                0.04054732894897461,
                0.04055039978027344,
                0.040612865447998046,
                0.040561664581298826,
                0.04054732894897461,
                0.040551422119140625,
                0.040591358184814456,
                0.040615936279296876,
                0.040606689453125,
                0.04058828735351563,
                0.04058828735351563,
                0.04055756759643555,
                0.040576000213623044,
                0.040602622985839845,
                0.04054937744140625,
                0.040545280456542966,
                0.040567806243896484,
                0.04054729461669922,
                0.04056063842773437,
                0.04062412643432617,
                0.04058214569091797,
                0.04057292938232422,
                0.040602622985839845,
                0.04061180877685547,
                0.04062412643432617,
                0.04060671997070313,
                0.04060467147827149,
                0.04055551910400391,
                0.04062617492675781,
                0.040559616088867184,
                0.040581119537353515,
                0.04056883239746094,
                0.04057395172119141,
                0.04057292938232422,
                0.040600574493408204,
                0.040591358184814456,
                0.04052479934692383,
                0.04057395172119141,
                0.04058828735351563,
                0.04057702255249023,
                0.040592384338378903,
                0.04061183929443359,
                0.04059852981567383,
                0.040602622985839845,
                0.040591358184814456,
                0.04052787017822266,
                0.04058214569091797,
                0.04060160064697266,
                0.04055756759643555,
                0.040574977874755856,
                0.04053708648681641,
                0.040594432830810545,
                0.040559616088867184,
                0.04059033584594727,
                0.04056371307373047,
                0.0405401611328125,
                0.0405667839050293,
                0.04055449676513672,
                0.04058515167236328,
                0.040600574493408204,
                0.040630271911621094,
                0.040621055603027346,
                0.04057395172119141,
                0.040581119537353515,
                0.0405852165222168,
                0.04056268692016601,
                0.040594432830810545,
                0.04053913497924805,
                0.04057907104492187,
                0.04054323196411133,
                0.040586238861083986,
                0.04057702255249023,
                0.04057292938232422,
                0.04059545516967773,
                0.040561664581298826,
                0.04056063842773437,
                0.04056982421875,
                0.0406036491394043,
                0.04055039978027344,
                0.04054425430297852,
                0.04058214569091797,
                0.04056063842773437,
                0.04058009719848633,
                0.040583168029785156,
                0.04062515258789062,
                0.04062003326416016,
                0.040645633697509766,
                0.0405667839050293,
                0.04057187271118164,
                0.04056572723388672,
                0.040600574493408204,
                0.04051251220703125,
                0.04058726501464844,
                0.04056063842773437,
                0.040583168029785156,
                0.04052275085449219,
                0.04058828735351563,
                0.04054937744140625,
                0.04058214569091797,
                0.04057395172119141,
                0.04061183929443359,
                0.04057088088989258,
                0.040576000213623044,
                0.04057190322875977,
                0.04057190322875977,
                0.040541152954101566,
                0.04055039978027344,
                0.040574977874755856,
                0.040578048706054685,
                0.04056371307373047,
                0.040583168029785156,
                0.040545280456542966,
                0.04052275085449219,
                0.04053299331665039,
                0.04056063842773437,
                0.0406036491394043,
                0.04054732894897461,
                0.04055449676513672,
                0.040574977874755856,
                0.040545280456542966,
                0.0405852165222168,
                0.04057907104492187,
                0.040564735412597655,
                0.04054118347167969,
                0.04053401565551758,
                0.04055039978027344,
                0.04054732894897461,
                0.040599552154541016,
                0.04057702255249023,
                0.04054425430297852,
                0.04057907104492187,
                0.04052684783935547,
                0.040576000213623044,
                0.040529918670654294,
                0.04053094482421875,
                0.04057702255249023,
                0.04060467147827149,
                0.04061183929443359,
                0.04060160064697266,
                0.040578048706054685,
                0.04057907104492187,
                0.04055039978027344,
                0.04058931350708008,
                0.04058620834350586,
                0.040576000213623044,
                0.04057292938232422,
                0.04054224014282227,
                0.040542209625244144,
                0.040584190368652344,
                0.040574977874755856,
                0.040597503662109374,
                0.040564735412597655,
                0.04057395172119141,
                0.04057190322875977,
                0.04059340667724609,
                0.040574977874755856,
                0.040586238861083986,
                0.04060467147827149,
                0.04055756759643555,
                0.040583168029785156,
                0.04057395172119141,
                0.04054009628295899,
                0.040581119537353515,
                0.04055039978027344,
                0.040559616088867184,
                0.04052275085449219,
                0.04055859375,
                0.04057190322875977,
                0.04059852981567383,
                0.040591358184814456,
                0.04060979080200195,
                0.04058726501464844,
                0.04061183929443359,
                0.04057190322875977,
                0.040610816955566405,
                0.04057395172119141,
                0.04056576156616211,
                0.040542209625244144,
                0.04051968002319336,
                0.04054425430297852,
                0.040531967163085936,
                0.040509407043457034,
                0.04050431823730469,
                0.04052479934692383
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 788.3017884536155
        },
        "energy": null,
        "efficiency": null
    },
    "decode": {
        "memory": {
            "unit": "MB",
            "max_ram": 1487.122432,
            "max_vram": 15739.12576,
            "max_reserved": 15254.683648,
            "max_allocated": 15101.761024
        },
        "latency": {
            "unit": "s",
            "mean": 3.706289138793945,
            "stdev": 3.810517169861684e-05,
            "values": [
                3.706292221069336,
                3.7062410049438466,
                3.706334190368653
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 53.4227073456095
        },
        "energy": null,
        "efficiency": null
    },
    "per_token": {
        "memory": null,
        "latency": {
            "unit": "s",
            "mean": 0.03743726402822167,
            "stdev": 0.0001591921532385222,
            "values": [
                0.037212158203125,
                0.03712409591674805,
                0.0371671028137207,
                0.03717836761474609,
                0.03711283111572266,
                0.037185535430908204,
                0.03718963241577149,
                0.03719475173950195,
                0.03722444915771484,
                0.03724185562133789,
                0.037222400665283206,
                0.03723468780517578,
                0.03724288177490234,
                0.03727974319458008,
                0.03723775863647461,
                0.03725209426879883,
                0.03724697494506836,
                0.03723775863647461,
                0.037222400665283206,
                0.03728486251831055,
                0.03722854232788086,
                0.037250049591064455,
                0.03723263931274414,
                0.03727360153198242,
                0.03723775863647461,
                0.03730739212036133,
                0.03731148910522461,
                0.037321727752685545,
                0.0373125114440918,
                0.037335041046142575,
                0.037343231201171875,
                0.03736064147949219,
                0.037326847076416016,
                0.03733299255371094,
                0.0373493766784668,
                0.0373831672668457,
                0.03785728073120117,
                0.03742617416381836,
                0.03733196640014649,
                0.03741183853149414,
                0.037381118774414065,
                0.037424129486083986,
                0.037389312744140625,
                0.03746713638305664,
                0.03739955139160156,
                0.037473281860351565,
                0.03745280075073242,
                0.03753472137451172,
                0.037395454406738284,
                0.037550079345703126,
                0.03737702560424805,
                0.037664768218994144,
                0.037427200317382815,
                0.03760844802856445,
                0.03743948745727539,
                0.037564414978027344,
                0.03747840118408203,
                0.03759308624267578,
                0.03745280075073242,
                0.03758694458007812,
                0.03744563293457031,
                0.03759820938110352,
                0.03746303939819336,
                0.03755417633056641,
                0.03745894241333008,
                0.03758899307250976,
                0.03747840118408203,
                0.03765862274169922,
                0.03751731109619141,
                0.0376545295715332,
                0.03753267288208008,
                0.03756748962402344,
                0.037539840698242184,
                0.037550079345703126,
                0.037525505065917966,
                0.03755929565429687,
                0.03751731109619141,
                0.037563392639160156,
                0.03751935958862305,
                0.03763814544677734,
                0.03752243041992188,
                0.037564414978027344,
                0.03755417633056641,
                0.03755212783813477,
                0.03759718322753906,
                0.03764326477050781,
                0.037571582794189456,
                0.0376995849609375,
                0.03757056045532227,
                0.03765043258666992,
                0.03758489608764649,
                0.03759718322753906,
                0.03758182525634766,
                0.03761459350585938,
                0.03760639953613281,
                0.037749759674072264,
                0.03758694458007812,
                0.037644287109375,
                0.037582847595214845,
                0.03717529678344727,
                0.03712716674804688,
                0.03721625518798828,
                0.03719168090820312,
                0.037179393768310545,
                0.03717836761474609,
                0.037203968048095705,
                0.03718143844604492,
                0.037177345275878904,
                0.03718963241577149,
                0.03725823974609375,
                0.03724390411376953,
                0.03726847839355469,
                0.03723775863647461,
                0.03721830368041992,
                0.03723980712890625,
                0.03726028823852539,
                0.03727360153198242,
                0.03725721740722656,
                0.03726847839355469,
                0.03725823974609375,
                0.03722751998901367,
                0.037266433715820314,
                0.03727155303955078,
                0.037288959503173826,
                0.03733606338500976,
                0.03730124664306641,
                0.03729305648803711,
                0.03732992172241211,
                0.037343231201171875,
                0.037285888671875,
                0.03737497711181641,
                0.03732070541381836,
                0.037341182708740234,
                0.03733196640014649,
                0.03740467071533203,
                0.03738623809814453,
                0.037362686157226564,
                0.03736064147949219,
                0.03741183853149414,
                0.037381118774414065,
                0.03743641662597656,
                0.03736883163452148,
                0.037482494354248046,
                0.0374015998840332,
                0.037477375030517575,
                0.03743436813354492,
                0.03745587158203125,
                0.03743539047241211,
                0.037596160888671876,
                0.03742924880981445,
                0.03750912094116211,
                0.037372928619384765,
                0.037577728271484374,
                0.0374917106628418,
                0.03749785614013672,
                0.03745075225830078,
                0.0376360969543457,
                0.03746815872192383,
                0.037526527404785154,
                0.0374835205078125,
                0.037664768218994144,
                0.03747840118408203,
                0.03755110549926758,
                0.0374917106628418,
                0.03760947036743164,
                0.03751628875732422,
                0.037663745880126956,
                0.03748863983154297,
                0.03767910385131836,
                0.037515262603759765,
                0.03760947036743164,
                0.0375551986694336,
                0.037556224822998044,
                0.03755929565429687,
                0.037553150177001955,
                0.037544960021972655,
                0.03761356735229492,
                0.03755110549926758,
                0.03762278366088867,
                0.03755724716186523,
                0.03754598236083984,
                0.037610496520996094,
                0.037607425689697264,
                0.03758694458007812,
                0.03764326477050781,
                0.0375551986694336,
                0.037664768218994144,
                0.0375654411315918,
                0.03760332870483398,
                0.03759308624267578,
                0.037615615844726565,
                0.037602302551269534,
                0.03762483215332031,
                0.03764019012451172,
                0.03773132705688476,
                0.03757875061035156,
                0.037700607299804685,
                0.03763302230834961,
                0.037144577026367184,
                0.03714559936523437,
                0.03716505432128906,
                0.037190654754638675,
                0.03718143844604492,
                0.03721318435668945,
                0.037182464599609374,
                0.03720499038696289,
                0.03717529678344727,
                0.03721420669555664,
                0.03722547149658203,
                0.03722956848144531,
                0.037233665466308595,
                0.03732582473754883,
                0.03721011352539062,
                0.037207038879394534,
                0.03723775863647461,
                0.0372490234375,
                0.0372592658996582,
                0.037269504547119144,
                0.037236736297607424,
                0.03727462387084961,
                0.03728179168701172,
                0.03730022430419922,
                0.0372408332824707,
                0.03731148910522461,
                0.03729612731933594,
                0.03733196640014649,
                0.037305343627929685,
                0.03736576080322265,
                0.03734527969360352,
                0.0373309440612793,
                0.03736371231079102,
                0.037343231201171875,
                0.037348350524902346,
                0.037394432067871096,
                0.03735244750976562,
                0.03739852905273437,
                0.0373493766784668,
                0.03737395095825195,
                0.03742617416381836,
                0.03744255828857422,
                0.03736678314208984,
                0.03747635269165039,
                0.03742617416381836,
                0.03747532653808594,
                0.03743948745727539,
                0.037454849243164064,
                0.03743743896484375,
                0.037544960021972655,
                0.037395454406738284,
                0.03755110549926758,
                0.03742617416381836,
                0.03761151885986328,
                0.03748454284667969,
                0.03755929565429687,
                0.037477375030517575,
                0.03757567977905273,
                0.037477375030517575,
                0.03760947036743164,
                0.037501953125,
                0.03759718322753906,
                0.03747532653808594,
                0.037539840698242184,
                0.037473281860351565,
                0.03760435104370117,
                0.03751116943359375,
                0.037648384094238284,
                0.03749990463256836,
                0.03765248107910156,
                0.03752345657348633,
                0.037628929138183595,
                0.0375654411315918,
                0.037566463470458986,
                0.03759001541137695,
                0.0375654411315918,
                0.037493759155273435,
                0.03754598236083984,
                0.03753676986694336,
                0.03762688064575195,
                0.0375470085144043,
                0.03755417633056641,
                0.0375470085144043,
                0.037564414978027344,
                0.03759513473510742,
                0.037617664337158206,
                0.03757670211791992,
                0.03771289443969727,
                0.0375838737487793,
                0.037651454925537106,
                0.037585918426513674,
                0.03766579055786133,
                0.03759820938110352,
                0.03767295837402344,
                0.03762688064575195,
                0.03771187210083008,
                0.037594112396240234,
                0.03767091369628906,
                0.03764223861694336
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 53.422707345609496
        },
        "energy": null,
        "efficiency": null
    }
}