benchmark_report.json

{
    "prefill": {
        "memory": {
            "unit": "MB",
            "max_ram": 1476.255744,
            "max_vram": 15674.114048,
            "max_reserved": 15189.671936,
            "max_allocated": 15039.57248
        },
        "latency": {
            "unit": "s",
            "mean": 0.04060185148657821,
            "stdev": 0.00011117871291931433,
            "values": [
                0.041291744232177734,
                0.04127231979370117,
                0.04105215835571289,
                0.041289726257324216,
                0.04114636611938476,
                0.04116582489013672,
                0.0411228141784668,
                0.041090049743652345,
                0.04072652816772461,
                0.040635391235351564,
                0.04060774230957031,
                0.04075212860107422,
                0.04060665512084961,
                0.04059648132324219,
                0.040578048706054685,
                0.040591358184814456,
                0.04064153671264648,
                0.040584190368652344,
                0.04057702255249023,
                0.040613887786865234,
                0.040574977874755856,
                0.04059648132324219,
                0.04063436889648438,
                0.040592384338378903,
                0.04063948822021484,
                0.04061695861816406,
                0.04064255905151367,
                0.04063436889648438,
                0.04063948822021484,
                0.040586238861083986,
                0.040586238861083986,
                0.04062617492675781,
                0.04060774230957031,
                0.040645633697509766,
                0.040597503662109374,
                0.04061491012573242,
                0.04055449676513672,
                0.04058009719848633,
                0.040597503662109374,
                0.04065894317626953,
                0.04061183929443359,
                0.04062310409545898,
                0.04059033584594727,
                0.04059852981567383,
                0.04060160064697266,
                0.040591358184814456,
                0.04059545516967773,
                0.04057292938232422,
                0.04058931350708008,
                0.0406036491394043,
                0.04060774230957031,
                0.040584190368652344,
                0.04055551910400391,
                0.04056576156616211,
                0.04057798385620117,
                0.04058108901977539,
                0.04058828735351563,
                0.04056572723388672,
                0.040559616088867184,
                0.040528896331787106,
                0.04060774230957031,
                0.040574977874755856,
                0.04055551910400391,
                0.04056063842773437,
                0.040574977874755856,
                0.04060876846313476,
                0.040564735412597655,
                0.04058009719848633,
                0.040610816955566405,
                0.040621055603027346,
                0.040600574493408204,
                0.040600574493408204,
                0.04058726501464844,
                0.04054425430297852,
                0.04056371307373047,
                0.04055244827270508,
                0.040513534545898434,
                0.04055244827270508,
                0.040581119537353515,
                0.04055449676513672,
                0.0405667839050293,
                0.04056576156616211,
                0.040602622985839845,
                0.04060671997070313,
                0.04060671997070313,
                0.040580158233642576,
                0.04062310409545898,
                0.040594432830810545,
                0.040569854736328126,
                0.0405852165222168,
                0.040583168029785156,
                0.040583168029785156,
                0.040581119537353515,
                0.040581119537353515,
                0.04055859375,
                0.04055756759643555,
                0.040584190368652344,
                0.0406036491394043,
                0.04055654525756836,
                0.040578048706054685,
                0.04056576156616211,
                0.04057907104492187,
                0.040584190368652344,
                0.040619007110595705,
                0.04061491012573242,
                0.040574977874755856,
                0.04061692810058594,
                0.04055756759643555,
                0.040645633697509766,
                0.04060979080200195,
                0.04054118347167969,
                0.04055654525756836,
                0.040559616088867184,
                0.04058214569091797,
                0.040551422119140625,
                0.04061183929443359,
                0.040556449890136716,
                0.04057907104492187,
                0.04056268692016601,
                0.040574977874755856,
                0.04056576156616211,
                0.04063334274291992,
                0.040581119537353515,
                0.04052377700805664,
                0.04054425430297852,
                0.040559616088867184,
                0.0405852165222168,
                0.040591358184814456,
                0.04056268692016601,
                0.04058828735351563,
                0.04058931350708008,
                0.04054732894897461,
                0.04060569763183594,
                0.04059340667724609,
                0.04051968002319336,
                0.04055449676513672,
                0.04054732894897461,
                0.04059033584594727,
                0.040581119537353515,
                0.040600574493408204,
                0.04054118347167969,
                0.04055756759643555,
                0.040586238861083986,
                0.04057702255249023,
                0.04060467147827149,
                0.040613887786865234,
                0.04060876846313476,
                0.04061183929443359,
                0.040630271911621094,
                0.04057190322875977,
                0.04060774230957031,
                0.040592384338378903,
                0.04057088088989258,
                0.04053606414794922,
                0.04059648132324219,
                0.04056268692016601,
                0.04058009719848633,
                0.040548351287841795,
                0.04058214569091797,
                0.04054732894897461,
                0.0405852165222168,
                0.04055551910400391,
                0.04060160064697266,
                0.040591358184814456,
                0.04054425430297852,
                0.04057395172119141,
                0.04055449676513672,
                0.040586238861083986,
                0.04058931350708008,
                0.04060160064697266,
                0.040551422119140625,
                0.040561664581298826,
                0.040592384338378903,
                0.04058828735351563,
                0.0405401611328125,
                0.040576000213623044,
                0.04055449676513672,
                0.04054214477539062,
                0.04057292938232422,
                0.04055859375,
                0.04053708648681641,
                0.04057907104492187,
                0.040548351287841795,
                0.04055551910400391,
                0.040553470611572266,
                0.04055859375,
                0.04053299331665039,
                0.04057702255249023,
                0.04056576156616211,
                0.04056883239746094,
                0.040584190368652344,
                0.040599552154541016,
                0.040629249572753906,
                0.04057907104492187,
                0.04055244827270508,
                0.040559616088867184,
                0.04054425430297852,
                0.04063334274291992,
                0.04060979080200195,
                0.040621055603027346,
                0.040612865447998046,
                0.040599552154541016,
                0.04060876846313476,
                0.04059033584594727,
                0.040606689453125,
                0.040584190368652344,
                0.040574977874755856,
                0.04057395172119141,
                0.040548351287841795,
                0.040564735412597655,
                0.04057088088989258,
                0.04055654525756836,
                0.040553470611572266,
                0.040553470611572266,
                0.040584190368652344,
                0.040602622985839845,
                0.0405852165222168,
                0.04055244827270508,
                0.04055756759643555,
                0.04055244827270508,
                0.04056063842773437,
                0.040582080841064454,
                0.04057088088989258,
                0.04056268692016601,
                0.04054937744140625,
                0.04056576156616211,
                0.04054118347167969,
                0.04055449676513672,
                0.040569854736328126,
                0.040610816955566405,
                0.04056268692016601,
                0.040578048706054685,
                0.04064044952392578,
                0.04059852981567383,
                0.040615936279296876,
                0.04056063842773437,
                0.04054425430297852,
                0.04057702255249023,
                0.040583168029785156,
                0.04057907104492187,
                0.04053606414794922,
                0.04056268692016601,
                0.04053708648681641,
                0.04057907104492187,
                0.04053299331665039,
                0.04058003234863281
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 788.1413981965396
        },
        "energy": null,
        "efficiency": null
    },
    "decode": {
        "memory": {
            "unit": "MB",
            "max_ram": 1487.167488,
            "max_vram": 15739.12576,
            "max_reserved": 15254.683648,
            "max_allocated": 15101.761024
        },
        "latency": {
            "unit": "s",
            "mean": 3.707151365915934,
            "stdev": 0.0009040982757821327,
            "values": [
                3.7083924446105963,
                3.706264587402344,
                3.7067970657348623
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 53.41028203499851
        },
        "energy": null,
        "efficiency": null
    },
    "per_token": {
        "memory": null,
        "latency": {
            "unit": "s",
            "mean": 0.03744597339309023,
            "stdev": 0.0001584698492390454,
            "values": [
                0.0372674560546875,
                0.0372408332824707,
                0.03715071868896484,
                0.037149696350097655,
                0.037163009643554686,
                0.03719680023193359,
                0.03718348693847656,
                0.03721420669555664,
                0.03723571014404297,
                0.03724492645263672,
                0.037253120422363284,
                0.03728076934814453,
                0.03722956848144531,
                0.03727155303955078,
                0.037282817840576174,
                0.03724390411376953,
                0.03722956848144531,
                0.03724288177490234,
                0.03728793716430664,
                0.03727667236328125,
                0.03726847839355469,
                0.03729612731933594,
                0.03727872085571289,
                0.03725823974609375,
                0.037308414459228514,
                0.03728793716430664,
                0.03732275390625,
                0.03734220886230469,
                0.037310462951660156,
                0.037351425170898435,
                0.037359615325927735,
                0.037407745361328126,
                0.03734630584716797,
                0.03737702560424805,
                0.03741798400878906,
                0.03740262222290039,
                0.037397502899169925,
                0.03739136123657227,
                0.03736883163452148,
                0.03744768142700195,
                0.03735551834106445,
                0.03745587158203125,
                0.03741900634765625,
                0.03745280075073242,
                0.0374466552734375,
                0.037466110229492186,
                0.03746099090576172,
                0.037539840698242184,
                0.03741491317749023,
                0.03755724716186523,
                0.03748556900024414,
                0.03757875061035156,
                0.03750297546386719,
                0.03768012619018555,
                0.03749068832397461,
                0.03757670211791992,
                0.03746406555175781,
                0.03762688064575195,
                0.03747020721435547,
                0.03758796691894531,
                0.037539840698242184,
                0.03763097763061524,
                0.03748556900024414,
                0.03762790298461914,
                0.037510143280029294,
                0.03765555191040039,
                0.037526527404785154,
                0.03761971282958984,
                0.03754905700683594,
                0.03769343948364258,
                0.0375551986694336,
                0.03761971282958984,
                0.03753881454467774,
                0.0376360969543457,
                0.037563392639160156,
                0.03759820938110352,
                0.037574657440185545,
                0.037585918426513674,
                0.037610496520996094,
                0.037684223175048825,
                0.03756851196289063,
                0.03759718322753906,
                0.0375838737487793,
                0.037599231719970705,
                0.037571582794189456,
                0.03763711929321289,
                0.037623809814453124,
                0.03771187210083008,
                0.0376258544921875,
                0.03763711929321289,
                0.037628929138183595,
                0.03762278366088867,
                0.03762995147705078,
                0.03762790298461914,
                0.03762790298461914,
                0.03776102447509765,
                0.03763507080078125,
                0.037667839050292966,
                0.03760947036743164,
                0.03720806503295898,
                0.037131263732910154,
                0.037174270629882815,
                0.03721011352539062,
                0.037190654754638675,
                0.03722649765014648,
                0.037222400665283206,
                0.03721420669555664,
                0.03718963241577149,
                0.03724185562133789,
                0.03723775863647461,
                0.03721420669555664,
                0.037200897216796876,
                0.03726540756225586,
                0.03728076934814453,
                0.03723571014404297,
                0.03722956848144531,
                0.037266433715820314,
                0.03721420669555664,
                0.03729305648803711,
                0.03721011352539062,
                0.03726540756225586,
                0.03726028823852539,
                0.03727769470214844,
                0.03722751998901367,
                0.0373043212890625,
                0.03728486251831055,
                0.037324798583984374,
                0.03728793716430664,
                0.037354496002197264,
                0.03731148910522461,
                0.03739033508300781,
                0.037335041046142575,
                0.037335041046142575,
                0.037353473663330077,
                0.037359615325927735,
                0.037353473663330077,
                0.037400577545166014,
                0.037379070281982424,
                0.03743539047241211,
                0.037384193420410154,
                0.03742617416381836,
                0.037381118774414065,
                0.03748863983154297,
                0.03738828659057617,
                0.037501953125,
                0.037407745361328126,
                0.03750400161743164,
                0.0374128646850586,
                0.03757567977905273,
                0.037416961669921874,
                0.03755929565429687,
                0.03744153594970703,
                0.03760947036743164,
                0.03750502395629883,
                0.03754905700683594,
                0.037493759155273435,
                0.03764019012451172,
                0.03747635269165039,
                0.03757670211791992,
                0.03747123336791992,
                0.037602302551269534,
                0.037477375030517575,
                0.037544960021972655,
                0.03747123336791992,
                0.03759820938110352,
                0.037479423522949216,
                0.037645313262939455,
                0.03748147201538086,
                0.03766681671142578,
                0.037561344146728515,
                0.03761459350585938,
                0.03753472137451172,
                0.03754393768310547,
                0.03751731109619141,
                0.03758489608764649,
                0.03749990463256836,
                0.037594112396240234,
                0.037515262603759765,
                0.03763916778564453,
                0.03754086303710937,
                0.03757056045532227,
                0.037591041564941405,
                0.037574657440185545,
                0.037596160888671876,
                0.03762995147705078,
                0.03755724716186523,
                0.03770982360839844,
                0.03752345657348633,
                0.03760639953613281,
                0.037571582794189456,
                0.03763711929321289,
                0.03759820938110352,
                0.037664768218994144,
                0.03765248107910156,
                0.03772825622558594,
                0.03758694458007812,
                0.03765760040283203,
                0.03758694458007812,
                0.0371701774597168,
                0.037154815673828126,
                0.037174270629882815,
                0.03719168090820312,
                0.037176319122314457,
                0.03720806503295898,
                0.03718963241577149,
                0.03719372940063476,
                0.03721830368041992,
                0.0372490234375,
                0.03724390411376953,
                0.03720601654052735,
                0.03724288177490234,
                0.03726335906982422,
                0.03724492645263672,
                0.03722649765014648,
                0.03726131057739258,
                0.03728691101074219,
                0.03722956848144531,
                0.03727667236328125,
                0.03726540756225586,
                0.037253120422363284,
                0.0373043212890625,
                0.03725823974609375,
                0.0372408332824707,
                0.037275646209716795,
                0.03729510498046875,
                0.03733196640014649,
                0.03731660842895508,
                0.037318656921386716,
                0.037308414459228514,
                0.037348350524902346,
                0.037326847076416016,
                0.03732889556884766,
                0.03733401489257813,
                0.03739340972900391,
                0.03736064147949219,
                0.03738828659057617,
                0.037407745361328126,
                0.0374015998840332,
                0.037564414978027344,
                0.03742822265625,
                0.037375999450683595,
                0.03745382308959961,
                0.03739648056030274,
                0.03747840118408203,
                0.03741183853149414,
                0.037454849243164064,
                0.03745792007446289,
                0.0375551986694336,
                0.03745177459716797,
                0.037558273315429686,
                0.03743539047241211,
                0.03762073516845703,
                0.03747840118408203,
                0.0375654411315918,
                0.03747020721435547,
                0.037591041564941405,
                0.03746713638305664,
                0.03762688064575195,
                0.0375203857421875,
                0.03761971282958984,
                0.03748147201538086,
                0.037526527404785154,
                0.03749478530883789,
                0.0375838737487793,
                0.03750400161743164,
                0.03764633560180664,
                0.037539840698242184,
                0.03766067123413086,
                0.037510143280029294,
                0.037612545013427735,
                0.03757567977905273,
                0.03756851196289063,
                0.037550079345703126,
                0.037533695220947266,
                0.03757670211791992,
                0.03758489608764649,
                0.03755110549926758,
                0.03762688064575195,
                0.037553150177001955,
                0.03759206390380859,
                0.037574657440185545,
                0.037591041564941405,
                0.037594112396240234,
                0.03760537719726562,
                0.03756748962402344,
                0.037768192291259765,
                0.0375838737487793,
                0.03762790298461914,
                0.03760947036743164,
                0.03763916778564453,
                0.03762790298461914,
                0.03765248107910156,
                0.0376545295715332,
                0.03769753646850586,
                0.03760947036743164,
                0.037645313262939455,
                0.03762688064575195
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 53.41028203499853
        },
        "energy": null,
        "efficiency": null
    }
}