benchmark_report.json

{
    "prefill": {
        "memory": {
            "unit": "MB",
            "max_ram": 1476.337664,
            "max_vram": 15674.114048,
            "max_reserved": 15189.671936,
            "max_allocated": 15039.57248
        },
        "latency": {
            "unit": "s",
            "mean": 0.04062422609716893,
            "stdev": 0.00010050440174907307,
            "values": [
                0.041370719909667966,
                0.04117913436889648,
                0.04108595275878906,
                0.04110847854614258,
                0.041082878112792966,
                0.04110028839111328,
                0.041022464752197264,
                0.04113919830322266,
                0.04074905776977539,
                0.04062617492675781,
                0.04065792083740234,
                0.040648704528808595,
                0.040668159484863284,
                0.040627201080322264,
                0.040629249572753906,
                0.04065075302124024,
                0.04069375991821289,
                0.04066099166870117,
                0.04063846588134765,
                0.040651775360107424,
                0.04060870361328125,
                0.04064153671264648,
                0.040653823852539066,
                0.04065484619140625,
                0.04063334274291992,
                0.04061491012573242,
                0.040629249572753906,
                0.04061798477172852,
                0.040681472778320314,
                0.040627201080322264,
                0.04065484619140625,
                0.04067942428588867,
                0.0406036491394043,
                0.04064153671264648,
                0.04065788650512695,
                0.04061695861816406,
                0.040597503662109374,
                0.04061183929443359,
                0.040627201080322264,
                0.040645633697509766,
                0.04062412643432617,
                0.04062412643432617,
                0.04062617492675781,
                0.040613887786865234,
                0.040635391235351564,
                0.04061183929443359,
                0.04062617492675781,
                0.040564735412597655,
                0.040619007110595705,
                0.04065792083740234,
                0.04064767837524414,
                0.04062515258789062,
                0.04057292938232422,
                0.040559616088867184,
                0.040627201080322264,
                0.04053401565551758,
                0.04064972686767578,
                0.04055654525756836,
                0.04059648132324219,
                0.040567806243896484,
                0.04060774230957031,
                0.04062611389160156,
                0.040591358184814456,
                0.04058726501464844,
                0.040610816955566405,
                0.040622081756591794,
                0.040613887786865234,
                0.040619007110595705,
                0.04060671997070313,
                0.04061491012573242,
                0.04060876846313476,
                0.04061798477172852,
                0.04061491012573242,
                0.04059648132324219,
                0.040576000213623044,
                0.04062003326416016,
                0.04057395172119141,
                0.0405852165222168,
                0.04058828735351563,
                0.040583168029785156,
                0.04059340667724609,
                0.04058931350708008,
                0.040612865447998046,
                0.040629249572753906,
                0.04064051055908203,
                0.0406343994140625,
                0.04064972686767578,
                0.04062412643432617,
                0.040583168029785156,
                0.040615936279296876,
                0.040671230316162106,
                0.040548351287841795,
                0.040559616088867184,
                0.040599552154541016,
                0.04060563278198242,
                0.04063436889648438,
                0.04060160064697266,
                0.040635391235351564,
                0.040551422119140625,
                0.040597503662109374,
                0.040613887786865234,
                0.04060876846313476,
                0.040632320404052735,
                0.04060467147827149,
                0.04059852981567383,
                0.040600574493408204,
                0.040583168029785156,
                0.04057395172119141,
                0.040615936279296876,
                0.04060876846313476,
                0.04060774230957031,
                0.04062412643432617,
                0.040629249572753906,
                0.04064767837524414,
                0.04055244827270508,
                0.04061894226074219,
                0.040576000213623044,
                0.040591358184814456,
                0.04063948822021484,
                0.04060774230957031,
                0.040594432830810545,
                0.04063641738891602,
                0.040653823852539066,
                0.04062617492675781,
                0.040635391235351564,
                0.04056268692016601,
                0.040586238861083986,
                0.04057190322875977,
                0.040584190368652344,
                0.04058723068237305,
                0.04060569763183594,
                0.04057395172119141,
                0.04059545516967773,
                0.040635391235351564,
                0.040597503662109374,
                0.040621055603027346,
                0.040630271911621094,
                0.04061798477172852,
                0.04058828735351563,
                0.040612865447998046,
                0.040584190368652344,
                0.040635391235351564,
                0.0405852165222168,
                0.04060876846313476,
                0.040619007110595705,
                0.04064255905151367,
                0.040632320404052735,
                0.04064153671264648,
                0.04064051055908203,
                0.040578048706054685,
                0.040594432830810545,
                0.04056883239746094,
                0.040643585205078124,
                0.0405667839050293,
                0.04060979080200195,
                0.040584190368652344,
                0.04061491012573242,
                0.04060671997070313,
                0.040600574493408204,
                0.040578048706054685,
                0.04060671997070313,
                0.040583168029785156,
                0.040637439727783206,
                0.040578048706054685,
                0.04061798477172852,
                0.040602622985839845,
                0.040610782623291014,
                0.04059952163696289,
                0.040592384338378903,
                0.04060774230957031,
                0.04062003326416016,
                0.040567806243896484,
                0.0406036491394043,
                0.04057907104492187,
                0.040567806243896484,
                0.0406036491394043,
                0.04056371307373047,
                0.0405667839050293,
                0.040602622985839845,
                0.040586238861083986,
                0.04061177444458008,
                0.040613887786865234,
                0.040568801879882814,
                0.040610816955566405,
                0.040602622985839845,
                0.04057088088989258,
                0.04059033584594727,
                0.040583168029785156,
                0.04054937744140625,
                0.04060774230957031,
                0.04056883239746094,
                0.04057088088989258,
                0.040592384338378903,
                0.040594432830810545,
                0.040551422119140625,
                0.04057190322875977,
                0.04055654525756836,
                0.04062617492675781,
                0.04064972686767578,
                0.04064665603637695,
                0.04064150238037109,
                0.04059648132324219,
                0.040594432830810545,
                0.0406640625,
                0.040653823852539066,
                0.040627201080322264,
                0.04060870361328125,
                0.04061695861816406,
                0.04055756759643555,
                0.040621055603027346,
                0.04062822341918945,
                0.04059648132324219,
                0.04058828735351563,
                0.04060467147827149,
                0.04060774230957031,
                0.04058726501464844,
                0.04056063842773437,
                0.04058009719848633,
                0.040594432830810545,
                0.04061183929443359,
                0.04059648132324219,
                0.04060160064697266,
                0.04054323196411133,
                0.040559616088867184,
                0.04061183929443359,
                0.04057702255249023,
                0.040602622985839845,
                0.040569854736328126,
                0.0405555534362793,
                0.040564735412597655,
                0.040564735412597655,
                0.04058931350708008,
                0.040681472778320314,
                0.04062003326416016,
                0.04063641738891602,
                0.0406036491394043,
                0.04060876846313476,
                0.0405852165222168,
                0.04061798477172852,
                0.04055756759643555,
                0.04053708648681641,
                0.04055654525756836,
                0.04055654525756836,
                0.04058009719848633,
                0.04057088088989258,
                0.04057190322875977
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 787.7073134503368
        },
        "energy": null,
        "efficiency": null
    },
    "decode": {
        "memory": {
            "unit": "MB",
            "max_ram": 1487.241216,
            "max_vram": 15739.12576,
            "max_reserved": 15254.683648,
            "max_allocated": 15101.761024
        },
        "latency": {
            "unit": "s",
            "mean": 3.709987153371175,
            "stdev": 0.0002519454973644119,
            "values": [
                3.710313472747803,
                3.7097000923156727,
                3.7099478950500484
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 53.369457039785765
        },
        "energy": null,
        "efficiency": null
    },
    "per_token": {
        "memory": null,
        "latency": {
            "unit": "s",
            "mean": 0.03747461771081997,
            "stdev": 0.00015825306347794503,
            "values": [
                0.03728179168701172,
                0.03722751998901367,
                0.037203968048095705,
                0.03723263931274414,
                0.037195777893066405,
                0.037198848724365234,
                0.03725516891479492,
                0.037233665466308595,
                0.03720908737182617,
                0.0372592658996582,
                0.03727872085571289,
                0.03729919815063477,
                0.037236736297607424,
                0.03731763076782227,
                0.03727769470214844,
                0.03727155303955078,
                0.03726847839355469,
                0.037266433715820314,
                0.03724390411376953,
                0.037362686157226564,
                0.03730124664306641,
                0.03729715347290039,
                0.03727462387084961,
                0.037324798583984374,
                0.03729817581176758,
                0.03739033508300781,
                0.03735551834106445,
                0.037384193420410154,
                0.03735859298706055,
                0.0373493766784668,
                0.037364734649658206,
                0.037440513610839846,
                0.037354496002197264,
                0.037381118774414065,
                0.03738726425170898,
                0.03740671920776367,
                0.03738623809814453,
                0.03744255828857422,
                0.037369857788085936,
                0.03748454284667969,
                0.03745382308959961,
                0.03746303939819336,
                0.03743436813354492,
                0.03750400161743164,
                0.03746406555175781,
                0.03748556900024414,
                0.037495807647705076,
                0.03753676986694336,
                0.03748659133911133,
                0.037591041564941405,
                0.037498878479003905,
                0.03759206390380859,
                0.037465087890625,
                0.03761971282958984,
                0.03746201705932617,
                0.037582847595214845,
                0.03748556900024414,
                0.037628929138183595,
                0.03750297546386719,
                0.03762790298461914,
                0.03750809478759766,
                0.0376995849609375,
                0.03750092697143555,
                0.03761356735229492,
                0.03753062438964844,
                0.03764633560180664,
                0.03759513473510742,
                0.037694465637207034,
                0.03756032180786133,
                0.03770675277709961,
                0.037577728271484374,
                0.0376627197265625,
                0.037599231719970705,
                0.037615615844726565,
                0.0375470085144043,
                0.03758899307250976,
                0.037594112396240234,
                0.037602302551269534,
                0.03753881454467774,
                0.03767295837402344,
                0.0375551986694336,
                0.037612545013427735,
                0.037566463470458986,
                0.0376258544921875,
                0.03760435104370117,
                0.03763097763061524,
                0.03767091369628906,
                0.03770470428466797,
                0.03762688064575195,
                0.03768627166748047,
                0.03762790298461914,
                0.037667839050292966,
                0.03764121627807617,
                0.037718017578125,
                0.037634048461914066,
                0.037789695739746096,
                0.03763507080078125,
                0.03768115234375,
                0.03772518539428711,
                0.03724595260620117,
                0.03719168090820312,
                0.03719782257080078,
                0.037250049591064455,
                0.03724595260620117,
                0.037215232849121094,
                0.03723980712890625,
                0.03723571014404297,
                0.037207038879394534,
                0.03724492645263672,
                0.03724595260620117,
                0.037256191253662106,
                0.037222400665283206,
                0.03728384017944336,
                0.037266433715820314,
                0.037238784790039066,
                0.037253120422363284,
                0.03729305648803711,
                0.03723468780517578,
                0.037285888671875,
                0.03728691101074219,
                0.03729715347290039,
                0.03729817581176758,
                0.03728998565673828,
                0.037302272796630856,
                0.0373309440612793,
                0.0373309440612793,
                0.037354496002197264,
                0.03732992172241211,
                0.03735039901733399,
                0.037362686157226564,
                0.0374466552734375,
                0.037379070281982424,
                0.03735551834106445,
                0.03737190246582031,
                0.03739955139160156,
                0.03741183853149414,
                0.037405696868896485,
                0.037384193420410154,
                0.03742822265625,
                0.03746406555175781,
                0.03746406555175781,
                0.037438465118408204,
                0.037501953125,
                0.03742924880981445,
                0.03750297546386719,
                0.03746099090576172,
                0.03754393768310547,
                0.03746918487548828,
                0.0375838737487793,
                0.037482494354248046,
                0.03760748672485351,
                0.03745785522460938,
                0.03763814544677734,
                0.03750809478759766,
                0.03757875061035156,
                0.0374917106628418,
                0.037628929138183595,
                0.037531646728515625,
                0.03763814544677734,
                0.03754393768310547,
                0.03769651031494141,
                0.037531646728515625,
                0.037618686676025394,
                0.03753881454467774,
                0.03763302230834961,
                0.037564414978027344,
                0.03763916778564453,
                0.03751833724975586,
                0.03770265579223633,
                0.037528575897216795,
                0.03770880126953125,
                0.037564414978027344,
                0.03760639953613281,
                0.03759308624267578,
                0.03759718322753906,
                0.03762278366088867,
                0.03757363128662109,
                0.03761971282958984,
                0.03766067123413086,
                0.03758182525634766,
                0.037631999969482424,
                0.037594112396240234,
                0.03762176132202148,
                0.037585918426513674,
                0.0376258544921875,
                0.03765248107910156,
                0.03772518539428711,
                0.037594112396240234,
                0.03762790298461914,
                0.037615615844726565,
                0.03770880126953125,
                0.03761151885986328,
                0.03765043258666992,
                0.03762790298461914,
                0.03776409530639648,
                0.0376545295715332,
                0.03766886520385742,
                0.03770163345336914,
                0.03721420669555664,
                0.03719987106323242,
                0.03718963241577149,
                0.03722649765014648,
                0.03723468780517578,
                0.037212158203125,
                0.03723571014404297,
                0.03726028823852539,
                0.037236736297607424,
                0.03727462387084961,
                0.03724185562133789,
                0.03727667236328125,
                0.03724800109863281,
                0.03731353759765625,
                0.03729817581176758,
                0.037288959503173826,
                0.03726028823852539,
                0.03731660842895508,
                0.03725823974609375,
                0.03728998565673828,
                0.03728486251831055,
                0.0373043212890625,
                0.03728691101074219,
                0.03732582473754883,
                0.037269504547119144,
                0.03735551834106445,
                0.03734527969360352,
                0.037351425170898435,
                0.037354496002197264,
                0.03737497711181641,
                0.037318656921386716,
                0.037395454406738284,
                0.037389312744140625,
                0.03735756683349609,
                0.03741798400878906,
                0.03743436813354492,
                0.03741183853149414,
                0.037424129486083986,
                0.03739136123657227,
                0.03742822265625,
                0.03745280075073242,
                0.03746815872192383,
                0.03739340972900391,
                0.03748147201538086,
                0.03744672012329102,
                0.037521343231201175,
                0.03745894241333008,
                0.03750604629516602,
                0.03746099090576172,
                0.03760025787353516,
                0.037459968566894535,
                0.03759308624267578,
                0.03749273681640625,
                0.03766579055786133,
                0.0374917106628418,
                0.037566463470458986,
                0.037533695220947266,
                0.03763916778564453,
                0.03750604629516602,
                0.03763097763061524,
                0.037477375030517575,
                0.03767097473144531,
                0.03749881744384766,
                0.037599231719970705,
                0.037571582794189456,
                0.03767603302001953,
                0.037582847595214845,
                0.03766067123413086,
                0.03754086303710937,
                0.037705726623535156,
                0.0375551986694336,
                0.037644287109375,
                0.037577728271484374,
                0.037596160888671876,
                0.037569534301757815,
                0.0375838737487793,
                0.037574657440185545,
                0.03758899307250976,
                0.03759823989868164,
                0.03772003173828125,
                0.037563392639160156,
                0.03759820938110352,
                0.037558273315429686,
                0.037571582794189456,
                0.0376258544921875,
                0.037647361755371096,
                0.03762995147705078,
                0.0376995849609375,
                0.03763097763061524,
                0.03766886520385742,
                0.03762278366088867,
                0.03768115234375,
                0.03763814544677734,
                0.037684223175048825,
                0.03766579055786133,
                0.03776204681396484,
                0.03766067123413086,
                0.03769343948364258,
                0.037684223175048825
            ]
        },
        "throughput": {
            "unit": "tokens/s",
            "value": 53.369457039785736
        },
        "energy": null,
        "efficiency": null
    }
}