File size: 6,338 Bytes
ac5b365 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 |
{
"prefill": {
"memory": {
"unit": "MB",
"max_ram": 3494.793216,
"max_vram": 6029.312,
"max_reserved": 5544.869888,
"max_allocated": 5402.05824
},
"latency": {
"unit": "s",
"mean": 0.3201763744354248,
"stdev": 0.0009697763372516204,
"values": [
0.3255560607910156,
0.32034609985351564,
0.31998052978515623,
0.32002969360351563,
0.319931396484375,
0.3200337829589844,
0.31998052978515623,
0.31996417236328123,
0.3199129638671875,
0.31991192626953124,
0.32001638793945314,
0.3199836120605469,
0.32001739501953125,
0.31995391845703125,
0.3199856567382813,
0.31999591064453126,
0.319857666015625,
0.320005126953125,
0.3201034240722656,
0.32001739501953125,
0.3199559631347656,
0.3200604248046875,
0.32007986450195314,
0.32001739501953125,
0.3200419921875,
0.3199754333496094,
0.32003073120117187,
0.3198873596191406,
0.3200091857910156,
0.3200081481933594,
0.32007986450195314,
0.31991397094726565
]
},
"throughput": {
"unit": "tokens/s",
"value": 2248.7605503984937
},
"energy": null,
"efficiency": null
},
"decode": {
"memory": {
"unit": "MB",
"max_ram": 3494.793216,
"max_vram": 7511.998464,
"max_reserved": 7027.556352,
"max_allocated": 6811.666944
},
"latency": {
"unit": "s",
"mean": 15.420443618774414,
"stdev": 0,
"values": [
15.420443618774414
]
},
"throughput": {
"unit": "tokens/s",
"value": 288.90219439446156
},
"energy": null,
"efficiency": null
},
"per_token": {
"memory": null,
"latency": {
"unit": "s",
"mean": 0.1557620567552971,
"stdev": 0.003946284964659726,
"values": [
0.14904631042480468,
0.14914352416992188,
0.14919577026367187,
0.14954393005371094,
0.14946202087402344,
0.14966680908203125,
0.1497876434326172,
0.15007539367675782,
0.15004876708984374,
0.15028839111328124,
0.15029862976074218,
0.1507368927001953,
0.150614013671875,
0.15076658630371093,
0.15071334838867187,
0.15137893676757813,
0.15104103088378906,
0.15133491516113282,
0.15136665344238281,
0.15196876525878905,
0.15157862854003906,
0.15214591979980469,
0.1517322235107422,
0.15250125122070313,
0.15216844177246094,
0.1525913543701172,
0.15239474487304688,
0.15318118286132812,
0.1527019500732422,
0.15311564636230468,
0.15294566345214844,
0.1536522216796875,
0.1532508087158203,
0.153891845703125,
0.15347097778320312,
0.15422361755371095,
0.15370445251464843,
0.1540894775390625,
0.15398809814453124,
0.15469056701660155,
0.1542635498046875,
0.15466188049316407,
0.1545963592529297,
0.15538482666015624,
0.1548871612548828,
0.15524354553222655,
0.15513186645507812,
0.15583334350585937,
0.15544216918945314,
0.15583027648925782,
0.15565927124023438,
0.15656755065917968,
0.1559746551513672,
0.15632691955566405,
0.15616921997070313,
0.1570529327392578,
0.15650816345214844,
0.15684402465820313,
0.15670477294921875,
0.15763763427734376,
0.15698841857910156,
0.15741439819335937,
0.1571768341064453,
0.1581230010986328,
0.1574481964111328,
0.158055419921875,
0.15774412536621094,
0.15876710510253905,
0.15799090576171876,
0.15849267578125,
0.15835443115234374,
0.15930776977539063,
0.15858073425292968,
0.15907225036621095,
0.1588940734863281,
0.15995904541015624,
0.15911424255371093,
0.15965798950195312,
0.15930982971191407,
0.160500732421875,
0.15971328735351562,
0.16013311767578126,
0.16002149963378906,
0.16116018676757812,
0.16029592895507813,
0.16075059509277342,
0.16051303100585937,
0.16163737487792967,
0.16082432556152343,
0.16137522888183595,
0.16113357543945311,
0.16221696472167968,
0.1612349395751953,
0.16177049255371093,
0.16158924865722657,
0.16294706726074218,
0.16203570556640626,
0.16257023620605468,
0.16235110473632813
]
},
"throughput": {
"unit": "tokens/s",
"value": 288.90219439446156
},
"energy": null,
"efficiency": null
}
} |