File size: 6,101 Bytes
123793a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 |
{
"prefill": {
"memory": {
"unit": "MB",
"max_ram": 3602.481152,
"max_vram": 13600.03072,
"max_reserved": 13107.2,
"max_allocated": 10874.392064
},
"latency": {
"unit": "s",
"mean": 0.3773231913248697,
"stdev": 0.004032105846019261,
"values": [
0.39744345092773437,
0.38059417724609373,
0.3768924255371094,
0.37622784423828126,
0.37658932495117187,
0.37643670654296874,
0.37635992431640625,
0.3764459533691406,
0.37676953125,
0.3766363830566406,
0.3758919677734375,
0.3761131591796875,
0.3763240966796875,
0.3764326477050781,
0.37676953125,
0.3764981689453125,
0.37646951293945313,
0.3763261413574219,
0.3764234313964844,
0.37589297485351564,
0.3767470092773437,
0.3763292236328125,
0.3761827697753906,
0.3762247619628906,
0.37629031372070315,
0.37621145629882813,
0.3762032775878906
]
},
"throughput": {
"unit": "tokens/s",
"value": 848.0793318756936
},
"energy": null,
"efficiency": null
},
"decode": {
"memory": {
"unit": "MB",
"max_ram": 3622.457344,
"max_vram": 13816.037376,
"max_reserved": 13323.206656,
"max_allocated": 10993.732608
},
"latency": {
"unit": "s",
"mean": 28.168144866943358,
"stdev": 0,
"values": [
28.168144866943358
]
},
"throughput": {
"unit": "tokens/s",
"value": 70.29216902117055
},
"energy": null,
"efficiency": null
},
"per_token": {
"memory": null,
"latency": {
"unit": "s",
"mean": 0.28452671582771066,
"stdev": 0.0049875892887557205,
"values": [
0.2992322692871094,
0.2963630065917969,
0.2955704345703125,
0.29482598876953126,
0.29479730224609374,
0.29364532470703125,
0.2929387512207031,
0.29414093017578125,
0.29279232788085935,
0.2917713928222656,
0.29185842895507813,
0.2903203735351563,
0.2909624328613281,
0.29145086669921877,
0.2901166076660156,
0.2893209533691406,
0.29001214599609376,
0.29043405151367185,
0.28881613159179687,
0.28931686401367185,
0.28846795654296875,
0.28854989624023436,
0.2885509033203125,
0.28781976318359376,
0.28798974609375,
0.28707839965820314,
0.28717465209960935,
0.2871705627441406,
0.28670053100585935,
0.2863319091796875,
0.28658892822265625,
0.28629605102539063,
0.28560385131835936,
0.28722994995117185,
0.2855628662109375,
0.28619161987304687,
0.2864005126953125,
0.28537957763671873,
0.2857574462890625,
0.2856570739746094,
0.28499456787109373,
0.2846719970703125,
0.28515225219726564,
0.2836234130859375,
0.2847836303710938,
0.2847744140625,
0.2835599365234375,
0.28438424682617186,
0.28307763671875,
0.28367974853515626,
0.2833827819824219,
0.28306533813476564,
0.28318310546875,
0.283335693359375,
0.28157952880859377,
0.2825861206054687,
0.28271923828125,
0.2822932434082031,
0.28188568115234375,
0.2817966003417969,
0.2816174011230469,
0.2814392395019531,
0.28117196655273435,
0.28046029663085936,
0.2814535827636719,
0.28160614013671875,
0.2794721374511719,
0.2817894287109375,
0.2799134826660156,
0.28016229248046876,
0.27944244384765626,
0.28090060424804686,
0.27975372314453123,
0.2797445068359375,
0.28207308959960936,
0.2814228515625,
0.2799288330078125,
0.279947265625,
0.27979571533203124,
0.280489990234375,
0.27977828979492186,
0.2800814208984375,
0.27924581909179685,
0.28012442016601563,
0.2795233154296875,
0.2790953063964844,
0.2790983581542969,
0.2792755126953125,
0.278697998046875,
0.2785955810546875,
0.27879833984375,
0.2794700927734375,
0.2787348327636719,
0.2788065185546875,
0.27856280517578125,
0.2788147277832031,
0.27881063842773435,
0.2785607604980469,
0.2777671813964844
]
},
"throughput": {
"unit": "tokens/s",
"value": 70.29216902117055
},
"energy": null,
"efficiency": null
}
} |