Update README.md

README.md

@@ -284,3 +284,254 @@ lm_eval \
     </tr>
   </tbody>
 </table>
+
+
+## Inference Performance
+
+
+This model achieves up to 2.7x speedup in single-stream deployment and up to 1.5x speedup in multi-stream asynchronous deployment, depending on hardware and use-case scenario.
+The following performance benchmarks were conducted with [vLLM](https://docs.vllm.ai/en/latest/) version 0.6.7.2, and [GuideLLM](https://github.com/neuralmagic/guidellm).
+
+<details>
+<summary>Benchmarking Command</summary>
+
+```
+guidellm --model neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16 --target "http://localhost:8000/v1" --data-type emulated --data "prompt_tokens=<prompt_tokens>,generated_tokens=<generated_tokens>" --max seconds 360 --backend aiohttp_server
+```
+</details>
+
+### Single-stream performance (measured with vLLM version 0.7.2)
+<table>
+  <thead>
+    <tr>
+      <th></th>
+      <th></th>
+      <th></th>
+      <th></th>
+      <th style="text-align: center;" colspan="2" >Instruction Following<br>prefill: 256 tokens<br>decode: 128 tokens</th>
+      <th style="text-align: center;" colspan="2" >Multi-turn Chat<br>prefill: 512 tokens<br>decode: 256 tokens</th>
+      <th style="text-align: center;" colspan="2" >Docstring Generation<br>prefill: 768 tokens<br>decode: 128 tokens</th>
+      <th style="text-align: center;" colspan="2" >RAG<br>prefill: 1024 tokens<br>decode: 128 tokens</th>
+      <th style="text-align: center;" colspan="2" >Code Completion<br>prefill: 256 tokens<br>decode: 1024 tokens</th>
+      <th style="text-align: center;" colspan="2" >Code Fixing<br>prefill: 1024 tokens<br>decode: 1024 tokens</th>
+      <th style="text-align: center;" colspan="2" >Large Summarization<br>prefill: 4096 tokens<br>decode: 512 tokens</th>
+      <th style="text-align: center;" colspan="2" >Large RAG<br>prefill: 10240 tokens<br>decode: 1536 tokens</th>
+    </tr>
+    <tr>
+      <th>GPU class</th>
+      <th>Number of GPUs</th>
+      <th>Model</th>
+      <th>Cost reduction</th>
+      <th>Latency (s)</th>
+      <th>QPS</th>
+      <th>Latency (s)</th>
+      <th>QPS</th>
+      <th>Latency (s)</th>
+      <th>QPS</th>
+      <th>Latency (s)</th>
+      <th>QPS</th>
+      <th>Latency (s)</th>
+      <th>QPS</th>
+      <th>Latency (s)</th>
+      <th>QPS</th>
+      <th>Latency (s)</th>
+      <th>QPS</th>
+      <th>Latency (s)</th>
+      <th>QPS</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <th rowspan="3" valign="top">A6000</th>
+      <td>4</td>
+      <th>deepseek-ai/DeepSeek-R1-Distill-Llama-70B</th>
+      <td></td>
+      <td>7.4</td>
+      <td>151.6</td>
+      <td>14.9</td>
+      <td>75.6</td>
+      <td>7.5</td>
+      <td>149.1</td>
+      <td>7.7</td>
+      <td>146.4</td>
+      <td>57.2</td>
+      <td>19.7</td>
+      <td>58.9</td>
+      <td>19.1</td>
+      <td>31.9</td>
+      <td>35.2</td>
+      <td>98.4</td>
+      <td>11.4</td>
+    </tr>
+    <tr>
+      <td>2</td>
+      <th>neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8</th>
+      <td>1.93</td>
+      <td>7.7</td>
+      <td>291.8</td>
+      <td>15.2</td>
+      <td>147.7</td>
+      <td>7.8</td>
+      <td>287.5</td>
+      <td>8.0</td>
+      <td>282.5</td>
+      <td>60.7</td>
+      <td>37.1</td>
+      <td>60.2</td>
+      <td>37.4</td>
+      <td>32.3</td>
+      <td>69.6</td>
+      <td>104.0</td>
+      <td>21.6</td>
+    </tr>
+    <tr>
+      <td>2</td>
+      <th>neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16</th>
+      <td>2.83</td>
+      <td>4.9</td>
+      <td>457.1</td>
+      <td>10.0</td>
+      <td>225.1</td>
+      <td>5.5</td>
+      <td>411.5</td>
+      <td>5.8</td>
+      <td>388.9</td>
+      <td>38.9</td>
+      <td>57.8</td>
+      <td>39.2</td>
+      <td>57.4</td>
+      <td>23.7</td>
+      <td>94.8</td>
+      <td>76.6</td>
+      <td>29.4</td>
+    </tr>
+    <tr>
+      <th rowspan="3" valign="top">A100</th>
+      <td>2</td>
+      <th>deepseek-ai/DeepSeek-R1-Distill-Llama-70B</th>
+      <td></td>
+      <td>6.4</td>
+      <td>156.9</td>
+      <td>12.8</td>
+      <td>78.8</td>
+      <td>6.6</td>
+      <td>152.9</td>
+      <td>6.7</td>
+      <td>151.2</td>
+      <td>50.4</td>
+      <td>19.9</td>
+      <td>50.8</td>
+      <td>19.8</td>
+      <td>27.0</td>
+      <td>37.3</td>
+      <td>85.4</td>
+      <td>11.8</td>
+    </tr>
+    <tr>
+      <td>2</td>
+      <th>neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8</th>
+      <td>1.48</td>
+      <td>4.1</td>
+      <td>245.2</td>
+      <td>8.2</td>
+      <td>122.6</td>
+      <td>4.2</td>
+      <td>238.0</td>
+      <td>4.3</td>
+      <td>234.9</td>
+      <td>32.4</td>
+      <td>31.0</td>
+      <td>32.8</td>
+      <td>30.7</td>
+      <td>17.6</td>
+      <td>57.1</td>
+      <td>90.8</td>
+      <td>11.1</td>
+    </tr>
+    <tr>
+      <td>1</td>
+      <th>neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16</th>
+      <td>2.69</td>
+      <td>4.6</td>
+      <td>439.9</td>
+      <td>9.2</td>
+      <td>219.7</td>
+      <td>4.9</td>
+      <td>406.5</td>
+      <td>5.2</td>
+      <td>388.5</td>
+      <td>35.3</td>
+      <td>57.0</td>
+      <td>36.3</td>
+      <td>55.5</td>
+      <td>21.2</td>
+      <td>94.9</td>
+      <td>68.1</td>
+      <td>29.5</td>
+    </tr>
+    <tr>
+      <th rowspan="3" valign="top">H100</th>
+      <td>2</td>
+      <th>deepseek-ai/DeepSeek-R1-Distill-Llama-70B</th>
+      <td></td>
+      <td>3.8</td>
+      <td>149.0</td>
+      <td>7.6</td>
+      <td>74.2</td>
+      <td>3.9</td>
+      <td>145.9</td>
+      <td>3.9</td>
+      <td>143.9</td>
+      <td>30.0</td>
+      <td>18.8</td>
+      <td>30.4</td>
+      <td>18.6</td>
+      <td>16.1</td>
+      <td>34.9</td>
+      <td>56.5</td>
+      <td>10.0</td>
+    </tr>
+    <tr>
+      <td>2</td>
+      <th>neuralmagic/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic</th>
+      <td>1.39</td>
+      <td>2.7</td>
+      <td>209.9</td>
+      <td>5.3</td>
+      <td>105.6</td>
+      <td>2.7</td>
+      <td>206.7</td>
+      <td>2.8</td>
+      <td>203.2</td>
+      <td>21.1</td>
+      <td>26.7</td>
+      <td>21.4</td>
+      <td>26.4</td>
+      <td>11.5</td>
+      <td>48.9</td>
+      <td>47.2</td>
+      <td>12.0</td>
+    </tr>
+    <tr>
+      <td>1</td>
+      <th>neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16</th>
+      <td>1.83</td>
+      <td>4.0</td>
+      <td>276.7</td>
+      <td>7.9</td>
+      <td>138.1</td>
+      <td>4.1</td>
+      <td>266.2</td>
+      <td>4.2</td>
+      <td>262.0</td>
+      <td>31.2</td>
+      <td>35.1</td>
+      <td>31.8</td>
+      <td>34.4</td>
+      <td>17.8</td>
+      <td>61.3</td>
+      <td>61.4</td>
+      <td>17.8</td>
+    </tr>
+  </tbody>
+</table>