Update README.md

README.md

@@ -90,170 +90,104 @@ This model was created by applying [LLM Compressor with calibration samples from
 
 ## Evaluation
 
-This model was evaluated on the well-known OpenLLM v1, OpenLLM v2, HumanEval, and HumanEval_64 benchmarks. All evaluations were conducted using [lm-evaluation-harness](https://github.com/neuralmagic/lm-evaluation-harness).
+This model was evaluated on the well-known OpenLLM v1 and HumanEval_64 benchmarks using [lm-evaluation-harness](https://github.com/neuralmagic/lm-evaluation-harness). The Reasoning evals were done using [ligheval](https://github.com/neuralmagic/lighteval).
+
+### Accuracy
+
 <table>
   <thead>
     <tr>
       <th>Category</th>
       <th>Metric</th>
       <th>DeepSeek-R1-Distill-Qwen-32B</th>
-      <th>DeepSeek-R1-Distill-Qwen-32B-NVFP4</th>
-      <th>Recovery (%)</th>
+      <th>DeepSeek-R1-Distill-Qwen-32B NVFP4</th>
+      <th>Recovery</th>
     </tr>
   </thead>
   <tbody>
+    <!-- OpenLLM V1 -->
     <tr>
       <td rowspan="7"><b>OpenLLM V1</b></td>
-      <td>ARC Challenge</td>
-      <td>67.66</td>
-      <td>64.25</td>
-      <td>94.94%</td>
+      <td>arc_challenge</td>
+      <td>63.48</td>
+      <td>62.12</td>
+      <td>97.86</td>
     </tr>
     <tr>
-      <td>GSM8K</td>
-      <td>83.02</td>
-      <td>84.84</td>
-      <td>102.19%</td>
+      <td>gsm8k</td>
+      <td>86.88</td>
+      <td>88.32</td>
+      <td>101.66</td>
     </tr>
     <tr>
-      <td>Hellaswag</td>
-      <td>83.79</td>
-      <td>83.28</td>
-      <td>99.39%</td>
+      <td>hellaswag</td>
+      <td>83.51</td>
+      <td>82.38</td>
+      <td>98.65</td>
     </tr>
     <tr>
-      <td>MMLU</td>
-      <td>81.25</td>
-      <td>80.79</td>
-      <td>99.43%</td>
+      <td>mmlu</td>
+      <td>80.97</td>
+      <td>80.42</td>
+      <td>99.32</td>
     </tr>
     <tr>
-      <td>TruthfulQA-mc2</td>
-      <td>58.37</td>
-      <td>57.50</td>
-      <td>98.51%</td>
+      <td>truthfulqa_mc2</td>
+      <td>56.82</td>
+      <td>55.75</td>
+      <td>98.12</td>
     </tr>
     <tr>
-      <td>Winogrande</td>
-      <td>75.77</td>
-      <td>76.40</td>
-      <td>100.83%</td>
+      <td>winogrande</td>
+      <td>75.93</td>
+      <td>75.14</td>
+      <td>98.96</td>
     </tr>
     <tr>
       <td><b>Average</b></td>
-      <td><b>74.98</b></td>
-      <td><b>74.51</b></td>
-      <td><b>99.38%</b></td>
-    </tr>
-    <tr>
-      <td rowspan="7"><b>OpenLLM V2</b></td>
-      <td>MMLU-Pro</td>
-      <td></td>
-      <td></td>
-      <td>%</td>
-    </tr>
-    <tr>
-      <td>IFEval</td>
-      <td></td>
-      <td></td>
-      <td>%</td>
-    </tr>
-    <tr>
-      <td>BBH</td>
-      <td></td>
-      <td></td>
-      <td>%</td>
-    </tr>
-    <tr>
-      <td>Math-Hard</td>
-      <td></td>
-      <td></td>
-      <td>%</td>
-    </tr>
-    <tr>
-      <td>GPQA</td>
-      <td></td>
-      <td></td>
-      <td>%</td>
-    </tr>
-    <tr>
-      <td>MuSR</td>
-      <td></td>
-      <td></td>
-      <td>%</td>
-    </tr>
-    <tr>
-      <td><b>Average</b></td>
-      <td><b></b></td>
-      <td><b></b></td>
-      <td><b>%</b></td>
+      <td><b>74.60</b></td>
+      <td><b>74.02</b></td>
+      <td><b>99.23</b></td>
     </tr>
+    <!-- Reasoning -->
     <tr>
       <td rowspan="4"><b>Reasoning</b></td>
-      <td>Math 500</td>
-      <td>95.09</td>
-      <td>95.60</td>
-      <td>100.54%</td>
-    </tr>
-    <tr>
-      <td>GPQA (diamond)</td>
-      <td>64.05</td>
-      <td>61.11</td>
-      <td>95.41%</td>
-    </tr>
-    <tr>
-      <td>AIME25</td>
-      <td>69.75 (AIME24)</td>
-      <td>53.33</td>
-      <td>76.45%</td>
-    </tr>
-    <tr>
-      <td>LCB: Code Generation</td>
-      <td>–</td>
-      <td>54.29</td>
-      <td>–</td>
+      <td>AIME24 (0-shot)</td>
+      <td>72.41</td>
+      <td>62.07</td>
+      <td>85.69</td>
     </tr>
     <tr>
-      <td rowspan="6"><b>Coding</b></td>
-      <td>HumanEval Instruct pass@1</td>
-      <td>–</td>
-      <td>–</td>
-      <td>–</td>
+      <td>AIME25 (0-shot)</td>
+      <td>58.62</td>
+      <td>62.07</td>
+      <td>105.89</td>
     </tr>
     <tr>
-      <td>HumanEval 64 Instruct pass@2</td>
-      <td>–</td>
-      <td>–</td>
-      <td>–</td>
+      <td>GPQA (Diamond, 0-shot)</td>
+      <td>68.02</td>
+      <td>65.48</td>
+      <td>96.27</td>
     </tr>
     <tr>
-      <td>HumanEval 64 Instruct pass@8</td>
-      <td>–</td>
-      <td>–</td>
-      <td>–</td>
-    </tr>
-    <tr>
-      <td>HumanEval 64 Instruct pass@16</td>
-      <td>–</td>
-      <td>–</td>
-      <td>–</td>
-    </tr>
-    <tr>
-      <td>HumanEval 64 Instruct pass@32</td>
-      <td>–</td>
-      <td>–</td>
-      <td>–</td>
+      <td><b>Average</b></td>
+      <td><b>66.35</b></td>
+      <td><b>63.21</b></td>
+      <td><b>95.95</b></td>
     </tr>
+    <!-- Coding -->
     <tr>
-      <td>HumanEval 64 Instruct pass@64</td>
-      <td>–</td>
-      <td>–</td>
-      <td>–</td>
+      <td rowspan="2"><b>Coding</b></td>
+      <td>HumanEval_64 pass@2</td>
+      <td>90.00</td>
+      <td>89.32</td>
+      <td>99.24</td>
     </tr>
   </tbody>
 </table>
 
 
+
 ### Reproduction
 
 The results were obtained using the following commands:
@@ -273,34 +207,41 @@ lm_eval \
 ```
 
 
-#### OpenLLM v2
-```
-lm_eval \
-  --model vllm \
-  --model_args pretrained="RedHatAI/DeepSeek-R1-Distill-Qwen-32B-NVFP4",dtype=auto,max_model_len=15000,tensor_parallel_size=2,enable_chunked_prefill=True,enforce_eager=True\
-  --apply_chat_template \
-  --fewshot_as_multiturn \
-  --tasks leaderboard \
-  --batch_size auto
-```
+####  HumanEval_64
 
-#### HumanEval and HumanEval_64
 ```
 lm_eval \
   --model vllm \
   --model_args pretrained="RedHatAI/DeepSeek-R1-Distill-Qwen-32B-NVFP4",dtype=auto,max_model_len=4096,tensor_parallel_size=2,enable_chunked_prefill=True,enforce_eager=True\
   --apply_chat_template \
   --fewshot_as_multiturn \
-  --tasks humaneval_instruct \
+  --tasks humaneval_64_instruct \
   --batch_size auto
+```
 
+#### LightEval 
+```
+# --- model_args.yaml ---
+cat > model_args.yaml <<'YAML'
+model_parameters:
+  model_name: "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-NVFP4"
+  dtype: auto
+  gpu_memory_utilization: 0.9
+  tensor_parallel_size: 2
+  max_model_length: 40960
+  generation_parameters:
+    seed: 42
+    temperature: 0.6
+    top_k: 20
+    top_p: 0.95
+    min_p: 0.0
+    max_new_tokens: 32768
+YAML
+
+lighteval vllm model_args.yaml \
+  "lighteval|aime24|0,lighteval|aime25|0,lighteval|gpqa:diamond|0" \
+  --max-samples -1 \
+  --output-dir out_dir
 
-lm_eval \
-  --model vllm \
-  --model_args pretrained="RedHatAI/DeepSeek-R1-Distill-Qwen-32B-NVFP4",dtype=auto,max_model_len=4096,tensor_parallel_size=2,enable_chunked_prefill=True,enforce_eager=True\
-  --apply_chat_template \
-  --fewshot_as_multiturn \
-  --tasks humaneval_64_instruct \
-  --batch_size auto
 ```
 </details>