Update README.md

README.md

@@ -150,9 +150,10 @@ Detailed model outputs are available as HuggingFace datasets for [Arena-Hard](ht
 
 ### Accuracy
 
-#### Open LLM Leaderboard evaluation scores
 <table>
   <tr>
+   <td><strong>Category</strong>
+   </td>
    <td><strong>Benchmark</strong>
    </td>
    <td><strong>Meta-Llama-3.1-8B-Instruct </strong>
@@ -163,7 +164,9 @@ Detailed model outputs are available as HuggingFace datasets for [Arena-Hard](ht
    </td>
   </tr>
   <tr>
-   <td><strong>Arena Hard</strong>
+   <td rowspan="1" ><strong>LLM as a judge</strong>
+   </td>    
+   <td>Arena Hard
    </td>
    <td>25.8 (25.1 / 26.5)
    </td>
@@ -173,10 +176,8 @@ Detailed model outputs are available as HuggingFace datasets for [Arena-Hard](ht
    </td>
   </tr>
   <tr>
-   <td><strong>OpenLLM v1</strong>
+   <td rowspan="8" ><strong>OpenLLM v1</strong>
    </td>
-  </tr>
-  <tr>
    <td>MMLU (5-shot)
    </td>
    <td>68.3
@@ -257,10 +258,8 @@ Detailed model outputs are available as HuggingFace datasets for [Arena-Hard](ht
    </td>
   </tr>
   <tr>
-   <td><strong>OpenLLM v2</strong>
+   <td rowspan="7" ><strong>OpenLLM v2</strong>
    </td>
-  </tr>
-  <tr>
    <td>MMLU-Pro (5-shot)
    </td>
    <td>30.8
@@ -291,7 +290,7 @@ Detailed model outputs are available as HuggingFace datasets for [Arena-Hard](ht
    </td>
   </tr>
   <tr>
-   <td>Math-|v|-5 (4-shot)
+   <td>Math-lvl-5 (4-shot)
    </td>
    <td>15.7
    </td>
@@ -331,10 +330,8 @@ Detailed model outputs are available as HuggingFace datasets for [Arena-Hard](ht
    </td>
   </tr>
   <tr>
-   <td><strong>Coding</strong>
+   <td rowspan="2" ><strong>Coding</strong>
    </td>
-  </tr>
-  <tr>
    <td>HumanEval pass@1
    </td>
    <td>67.3
@@ -354,8 +351,81 @@ Detailed model outputs are available as HuggingFace datasets for [Arena-Hard](ht
    <td>98.8%
    </td>
   </tr>
+  <tr>
+   <td rowspan="9" ><strong>Multilingual</strong>
+   </td>
+   <td>Portuguese MMLU (5-shot)
+   </td>
+   <td>59.96
+   </td>
+   <td>59.36
+   </td>
+   <td>99.0%
+   </td>
+  </tr>
+  <tr>
+   <td>Spanish MMLU (5-shot)
+   </td>
+   <td>60.25
+   </td>
+   <td>59.77
+   </td>
+   <td>99.2%
+   </td>
+  </tr>
+  <tr>
+   <td>Italian MMLU (5-shot)
+   </td>
+   <td>59.23
+   </td>
+   <td>58.61
+   </td>
+   <td>99.0%
+   </td>
+  </tr>
+  <tr>
+   <td>German MMLU (5-shot)
+   </td>
+   <td>58.63
+   </td>
+   <td>58.23
+   </td>
+   <td>99.3%
+   </td>
+  </tr>
+  <tr>
+   <td>French MMLU (5-shot)
+   </td>
+   <td>59.65
+   </td>
+   <td>58.70
+   </td>
+   <td>98.4%
+   </td>
+  </tr>
+  <tr>
+   <td>Hindi MMLU (5-shot)
+   </td>
+   <td>50.10
+   </td>
+   <td>49.33
+   </td>
+   <td>98.5%
+   </td>
+  </tr>
+  <tr>
+   <td>Thai MMLU (5-shot)
+   </td>
+   <td>49.12
+   </td>
+   <td>48.09
+   </td>
+   <td>97.9%
+   </td>
+  </tr>
 </table>
 
+
 ### Reproduction
 
 The results were obtained using the following commands:
@@ -447,6 +517,90 @@ lm_eval \
   --batch_size auto
 ```
 
+#### MMLU Portuguese
+```
+lm_eval \
+  --model vllm \
+  --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",dtype=auto,max_model_len=3850,max_gen_toks=10,tensor_parallel_size=1 \
+  --tasks mmlu_pt_llama_3.1_instruct \
+  --fewshot_as_multiturn \
+  --apply_chat_template \
+  --num_fewshot 5 \
+  --batch_size auto
+```
+
+#### MMLU Spanish
+```
+lm_eval \
+  --model vllm \
+  --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",dtype=auto,max_model_len=3850,max_gen_toks=10,tensor_parallel_size=1 \
+  --tasks mmlu_es_llama_3.1_instruct \
+  --fewshot_as_multiturn \
+  --apply_chat_template \
+  --num_fewshot 5 \
+  --batch_size auto
+```
+
+#### MMLU Italian
+```
+lm_eval \
+  --model vllm \
+  --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",dtype=auto,max_model_len=3850,max_gen_toks=10,tensor_parallel_size=1 \
+  --tasks mmlu_it_llama_3.1_instruct \
+  --fewshot_as_multiturn \
+  --apply_chat_template \
+  --num_fewshot 5 \
+  --batch_size auto
+```
+
+#### MMLU German
+```
+lm_eval \
+  --model vllm \
+  --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",dtype=auto,max_model_len=3850,max_gen_toks=10,tensor_parallel_size=1 \
+  --tasks mmlu_de_llama_3.1_instruct \
+  --fewshot_as_multiturn \
+  --apply_chat_template \
+  --num_fewshot 5 \
+  --batch_size auto
+```
+
+#### MMLU French
+```
+lm_eval \
+  --model vllm \
+  --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",dtype=auto,max_model_len=3850,max_gen_toks=10,tensor_parallel_size=1 \
+  --tasks mmlu_fr_llama_3.1_instruct \
+  --fewshot_as_multiturn \
+  --apply_chat_template \
+  --num_fewshot 5 \
+  --batch_size auto
+```
+
+#### MMLU Hindi
+```
+lm_eval \
+  --model vllm \
+  --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",dtype=auto,max_model_len=3850,max_gen_toks=10,tensor_parallel_size=1 \
+  --tasks mmlu_hi_llama_3.1_instruct \
+  --fewshot_as_multiturn \
+  --apply_chat_template \
+  --num_fewshot 5 \
+  --batch_size auto
+```
+
+#### MMLU Thai
+```
+lm_eval \
+  --model vllm \
+  --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",dtype=auto,max_model_len=3850,max_gen_toks=10,tensor_parallel_size=1 \
+  --tasks mmlu_th_llama_3.1_instruct \
+  --fewshot_as_multiturn \
+  --apply_chat_template \
+  --num_fewshot 5 \
+  --batch_size auto
+```
+
 #### HumanEval and HumanEval+
 ##### Generation
 ```