Update README.md

README.md

@@ -135,96 +135,133 @@ This version of the lm-evaluation-harness includes versions of MMLU, ARC-Challen
 
 ### Accuracy
 
-#### Open LLM Leaderboard evaluation scores
 <table>
   <tr>
+   <td><strong>Category</strong>
+   </td>
    <td><strong>Benchmark</strong>
    </td>
    <td><strong>Meta-Llama-3.1-8B-Instruct </strong>
    </td>
-   <td><strong>Meta-Llama-3.1-8B-Instruct-quantized.w8a16 (this model)</strong>
+   <td><strong>Meta-Llama-3.1-8B-Instruct-quantized.w8a8 (this model)</strong>
    </td>
    <td><strong>Recovery</strong>
    </td>
   </tr>
   <tr>
-   <td>MMLU (5-shot)
+   <td rowspan="5" ><strong>OpenLLM v1</strong>
+   </td>
+  </tr>
+  <tr>
+   <td>Hellaswag (10-shot)
+   </td>
+   <td>80.47
+   </td>
+   <td>80.48
+   </td>
+   <td>100.0%
+   </td>
+  </tr>
+  <tr>
+   <td>Winogrande (5-shot)
+   </td>
+   <td>78.06
+   </td>
+   <td>77.51
+   </td>
+   <td>99.3%
+   </td>
+  </tr>
+  <tr>
+  <td>TruthfulQA (0-shot, mc2)
    </td>
-   <td>68.32
+   <td>54.48
    </td>
-   <td>68.26
+   <td>54.41
    </td>
    <td>99.9%
    </td>
   </tr>
   <tr>
-   <td>MMLU (CoT, 0-shot)
+   <td><strong>Average</strong>
    </td>
-   <td>72.83
+   <td><strong>74.05</strong>
    </td>
-   <td>72.44
+   <td><strong>74.12</strong>
    </td>
-   <td>99.5%
+   <td><strong>100.1%</strong>
    </td>
   </tr>
   <tr>
-   <td>ARC Challenge (0-shot)
+   <td rowspan="9" ><strong>Multilingual</strong>
    </td>
-   <td>81.40
+   <td>Portuguese MMLU (5-shot)
    </td>
-   <td>81.40
+   <td>59.96
    </td>
-   <td>100.0%
+   <td>59.79
+   </td>
+   <td>99.8%
    </td>
   </tr>
   <tr>
-   <td>GSM-8K (CoT, 8-shot, strict-match)
+   <td>Spanish MMLU (5-shot)
    </td>
-   <td>82.79
+   <td>60.25
    </td>
-   <td>84.31
+   <td>59.92
    </td>
-   <td>101.8%
+   <td>99.4%
    </td>
   </tr>
   <tr>
-   <td>Hellaswag (10-shot)
+   <td>Italian MMLU (5-shot)
    </td>
-   <td>80.47
+   <td>59.23
    </td>
-   <td>80.48
+   <td>59.25
    </td>
    <td>100.0%
    </td>
   </tr>
   <tr>
-   <td>Winogrande (5-shot)
+   <td>German MMLU (5-shot)
    </td>
-   <td>78.06
+   <td>58.63
    </td>
-   <td>77.51
+   <td>58.31
    </td>
-   <td>99.3%
+   <td>99.5%
    </td>
   </tr>
   <tr>
-  <td>TruthfulQA (0-shot, mc2)
+   <td>French MMLU (5-shot)
    </td>
-   <td>54.48
+   <td>59.65
    </td>
-   <td>54.41
+   <td>59.57
    </td>
    <td>99.9%
    </td>
   </tr>
   <tr>
-   <td><strong>Average</strong>
+   <td>Hindi MMLU (5-shot)
    </td>
-   <td><strong>74.05</strong>
+   <td>50.10
    </td>
-   <td><strong>74.12</strong>
+   <td>49.97
    </td>
-   <td><strong>100.1%</strong>
+   <td>99.7%
+   </td>
+  </tr>
+  <tr>
+   <td>Thai MMLU (5-shot)
+   </td>
+   <td>49.12
+   </td>
+   <td>49.09
+   </td>
+   <td>99.9%
    </td>
   </tr>
 </table>
@@ -307,4 +344,88 @@ lm_eval \
   --tasks truthfulqa \
   --num_fewshot 0 \
   --batch_size auto
+```
+
+#### MMLU Portuguese
+```
+lm_eval \
+  --model vllm \
+  --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",dtype=auto,max_model_len=3850,max_gen_toks=10,tensor_parallel_size=1 \
+  --tasks mmlu_pt_llama_3.1_instruct \
+  --fewshot_as_multiturn \
+  --apply_chat_template \
+  --num_fewshot 5 \
+  --batch_size auto
+```
+
+#### MMLU Spanish
+```
+lm_eval \
+  --model vllm \
+  --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",dtype=auto,max_model_len=3850,max_gen_toks=10,tensor_parallel_size=1 \
+  --tasks mmlu_es_llama_3.1_instruct \
+  --fewshot_as_multiturn \
+  --apply_chat_template \
+  --num_fewshot 5 \
+  --batch_size auto
+```
+
+#### MMLU Italian
+```
+lm_eval \
+  --model vllm \
+  --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",dtype=auto,max_model_len=3850,max_gen_toks=10,tensor_parallel_size=1 \
+  --tasks mmlu_it_llama_3.1_instruct \
+  --fewshot_as_multiturn \
+  --apply_chat_template \
+  --num_fewshot 5 \
+  --batch_size auto
+```
+
+#### MMLU German
+```
+lm_eval \
+  --model vllm \
+  --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",dtype=auto,max_model_len=3850,max_gen_toks=10,tensor_parallel_size=1 \
+  --tasks mmlu_de_llama_3.1_instruct \
+  --fewshot_as_multiturn \
+  --apply_chat_template \
+  --num_fewshot 5 \
+  --batch_size auto
+```
+
+#### MMLU French
+```
+lm_eval \
+  --model vllm \
+  --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",dtype=auto,max_model_len=3850,max_gen_toks=10,tensor_parallel_size=1 \
+  --tasks mmlu_fr_llama_3.1_instruct \
+  --fewshot_as_multiturn \
+  --apply_chat_template \
+  --num_fewshot 5 \
+  --batch_size auto
+```
+
+#### MMLU Hindi
+```
+lm_eval \
+  --model vllm \
+  --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",dtype=auto,max_model_len=3850,max_gen_toks=10,tensor_parallel_size=1 \
+  --tasks mmlu_hi_llama_3.1_instruct \
+  --fewshot_as_multiturn \
+  --apply_chat_template \
+  --num_fewshot 5 \
+  --batch_size auto
+```
+
+#### MMLU Thai
+```
+lm_eval \
+  --model vllm \
+  --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",dtype=auto,max_model_len=3850,max_gen_toks=10,tensor_parallel_size=1 \
+  --tasks mmlu_th_llama_3.1_instruct \
+  --fewshot_as_multiturn \
+  --apply_chat_template \
+  --num_fewshot 5 \
+  --batch_size auto
 ```