Update repeated evaluation results and reproduction commands

#2
by jiaxwang - opened
Files changed (1) hide show
  1. README.md +11 -9
README.md CHANGED
@@ -98,13 +98,13 @@ The model was evaluated using vLLM and EvalScope.
98
  | **Benchmark** | **MiniMax/MiniMax-M3.1-Flash** | **amd/MiniMax-M3.1-Flash-Quark-MXFP4 (this model)** | **Recovery** |
99
  | --- | --- | --- | --- |
100
  | GSM8K (4-shot) | 97.20 | 96.90 | 99.69% |
101
- | GPQA Diamond (mean@8) | 86.36 | 87.37 | 101.17% |
102
  | AIME26 (mean@16) | 93.54 | 93.75 | 100.22% |
103
  | MMMU Pro, standard (mean@1) | 79.19 | 79.13 | 99.92% |
104
  | MMMU Pro, vision (mean@1) | 76.99 | 77.57 | 100.75% |
105
  | SciCode, subproblem (mean@8) | 48.11 | 49.27 | 102.41% |
106
- | AA-LCR (mean@8) | 73.13 | 74.00 | 101.19% |
107
- | IFBench (mean@2) | 50.33 | 50.83 | 100.99% |
108
 
109
  Note: Results may vary from run to run.
110
 
@@ -138,7 +138,7 @@ vllm serve amd/MiniMax-M3.1-Flash-Quark-MXFP4 \
138
  ```bash
139
  evalscope eval \
140
  --model amd/MiniMax-M3.1-Flash-Quark-MXFP4 \
141
- --api-url http://localhost:8101/v1/chat/completions \
142
  --api-key EMPTY \
143
  --datasets gsm8k \
144
  --dataset-hub huggingface \
@@ -155,6 +155,7 @@ evalscope eval \
155
  --model amd/MiniMax-M3.1-Flash-Quark-MXFP4 \
156
  --api-url http://127.0.0.1:8000/v1/chat/completions \
157
  --datasets gpqa_diamond \
 
158
  --eval-batch-size 32 \
159
  --generation-config '{"temperature":1.0,"top_p":0.95,"max_tokens":200000,"timeout":7200}' \
160
  --work-dir logs/gpqa_diamond
@@ -210,7 +211,7 @@ evalscope eval \
210
  --work-dir logs/scicode
211
  ```
212
 
213
- **AA-LCR (v1.0.0)**
214
 
215
  Judge configuration (aa-lcr-judge.json; replace `<YOUR_SUBSCRIPTION_KEY>`):
216
 
@@ -244,11 +245,11 @@ evalscope eval \
244
  --model amd/MiniMax-M3.1-Flash-Quark-MXFP4 \
245
  --api-url http://127.0.0.1:8000/v1/chat/completions \
246
  --datasets aa_lcr \
247
- --eval-batch-size 32 \
248
- --repeats 8 \
249
- --generation-config '{"temperature":1.0,"top_p":0.95,"max_tokens":200000,"timeout":7200}' \
250
  --dataset-hub huggingface \
251
- --dataset-args '{"aa_lcr":{"hf_dataset_revision":"bdae010bbce259820c0e34c1d7cce210d966fb75"}}' \
252
  --judge "$(cat aa-lcr-judge.json)" \
253
  --work-dir logs/aa_lcr
254
  ```
@@ -259,6 +260,7 @@ evalscope eval \
259
  --model amd/MiniMax-M3.1-Flash-Quark-MXFP4 \
260
  --api-url http://127.0.0.1:8000/v1/chat/completions \
261
  --datasets ifbench \
 
262
  --eval-batch-size 32 \
263
  --generation-config '{"temperature":1.0,"top_p":0.95,"max_tokens":200000,"timeout":7200}' \
264
  --dataset-args '{"ifbench":{"primary_metric":"prompt_level_loose"}}' \
 
98
  | **Benchmark** | **MiniMax/MiniMax-M3.1-Flash** | **amd/MiniMax-M3.1-Flash-Quark-MXFP4 (this model)** | **Recovery** |
99
  | --- | --- | --- | --- |
100
  | GSM8K (4-shot) | 97.20 | 96.90 | 99.69% |
101
+ | GPQA Diamond (mean@8) | 87.69 | 87.63 | 99.93% |
102
  | AIME26 (mean@16) | 93.54 | 93.75 | 100.22% |
103
  | MMMU Pro, standard (mean@1) | 79.19 | 79.13 | 99.92% |
104
  | MMMU Pro, vision (mean@1) | 76.99 | 77.57 | 100.75% |
105
  | SciCode, subproblem (mean@8) | 48.11 | 49.27 | 102.41% |
106
+ | AA-LCR (mean@16) | 76.75 | 77.00 | 100.33% |
107
+ | IFBench (mean@5) | 50.53 | 52.87 | 104.62% |
108
 
109
  Note: Results may vary from run to run.
110
 
 
138
  ```bash
139
  evalscope eval \
140
  --model amd/MiniMax-M3.1-Flash-Quark-MXFP4 \
141
+ --api-url http://localhost:8000/v1/chat/completions \
142
  --api-key EMPTY \
143
  --datasets gsm8k \
144
  --dataset-hub huggingface \
 
155
  --model amd/MiniMax-M3.1-Flash-Quark-MXFP4 \
156
  --api-url http://127.0.0.1:8000/v1/chat/completions \
157
  --datasets gpqa_diamond \
158
+ --repeats 8 \
159
  --eval-batch-size 32 \
160
  --generation-config '{"temperature":1.0,"top_p":0.95,"max_tokens":200000,"timeout":7200}' \
161
  --work-dir logs/gpqa_diamond
 
211
  --work-dir logs/scicode
212
  ```
213
 
214
+ **AA-LCR (v1.1)**
215
 
216
  Judge configuration (aa-lcr-judge.json; replace `<YOUR_SUBSCRIPTION_KEY>`):
217
 
 
245
  --model amd/MiniMax-M3.1-Flash-Quark-MXFP4 \
246
  --api-url http://127.0.0.1:8000/v1/chat/completions \
247
  --datasets aa_lcr \
248
+ --eval-batch-size 64 \
249
+ --repeats 16 \
250
+ --generation-config '{"temperature":1.0,"top_p":0.95,"max_tokens":200000,"timeout":43200}' \
251
  --dataset-hub huggingface \
252
+ --dataset-args '{"aa_lcr":{"hf_dataset_revision":"9a77ef56b717057ade24ceab4d273712a0b4f19e"}}' \
253
  --judge "$(cat aa-lcr-judge.json)" \
254
  --work-dir logs/aa_lcr
255
  ```
 
260
  --model amd/MiniMax-M3.1-Flash-Quark-MXFP4 \
261
  --api-url http://127.0.0.1:8000/v1/chat/completions \
262
  --datasets ifbench \
263
+ --repeats 5 \
264
  --eval-batch-size 32 \
265
  --generation-config '{"temperature":1.0,"top_p":0.95,"max_tokens":200000,"timeout":7200}' \
266
  --dataset-args '{"ifbench":{"primary_metric":"prompt_level_loose"}}' \