| --- |
| language: |
| - en |
| license: apache-2.0 |
| pipeline_tag: text-generation |
| tags: |
| - text-generation |
|
|
| model-index: |
| - name: Dumb-1.1-Rc-2 |
| results: |
| - task: |
| type: text-generation |
| name: Text Generation |
| dataset: |
| name: MMLU |
| type: cais/mmlu |
| config: all |
| split: test |
| metrics: |
| |
| - name: MMLU (Overall) |
| type: accuracy |
| value: 23.22 |
| - name: Humanities (Category) |
| type: accuracy |
| value: 24.31 |
| - name: Social Sciences (Category) |
| type: accuracy |
| value: 21.74 |
| - name: STEM (Category) |
| type: accuracy |
| value: 22.33 |
| - name: Other (Category) |
| type: accuracy |
| value: 23.91 |
|
|
| |
| - name: Subtask - Formal Logic |
| type: accuracy |
| value: 34.13 |
| - name: Subtask - High School European History |
| type: accuracy |
| value: 21.82 |
| - name: Subtask - High School US History |
| type: accuracy |
| value: 25.00 |
| - name: Subtask - High School World History |
| type: accuracy |
| value: 26.16 |
| - name: Subtask - International Law |
| type: accuracy |
| value: 23.97 |
| - name: Subtask - Jurisprudence |
| type: accuracy |
| value: 25.93 |
| - name: Subtask - Logical Fallacies |
| type: accuracy |
| value: 22.09 |
| - name: Subtask - Moral Disputes |
| type: accuracy |
| value: 24.86 |
| - name: Subtask - Moral Scenarios |
| type: accuracy |
| value: 23.80 |
| - name: Subtask - Philosophy |
| type: accuracy |
| value: 18.65 |
| - name: Subtask - Prehistory |
| type: accuracy |
| value: 21.30 |
| - name: Subtask - Professional Law |
| type: accuracy |
| value: 24.64 |
| - name: Subtask - World Religions |
| type: accuracy |
| value: 32.16 |
|
|
| |
| - name: Subtask - Business Ethics |
| type: accuracy |
| value: 30.00 |
| - name: Subtask - Clinical Knowledge |
| type: accuracy |
| value: 20.38 |
| - name: Subtask - College Medicine |
| type: accuracy |
| value: 21.39 |
| - name: Subtask - Global Facts |
| type: accuracy |
| value: 18.00 |
| - name: Subtask - Human Aging |
| type: accuracy |
| value: 31.39 |
| - name: Subtask - Management |
| type: accuracy |
| value: 17.48 |
| - name: Subtask - Marketing |
| type: accuracy |
| value: 29.06 |
| - name: Subtask - Medical Genetics |
| type: accuracy |
| value: 29.00 |
| - name: Subtask - Miscellaneous |
| type: accuracy |
| value: 23.75 |
| - name: Subtask - Nutrition |
| type: accuracy |
| value: 21.57 |
| - name: Subtask - Professional Accounting |
| type: accuracy |
| value: 23.40 |
| - name: Subtask - Professional Medicine |
| type: accuracy |
| value: 19.85 |
| - name: Subtask - Virology |
| type: accuracy |
| value: 28.31 |
|
|
| |
| - name: Subtask - Econometrics |
| type: accuracy |
| value: 25.44 |
| - name: Subtask - High School Geography |
| type: accuracy |
| value: 17.68 |
| - name: Subtask - High School Government and Politics |
| type: accuracy |
| value: 19.69 |
| - name: Subtask - High School Macroeconomics |
| type: accuracy |
| value: 20.51 |
| - name: Subtask - High School Microeconomics |
| type: accuracy |
| value: 20.59 |
| - name: Subtask - High School Psychology |
| type: accuracy |
| value: 19.27 |
| - name: Subtask - Human Sexuality |
| type: accuracy |
| value: 25.19 |
| - name: Subtask - Professional Psychology |
| type: accuracy |
| value: 24.84 |
| - name: Subtask - Public Relations |
| type: accuracy |
| value: 21.82 |
| - name: Subtask - Security Studies |
| type: accuracy |
| value: 19.18 |
| - name: Subtask - Sociology |
| type: accuracy |
| value: 24.38 |
| - name: Subtask - US Foreign Policy |
| type: accuracy |
| value: 28.00 |
|
|
| |
| - name: Subtask - Abstract Algebra |
| type: accuracy |
| value: 19.00 |
| - name: Subtask - Anatomy |
| type: accuracy |
| value: 19.26 |
| - name: Subtask - Astronomy |
| type: accuracy |
| value: 20.39 |
| - name: Subtask - College Biology |
| type: accuracy |
| value: 26.39 |
| - name: Subtask - College Chemistry |
| type: accuracy |
| value: 22.00 |
| - name: Subtask - College Computer Science |
| type: accuracy |
| value: 26.00 |
| - name: Subtask - College Mathematics |
| type: accuracy |
| value: 23.00 |
| - name: Subtask - College Physics |
| type: accuracy |
| value: 20.59 |
| - name: Subtask - Computer Security |
| type: accuracy |
| value: 28.00 |
| - name: Subtask - Conceptual Physics |
| type: accuracy |
| value: 26.38 |
| - name: Subtask - Electrical Engineering |
| type: accuracy |
| value: 23.45 |
| - name: Subtask - Elementary Mathematics |
| type: accuracy |
| value: 21.43 |
| - name: Subtask - High School Biology |
| type: accuracy |
| value: 17.10 |
| - name: Subtask - High School Chemistry |
| type: accuracy |
| value: 23.15 |
| - name: Subtask - High School Computer Science |
| type: accuracy |
| value: 25.00 |
| - name: Subtask - High School Mathematics |
| type: accuracy |
| value: 22.59 |
| - name: Subtask - High School Physics |
| type: accuracy |
| value: 21.85 |
| - name: Subtask - High School Statistics |
| type: accuracy |
| value: 17.59 |
| - name: Subtask - Machine Learning |
| type: accuracy |
| value: 32.14 |
|
|
| - task: |
| type: text-generation |
| name: Text Generation |
| dataset: |
| name: HellaSwag |
| type: rowanz/hellaswag |
| split: test |
| metrics: |
| - name: Accuracy (acc) |
| type: accuracy |
| value: 25.83 |
| - name: Accuracy Normalized (acc_norm) |
| type: accuracy |
| value: 25.74 |
|
|
| - task: |
| type: text-generation |
| name: Text Generation |
| dataset: |
| name: ARC-Easy |
| type: ai2_arc |
| config: ARC-Easy |
| split: test |
| metrics: |
| - name: Accuracy (acc) |
| type: accuracy |
| value: 30.98 |
| - name: Accuracy Normalized (acc_norm) |
| type: accuracy |
| value: 30.35 |
|
|
| - task: |
| type: text-generation |
| name: Text Generation |
| dataset: |
| name: SciQ |
| type: sciq |
| split: test |
| metrics: |
| - name: Accuracy (acc) |
| type: accuracy |
| value: 33.30 |
| - name: Accuracy Normalized (acc_norm) |
| type: accuracy |
| value: 32.10 |
| --- |
| |
| # Dumb-1.1 |
| It is an AI model that balances Dumb-1.1-Rc-1. |
|
|
| You can tell more detailed lies, and the details of the benchmark are written. |
|
|
|  |
|
|
|
|
| # benchmark result |
|
|
| | Benchmark | Metis-1.5 (898M/A340M) | Dumb-1.1 (18.9M) | Ant (10M) | |
| | :--- | :---: | :---: | :---: | |
| | **MMLU (acc)** | 23.60% | 23.22% | **25.43%** | |
| | **HellaSwag (acc_norm)** | **30.40%** | 25.74% | 26.72% | |
| | **ARC-Easy (acc_norm)** | **41.30%** | 30.35% | 25.42% | |
| | **PIQA (acc_norm)** | **54.70%** | 51.74% | 50.72% | |
| | **BoolQ (acc)** | 47.70% | **62.17%** | 37.82% | |
| | **SciQ (acc_norm)** | - | **32.10%** | 21.50% | |
| | **ARC-Challenge (acc_norm)** | **25.90%** | 20.73% | - | |
| | **WinoGrande (acc)** | **51.50%** | - | 49.64% | 49.64% | |
| | **OpenBookQA (acc_norm)** | **29.60%** | 25.20% | - | |
