Add files using upload-large-folder tool

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/_direct_kmmlu_yaml

@@ -0,0 +1,27 @@
+tag:
+    - kmmlu
+    - kmmlu_direct
+dataset_path: HAERAE-HUB/KMMLU
+output_type: generate_until
+test_split: test
+fewshot_split: dev
+doc_to_text: "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답："
+doc_to_target: "{{['A', 'B', 'C', 'D'][answer-1]}}"
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+    regexes_to_ignore:
+          - " "
+generation_kwargs:
+  until:
+    - "Q:"
+    - "\n\n"
+    - "</s>"
+    - "."
+  do_sample: false
+  temperature: 0.0
+metadata:
+  version: 2.0

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_accounting.yaml

@@ -0,0 +1,3 @@
+dataset_name: Accounting
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_accounting

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_agricultural_sciences.yaml

@@ -0,0 +1,3 @@
+dataset_name: Agricultural-Sciences
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_agricultural_sciences

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_aviation_engineering_and_maintenance.yaml

@@ -0,0 +1,3 @@
+dataset_name: Aviation-Engineering-and-Maintenance
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_aviation_engineering_and_maintenance

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_biology.yaml

@@ -0,0 +1,3 @@
+dataset_name: Biology
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_biology

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_chemistry.yaml

@@ -0,0 +1,3 @@
+dataset_name: Chemistry
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_chemistry

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_civil_engineering.yaml

@@ -0,0 +1,3 @@
+dataset_name: Civil-Engineering
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_civil_engineering

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_computer_science.yaml

@@ -0,0 +1,3 @@
+dataset_name: Computer-Science
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_computer_science

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_construction.yaml

@@ -0,0 +1,3 @@
+dataset_name: Construction
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_construction

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_criminal_law.yaml

@@ -0,0 +1,3 @@
+dataset_name: Criminal-Law
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_criminal_law

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_ecology.yaml

@@ -0,0 +1,3 @@
+dataset_name: Ecology
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_ecology

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_economics.yaml

@@ -0,0 +1,3 @@
+dataset_name: Economics
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_economics

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_education.yaml

@@ -0,0 +1,3 @@
+dataset_name: Education
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_education

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_electrical_engineering.yaml

@@ -0,0 +1,3 @@
+dataset_name: Electrical-Engineering
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_electrical_engineering

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_electronics_engineering.yaml

@@ -0,0 +1,3 @@
+dataset_name: Electronics-Engineering
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_electronics_engineering

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_energy_management.yaml

@@ -0,0 +1,3 @@
+dataset_name: Energy-Management
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_energy_management

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_environmental_science.yaml

@@ -0,0 +1,3 @@
+dataset_name: Environmental-Science
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_environmental_science

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_fashion.yaml

@@ -0,0 +1,3 @@
+dataset_name: Fashion
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_fashion

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_food_processing.yaml

@@ -0,0 +1,3 @@
+dataset_name: Food-Processing
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_food_processing

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_gas_technology_and_engineering.yaml

@@ -0,0 +1,3 @@
+dataset_name: Gas-Technology-and-Engineering
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_gas_technology_and_engineering

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_geomatics.yaml

@@ -0,0 +1,3 @@
+dataset_name: Geomatics
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_geomatics

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_health.yaml

@@ -0,0 +1,3 @@
+dataset_name: Health
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_health

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_industrial_engineer.yaml

@@ -0,0 +1,3 @@
+dataset_name: Industrial-Engineer
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_industrial_engineer

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_information_technology.yaml

@@ -0,0 +1,3 @@
+dataset_name: Information-Technology
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_information_technology

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_interior_architecture_and_design.yaml

@@ -0,0 +1,3 @@
+dataset_name: Interior-Architecture-and-Design
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_interior_architecture_and_design

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_korean_history.yaml

@@ -0,0 +1,3 @@
+dataset_name: Korean-History
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_korean_history

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_law.yaml

@@ -0,0 +1,3 @@
+dataset_name: Law
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_law

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_machine_design_and_manufacturing.yaml

@@ -0,0 +1,3 @@
+dataset_name: Machine-Design-and-Manufacturing
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_machine_design_and_manufacturing

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_management.yaml

@@ -0,0 +1,3 @@
+dataset_name: Management
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_management

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_maritime_engineering.yaml

@@ -0,0 +1,3 @@
+dataset_name: Maritime-Engineering
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_maritime_engineering

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_marketing.yaml

@@ -0,0 +1,3 @@
+dataset_name: Marketing
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_marketing

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_materials_engineering.yaml

@@ -0,0 +1,3 @@
+dataset_name: Materials-Engineering
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_materials_engineering

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_math.yaml

@@ -0,0 +1,3 @@
+dataset_name: Math
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_math

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_mechanical_engineering.yaml

@@ -0,0 +1,3 @@
+dataset_name: Mechanical-Engineering
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_mechanical_engineering

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_nondestructive_testing.yaml

@@ -0,0 +1,3 @@
+dataset_name: Nondestructive-Testing
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_nondestructive_testing

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_patent.yaml

@@ -0,0 +1,3 @@
+dataset_name: Patent
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_patent

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_political_science_and_sociology.yaml

@@ -0,0 +1,3 @@
+dataset_name: Political-Science-and-Sociology
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_political_science_and_sociology

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_psychology.yaml

@@ -0,0 +1,3 @@
+dataset_name: Psychology
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_psychology

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_public_safety.yaml

@@ -0,0 +1,3 @@
+dataset_name: Public-Safety
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_public_safety

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_railway_and_automotive_engineering.yaml

@@ -0,0 +1,3 @@
+dataset_name: Railway-and-Automotive-Engineering
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_railway_and_automotive_engineering

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_real_estate.yaml

@@ -0,0 +1,3 @@
+dataset_name: Real-Estate
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_real_estate

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_refrigerating_machinery.yaml

@@ -0,0 +1,3 @@
+dataset_name: Refrigerating-Machinery
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_refrigerating_machinery

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_social_welfare.yaml

@@ -0,0 +1,3 @@
+dataset_name: Social-Welfare
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_social_welfare

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_taxation.yaml

@@ -0,0 +1,3 @@
+dataset_name: Taxation
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_taxation

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct/kmmlu_direct_telecommunications_and_wireless_technology.yaml

@@ -0,0 +1,3 @@
+dataset_name: Telecommunications-and-Wireless-Technology
+include: _direct_kmmlu_yaml
+task: kmmlu_direct_telecommunications_and_wireless_technology

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct_hard/_direct_hard_kmmlu_yaml

@@ -0,0 +1,27 @@
+tag:
+    - kmmlu
+    - kmmlu_hard_direct
+dataset_path: HAERAE-HUB/KMMLU-HARD
+output_type: generate_until
+test_split: test
+fewshot_split: dev
+doc_to_text: "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n정답："
+doc_to_target: "{{['A', 'B', 'C', 'D'][answer-1]}}"
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+    regexes_to_ignore:
+          - " "
+generation_kwargs:
+  until:
+    - "Q:"
+    - "\n\n"
+    - "</s>"
+    - "."
+  do_sample: false
+  temperature: 0.0
+metadata:
+  version: 2.0

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_aviation_engineering_and_maintenance.yaml

@@ -0,0 +1,3 @@
+dataset_name: aviation_engineering_and_maintenance
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_aviation_engineering_and_maintenance

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_biology.yaml

@@ -0,0 +1,3 @@
+dataset_name: biology
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_biology

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_chemistry.yaml

@@ -0,0 +1,3 @@
+dataset_name: chemistry
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_chemistry

scripts/yans/lm-evaluation-harness/lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_civil_engineering.yaml

@@ -0,0 +1,3 @@
+dataset_name: civil_engineering
+include: _direct_hard_kmmlu_yaml
+task: kmmlu_hard_direct_civil_engineering