Add files using upload-large-folder tool

.cache/dataset_size.json

@@ -0,0 +1,25 @@
+{
+    "demo_gsm8k": 64,
+    "LongBench_2wikimqa": 200,
+    "LongBench_hotpotqa": 200,
+    "LongBench_musique": 200,
+    "LongBench_multifieldqa_en": 150,
+    "LongBench_multifieldqa_zh": 200,
+    "LongBench_narrativeqa": 200,
+    "LongBench_qasper": 200,
+    "LongBench_triviaqa": 200,
+    "LongBench_gov_report": 200,
+    "LongBench_qmsum": 200,
+    "LongBench_vcsum": 200,
+    "LongBench_dureader": 200,
+    "LongBench_lcc": 500,
+    "LongBench_repobench-p": 500,
+    "LongBench_passage_retrieval_en": 200,
+    "LongBench_passage_retrieval_zh": 200,
+    "LongBench_passage_count": 200,
+    "LongBench_trec": 200,
+    "LongBench_lsht": 200,
+    "LongBench_multi_news": 200,
+    "LongBench_samsum": 200,
+    "triviaqa_wiki_1shot": 7993
+}

.codespellrc

@@ -0,0 +1,5 @@
+[codespell]
+skip = *.ipynb
+count =
+quiet-level = 3
+ignore-words-list = nd, ans, ques, rouge, softwares, wit

.eggs/README.txt

@@ -0,0 +1,6 @@
+This directory contains eggs that were downloaded by setuptools to build, test, and run plug-ins.
+
+This directory caches those eggs to prevent repeated downloads.
+
+However, it is safe to delete this directory.
+

.github/pull_request_template.md

@@ -0,0 +1,32 @@
+Thanks for your contribution and we appreciate it a lot. The following instructions would make your pull request more healthy and more easily get feedback. If you do not understand some items, don't worry, just make the pull request and seek help from maintainers.
+
+## Motivation
+
+Please describe the motivation of this PR and the goal you want to achieve through this PR.
+
+## Modification
+
+Please briefly describe what modification is made in this PR.
+
+## BC-breaking (Optional)
+
+Does the modification introduce changes that break the backward compatibility of the downstream repositories?
+If so, please describe how it breaks the compatibility and how the downstream projects should modify their code to keep compatibility with this PR.
+
+## Use cases (Optional)
+
+If this PR introduces a new feature, it is better to list some use cases here and update the documentation.
+
+## Checklist
+
+**Before PR**:
+
+- [ ] Pre-commit or other linting tools are used to fix the potential lint issues.
+- [ ] Bug fixes are fully covered by unit tests, the case that causes the bug should be added in the unit tests.
+- [ ] The modification is covered by complete unit tests. If not, please add more unit test to ensure the correctness.
+- [ ] The documentation has been modified accordingly, like docstring or example tutorials.
+
+**After PR**:
+
+- [ ] If the modification has potential influence on downstream or other related projects, this PR should be tested with those projects.
+- [ ] CLA has been signed and all committers have signed the CLA in this PR.

.gitignore

@@ -0,0 +1,135 @@
+.DS_Store
+output_*/
+outputs/
+scripts/
+icl_inference_output/
+.vscode/
+tmp/
+configs/eval_subjective_alignbench_test.py
+configs/openai_key.py
+configs/secrets.py
+configs/datasets/log.json
+configs/eval_debug*.py
+configs/viz_*.py
+configs/**/*_bkup.py
+opencompass/**/*_bkup.py
+data
+work_dirs
+outputs
+models/*
+configs/internal/
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+*.ipynb
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+.idea
+
+# Auto generate documentation
+docs/en/_build/
+docs/zh_cn/_build/
+
+# .zip
+*.zip
+
+# sft config ignore list
+configs/sft_cfg/*B_*
+configs/sft_cfg/1B/*
+configs/sft_cfg/7B/*
+configs/sft_cfg/20B/*
+configs/sft_cfg/60B/*
+configs/sft_cfg/100B/*
+
+configs/cky/
+configs/_internal_legacy*
+# in case llama clone in the opencompass
+llama/
+
+# in case ilagent clone in the opencompass
+ilagent/
+
+# ignore the config file for criticbench evaluation
+configs/sft_cfg/criticbench_eval/*
+
+# path of turbomind's model after runing `lmdeploy.serve.turbomind.deploy`
+turbomind/
+
+# cibench output
+*.db
+*.pth
+*.pt
+*.onnx
+*.gz
+*.gz.*
+*.png
+*.txt
+*.jpg
+*.json
+*.jsonl
+*.csv
+*.npy
+*.c
+
+# aliyun
+core.*

.owners.yml

@@ -0,0 +1,14 @@
+assign:
+  issues: enabled
+  pull_requests: disabled
+  strategy:
+    # random
+    daily-shift-based
+  scedule:
+    '*/1 * * * *'
+  assignees:
+    - bittersweet1999
+    - liushz
+    - MaiziXiao
+    - acylam
+    - tonysy

.pre-commit-config-zh-cn.yaml

@@ -0,0 +1,123 @@
+exclude: |
+    (?x)^(
+      tests/data/|
+      tests/dataset/|
+      opencompass/models/internal/|
+      opencompass/utils/internal/|
+      opencompass/openicl/icl_evaluator/hf_metrics/|
+      opencompass/datasets/lawbench/utils|
+      opencompass/datasets/lawbench/evaluation_functions/|
+      opencompass/datasets/medbench/|
+      opencompass/datasets/teval/|
+      opencompass/datasets/NPHardEval/|
+      opencompass/datasets/TheoremQA|
+      opencompass/datasets/subjective/mtbench101.py|
+      docs/zh_cn/advanced_guides/compassbench_intro.md |
+      docs/zh_cn/advanced_guides/compassbench_v2_0.md |
+      opencompass/utils/datasets.py |
+      opencompass/utils/datasets_info.py
+    )
+repos:
+  - repo: https://gitee.com/openmmlab/mirrors-flake8
+    rev: 5.0.4
+    hooks:
+      - id: flake8
+        exclude: |
+            (?x)^(
+                opencompass/configs/|
+                examples/
+            )
+  - repo: https://gitee.com/openmmlab/mirrors-isort
+    rev: 5.11.5
+    hooks:
+      - id: isort
+        exclude: |
+            (?x)^(
+                opencompass/configs/|
+                examples/
+            )
+  - repo: https://gitee.com/openmmlab/mirrors-yapf
+    rev: v0.32.0
+    hooks:
+      - id: yapf
+        exclude: |
+            (?x)^(
+                opencompass/configs/|
+                examples/
+            )
+  - repo: https://gitee.com/openmmlab/mirrors-codespell
+    rev: v2.2.1
+    hooks:
+      - id: codespell
+        exclude: |
+            (?x)^(
+                .*\.jsonl|
+                .*\.md.template|
+                opencompass/configs/ |
+                examples/
+            )
+  - repo: https://gitee.com/openmmlab/mirrors-pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: trailing-whitespace
+        exclude: |
+            (?x)^(
+              dicts/|
+              projects/.*?/dicts/|
+            )
+      - id: check-yaml
+      - id: end-of-file-fixer
+        exclude: |
+            (?x)^(
+              dicts/|
+              projects/.*?/dicts/|
+            )
+      - id: requirements-txt-fixer
+      - id: double-quote-string-fixer
+      - id: check-merge-conflict
+      - id: fix-encoding-pragma
+        args: ["--remove"]
+      - id: mixed-line-ending
+        args: ["--fix=lf"]
+  - repo: https://gitee.com/openmmlab/mirrors-mdformat
+    rev: 0.7.9
+    hooks:
+      - id: mdformat
+        args: ["--number", "--table-width", "200"]
+        additional_dependencies:
+          - mdformat-openmmlab
+          - mdformat_frontmatter
+          - linkify-it-py
+        exclude: configs/
+  - repo: https://gitee.com/openmmlab/mirrors-docformatter
+    rev: v1.3.1
+    hooks:
+      - id: docformatter
+        args: ["--in-place", "--wrap-descriptions", "79"]
+  - repo: local
+    hooks:
+    -   id: update-dataset-suffix
+        name: dataset suffix updater
+        entry: ./tools/update_dataset_suffix.py
+        language: script
+        pass_filenames: true
+        require_serial: true
+        files: ^opencompass/configs/datasets
+  - repo: local
+    hooks:
+    -   id: update-dataset-suffix-pacakge
+        name: dataset suffix updater(package)
+        entry: ./tools/update_dataset_suffix.py
+        language: script
+        pass_filenames: false
+        # require_serial: true
+        # files: ^opencompass/configs/datasets
+        args:
+          - --root_folder
+          - opencompass/configs/datasets
+  # - repo: https://github.com/open-mmlab/pre-commit-hooks
+  #   rev: v0.2.0  # Use the ref you want to point at
+  #   hooks:
+  #     - id: check-algo-readme
+      # - id: check-copyright
+      #   args: ["mmocr", "tests", "tools"]  # these directories will be checked

.pre-commit-config.yaml

@@ -0,0 +1,124 @@
+exclude: |
+    (?x)^(
+      tests/data/|
+      tests/dataset/|
+      opencompass/models/internal/|
+      opencompass/utils/internal/|
+      opencompass/openicl/icl_evaluator/hf_metrics/|
+      opencompass/datasets/lawbench/utils|
+      opencompass/datasets/lawbench/evaluation_functions/|
+      opencompass/datasets/medbench/|
+      opencompass/datasets/matbench/|
+      opencompass/datasets/teval/|
+      opencompass/datasets/NPHardEval/|
+      opencompass/datasets/TheoremQA|
+      opencompass/datasets/subjective/mtbench101.py|
+      docs/zh_cn/advanced_guides/compassbench_intro.md |
+      docs/zh_cn/advanced_guides/compassbench_v2_0.md |
+      opencompass/utils/datasets.py |
+      opencompass/utils/datasets_info.py
+    )
+repos:
+  - repo: https://github.com/PyCQA/flake8
+    rev: 5.0.4
+    hooks:
+      - id: flake8
+        exclude: |
+            (?x)^(
+                opencompass/configs/|
+                examples/
+            )
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.11.5
+    hooks:
+      - id: isort
+        exclude: |
+            (?x)^(
+                opencompass/configs/|
+                examples/
+            )
+  - repo: https://github.com/pre-commit/mirrors-yapf
+    rev: v0.32.0
+    hooks:
+      - id: yapf
+        exclude: |
+            (?x)^(
+                opencompass/configs/|
+                examples/
+            )
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.1
+    hooks:
+      - id: codespell
+        exclude: |
+            (?x)^(
+                .*\.jsonl|
+                .*\.md.template|
+                opencompass/configs/ |
+                examples/
+            )
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+        exclude: |
+            (?x)^(
+              dicts/|
+              projects/.*?/dicts/|
+            )
+      - id: check-yaml
+      - id: end-of-file-fixer
+        exclude: |
+            (?x)^(
+              dicts/|
+              projects/.*?/dicts/|
+            )
+      - id: requirements-txt-fixer
+      - id: double-quote-string-fixer
+      - id: check-merge-conflict
+      - id: fix-encoding-pragma
+        args: ["--remove"]
+      - id: mixed-line-ending
+        args: ["--fix=lf"]
+  - repo: https://github.com/executablebooks/mdformat
+    rev: 0.7.9
+    hooks:
+      - id: mdformat
+        args: ["--number", "--table-width", "200"]
+        additional_dependencies:
+          - mdformat-openmmlab
+          - mdformat_frontmatter
+          - linkify-it-py
+        exclude: configs/
+  # - repo: https://github.com/myint/docformatter
+  #   rev: v1.3.1
+  #   hooks:
+  #     - id: docformatter
+  #       args: ["--in-place", "--wrap-descriptions", "79"]
+  - repo: local
+    hooks:
+    -   id: update-dataset-suffix
+        name: dataset suffix updater
+        entry: ./tools/update_dataset_suffix.py
+        language: script
+        pass_filenames: true
+        require_serial: true
+        files: ^opencompass/configs/datasets
+  - repo: local
+    hooks:
+    -   id: update-dataset-suffix-pacakge
+        name: dataset suffix updater(package)
+        entry: ./tools/update_dataset_suffix.py
+        language: script
+        pass_filenames: false
+        # require_serial: true
+        # files: ^opencompass/configs/datasets
+        args:
+          - --root_folder
+          - opencompass/configs/datasets
+  # - repo: https://github.com/open-mmlab/pre-commit-hooks
+  #   rev: v0.2.0  # Use the ref you want to point at
+  #   hooks:
+  #     - id: check-algo-readme
+      # - id: check-copyright
+      #   args: ["mmocr", "tests", "tools"]  # these directories will be checked

LICENSE

@@ -0,0 +1,203 @@
+Copyright 2020 OpenCompass Authors. All rights reserved.
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2020 OpenCompass Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

MANIFEST.in

@@ -0,0 +1,3 @@
+recursive-include opencompass/configs *.py *.yml *.json *.txt *.md
+recursive-include opencompass/openicl/icl_evaluator/hf_metrics *.py
+recursive-include opencompass/datasets *.py *.yml *.json *.txt *.md *.yaml

README.md

@@ -0,0 +1,439 @@
+<div align="center">
+  <img src="docs/en/_static/image/logo.svg" width="500px"/>
+  <br />
+  <br />
+
+[![][github-release-shield]][github-release-link]
+[![][github-releasedate-shield]][github-releasedate-link]
+[![][github-contributors-shield]][github-contributors-link]<br>
+[![][github-forks-shield]][github-forks-link]
+[![][github-stars-shield]][github-stars-link]
+[![][github-issues-shield]][github-issues-link]
+[![][github-license-shield]][github-license-link]
+
+<!-- [![PyPI](https://badge.fury.io/py/opencompass.svg)](https://pypi.org/project/opencompass/) -->
+
+[🌐Website](https://opencompass.org.cn/) |
+[📖CompassHub](https://hub.opencompass.org.cn/home) |
+[📊CompassRank](https://rank.opencompass.org.cn/home) |
+[📘Documentation](https://opencompass.readthedocs.io/en/latest/) |
+[🛠️Installation](https://opencompass.readthedocs.io/en/latest/get_started/installation.html) |
+[🤔Reporting Issues](https://github.com/open-compass/opencompass/issues/new/choose)
+
+English | [简体中文](README_zh-CN.md)
+
+[![][github-trending-shield]][github-trending-url]
+
+</div>
+
+<p align="center">
+    👋 join us on <a href="https://discord.gg/KKwfEbFj7U" target="_blank">Discord</a> and <a href="https://r.vansin.top/?r=opencompass" target="_blank">WeChat</a>
+</p>
+
+> \[!IMPORTANT\]
+>
+> **Star Us**, You will receive all release notifications from GitHub without any delay ~ ⭐️
+
+<details>
+  <summary><kbd>Star History</kbd></summary>
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=open-compass%2Fopencompass&theme=dark&type=Date">
+    <img width="100%" src="https://api.star-history.com/svg?repos=open-compass%2Fopencompass&type=Date">
+  </picture>
+</details>
+
+## 🧭	Welcome
+
+to **OpenCompass**!
+
+Just like a compass guides us on our journey, OpenCompass will guide you through the complex landscape of evaluating large language models. With its powerful algorithms and intuitive interface, OpenCompass makes it easy to assess the quality and effectiveness of your NLP models.
+
+🚩🚩🚩 Explore opportunities at OpenCompass! We're currently **hiring full-time researchers/engineers and interns**. If you're passionate about LLM and OpenCompass, don't hesitate to reach out to us via [email](mailto:zhangsongyang@pjlab.org.cn). We'd love to hear from you!
+
+🔥🔥🔥 We are delighted to announce that **the OpenCompass has been recommended by the Meta AI**, click [Get Started](https://ai.meta.com/llama/get-started/#validation) of Llama for more information.
+
+> **Attention**<br />
+> Breaking Change Notice: In version 0.4.0, we are consolidating all AMOTIC configuration files (previously located in ./configs/datasets, ./configs/models, and ./configs/summarizers) into the opencompass package. Users are advised to update their configuration references to reflect this structural change.
+
+## 🚀 What's New <a><img width="35" height="20" src="https://user-images.githubusercontent.com/12782558/212848161-5e783dd6-11e8-4fe0-bbba-39ffb77730be.png"></a>
+
+- **\[2025.07.26\]** OpenCompass now supports Intern-S1 related general and scientific evaluation benchmarks. Please check [Tutorial for Evaluating Intern-S1](https://opencompass.readthedocs.io/en/latest/user_guides/interns1.html) for more details! 🔥🔥🔥
+- **\[2025.04.01\]** OpenCompass now supports `CascadeEvaluator`, a flexible evaluation mechanism that allows multiple evaluators to work in sequence. This enables creating customized evaluation pipelines for complex assessment scenarios. Check out the [documentation](docs/en/advanced_guides/llm_judge.md) for more details! 🔥🔥🔥
+- **\[2025.03.11\]** We have supported evaluation for `SuperGPQA` which is a great benchmark for measuring LLM knowledge ability 🔥🔥🔥
+- **\[2025.02.28\]** We have added a tutorial for `DeepSeek-R1` series model, please check [Evaluating Reasoning Model](docs/en/user_guides/deepseek_r1.md) for more details! 🔥🔥🔥
+- **\[2025.02.15\]** We have added two powerful evaluation tools: `GenericLLMEvaluator` for LLM-as-judge evaluations and `MATHVerifyEvaluator` for mathematical reasoning assessments. Check out the documentation for [LLM Judge](docs/en/advanced_guides/llm_judge.md) and [Math Evaluation](docs/en/advanced_guides/general_math.md) for more details! 🔥🔥🔥
+- **\[2025.01.16\]** We now support the [InternLM3-8B-Instruct](https://huggingface.co/internlm/internlm3-8b-instruct) model which has enhanced performance on reasoning and knowledge-intensive tasks.
+- **\[2024.12.17\]** We have provided the evaluation script for the December [CompassAcademic](examples/eval_academic_leaderboard_202412.py), which allows users to easily reproduce the official evaluation results by configuring it.
+- **\[2024.11.14\]** OpenCompass now offers support for a sophisticated benchmark designed to evaluate complex reasoning skills — [MuSR](https://arxiv.org/pdf/2310.16049). Check out the [demo](examples/eval_musr.py) and give it a spin! 🔥🔥🔥
+- **\[2024.11.14\]** OpenCompass now supports the brand new long-context language model evaluation benchmark — [BABILong](https://arxiv.org/pdf/2406.10149). Have a look at the [demo](examples/eval_babilong.py) and give it a try! 🔥🔥🔥
+- **\[2024.10.14\]** We now support the OpenAI multilingual QA dataset [MMMLU](https://huggingface.co/datasets/openai/MMMLU). Feel free to give it a try! 🔥🔥🔥
+- **\[2024.09.19\]** We now support [Qwen2.5](https://huggingface.co/Qwen)(0.5B to 72B) with multiple backend(huggingface/vllm/lmdeploy). Feel free to give them a try! 🔥🔥🔥
+- **\[2024.09.17\]** We now support OpenAI o1(`o1-mini-2024-09-12` and `o1-preview-2024-09-12`). Feel free to give them a try! 🔥🔥🔥
+- **\[2024.09.05\]** We now support answer extraction through model post-processing to provide a more accurate representation of the model's capabilities. As part of this update, we have integrated [XFinder](https://github.com/IAAR-Shanghai/xFinder) as our first post-processing model. For more detailed information, please refer to the [documentation](opencompass/utils/postprocessors/xfinder/README.md), and give it a try! 🔥🔥🔥
+- **\[2024.08.20\]** OpenCompass now supports the [SciCode](https://github.com/scicode-bench/SciCode): A Research Coding Benchmark Curated by Scientists. 🔥🔥🔥
+- **\[2024.08.16\]** OpenCompass now supports the brand new long-context language model evaluation benchmark — [RULER](https://arxiv.org/pdf/2404.06654). RULER provides an evaluation of long-context including retrieval, multi-hop tracing, aggregation, and question answering through flexible configurations. Check out the [RULER](configs/datasets/ruler/README.md) evaluation config now! 🔥🔥🔥
+- **\[2024.08.09\]** We have released the example data and configuration for the CompassBench-202408, welcome to [CompassBench](https://opencompass.readthedocs.io/zh-cn/latest/advanced_guides/compassbench_intro.html) for more details. 🔥🔥🔥
+- **\[2024.08.01\]** We supported the [Gemma2](https://huggingface.co/collections/google/gemma-2-release-667d6600fd5220e7b967f315) models. Welcome to try! 🔥🔥🔥
+- **\[2024.07.23\]** We supported the [ModelScope](www.modelscope.cn) datasets, you can load them on demand without downloading all the data to your local disk. Welcome to try! 🔥🔥🔥
+- **\[2024.07.17\]** We are excited to announce the release of NeedleBench's [technical report](http://arxiv.org/abs/2407.11963). We invite you to visit our [support documentation](https://opencompass.readthedocs.io/en/latest/advanced_guides/needleinahaystack_eval.html) for detailed evaluation guidelines. 🔥🔥🔥
+- **\[2024.07.04\]** OpenCompass now supports InternLM2.5, which has **outstanding reasoning capability**, **1M Context window and** and **stronger tool use**, you can try the models in [OpenCompass Config](https://github.com/open-compass/opencompass/tree/main/configs/models/hf_internlm) and [InternLM](https://github.com/InternLM/InternLM) .🔥🔥🔥.
+- **\[2024.06.20\]** OpenCompass now supports one-click switching between inference acceleration backends, enhancing the efficiency of the evaluation process. In addition to the default HuggingFace inference backend, it now also supports popular backends [LMDeploy](https://github.com/InternLM/lmdeploy) and [vLLM](https://github.com/vllm-project/vllm). This feature is available via a simple command-line switch and through deployment APIs. For detailed usage, see the [documentation](docs/en/advanced_guides/accelerator_intro.md).🔥🔥🔥.
+
+> [More](docs/en/notes/news.md)
+
+## 📊 Leaderboard
+
+We provide [OpenCompass Leaderboard](https://rank.opencompass.org.cn/home) for the community to rank all public models and API models. If you would like to join the evaluation, please provide the model repository URL or a standard API interface to the email address `opencompass@pjlab.org.cn`.
+
+You can also refer to [Guide to Reproducing CompassAcademic Leaderboard Results](https://opencompass.readthedocs.io/zh-cn/latest/academic.html) to quickly reproduce the leaderboard results.
+
+<p align="right"><a href="#top">🔝Back to top</a></p>
+
+## 🛠️ Installation
+
+Below are the steps for quick installation and datasets preparation.
+
+### 💻 Environment Setup
+
+We highly recommend using conda to manage your python environment.
+
+- #### Create your virtual environment
+
+  ```bash
+  conda create --name opencompass python=3.10 -y
+  conda activate opencompass
+  ```
+
+- #### Install OpenCompass via pip
+
+  ```bash
+    pip install -U opencompass
+
+    ## Full installation (with support for more datasets)
+    # pip install "opencompass[full]"
+
+    ## Environment with model acceleration frameworks
+    ## Manage different acceleration frameworks using virtual environments
+    ## since they usually have dependency conflicts with each other.
+    # pip install "opencompass[lmdeploy]"
+    # pip install "opencompass[vllm]"
+
+    ## API evaluation (i.e. Openai, Qwen)
+    # pip install "opencompass[api]"
+  ```
+
+- #### Install OpenCompass from source
+
+  If you want to use opencompass's latest features, or develop new features, you can also build it from source
+
+  ```bash
+    git clone https://github.com/open-compass/opencompass opencompass
+    cd opencompass
+    pip install -e .
+    # pip install -e ".[full]"
+    # pip install -e ".[vllm]"
+  ```
+
+### 📂 Data Preparation
+
+You can choose one for the following method to prepare datasets.
+
+#### Offline Preparation
+
+You can download and extract the datasets with the following commands:
+
+```bash
+# Download dataset to data/ folder
+wget https://github.com/open-compass/opencompass/releases/download/0.2.2.rc1/OpenCompassData-core-20240207.zip
+unzip OpenCompassData-core-20240207.zip
+```
+
+#### Automatic Download from OpenCompass
+
+We have supported download datasets automatic from the OpenCompass storage server. You can run the evaluation with extra `--dry-run` to download these datasets.
+Currently, the supported datasets are listed in [here](https://github.com/open-compass/opencompass/blob/main/opencompass/utils/datasets_info.py#L259). More datasets will be uploaded recently.
+
+#### (Optional) Automatic Download with ModelScope
+
+Also you can use the [ModelScope](www.modelscope.cn) to load the datasets on demand.
+
+Installation:
+
+```bash
+pip install modelscope[framework]
+export DATASET_SOURCE=ModelScope
+```
+
+Then submit the evaluation task without downloading all the data to your local disk. Available datasets include:
+
+```bash
+humaneval, triviaqa, commonsenseqa, tydiqa, strategyqa, cmmlu, lambada, piqa, ceval, math, LCSTS, Xsum, winogrande, openbookqa, AGIEval, gsm8k, nq, race, siqa, mbpp, mmlu, hellaswag, ARC, BBH, xstory_cloze, summedits, GAOKAO-BENCH, OCNLI, cmnli
+```
+
+Some third-party features, like Humaneval and Llama, may require additional steps to work properly, for detailed steps please refer to the [Installation Guide](https://opencompass.readthedocs.io/en/latest/get_started/installation.html).
+
+<p align="right"><a href="#top">🔝Back to top</a></p>
+
+## 🏗️ ️Evaluation
+
+After ensuring that OpenCompass is installed correctly according to the above steps and the datasets are prepared. Now you can start your first evaluation using OpenCompass!
+
+### Your first evaluation with OpenCompass!
+
+OpenCompass support setting your configs via CLI or a python script. For simple evaluation settings we recommend using CLI, for more complex evaluation, it is suggested using the script way. You can find more example scripts under the configs folder.
+
+```bash
+# CLI
+opencompass --models hf_internlm2_5_1_8b_chat --datasets demo_gsm8k_chat_gen
+
+# Python scripts
+opencompass examples/eval_chat_demo.py
+```
+
+You can find more script examples under [examples](./examples) folder.
+
+### API evaluation
+
+OpenCompass, by its design, does not really discriminate between open-source models and API models. You can evaluate both model types in the same way or even in one settings.
+
+```bash
+export OPENAI_API_KEY="YOUR_OPEN_API_KEY"
+# CLI
+opencompass --models gpt_4o_2024_05_13 --datasets demo_gsm8k_chat_gen
+
+# Python scripts
+opencompass examples/eval_api_demo.py
+
+# You can use o1_mini_2024_09_12/o1_preview_2024_09_12  for o1 models, we set max_completion_tokens=8192 as default.
+```
+
+### Accelerated Evaluation
+
+Additionally, if you want to use an inference backend other than HuggingFace for accelerated evaluation, such as LMDeploy or vLLM, you can do so with the command below. Please ensure that you have installed the necessary packages for the chosen backend and that your model supports accelerated inference with it. For more information, see the documentation on inference acceleration backends [here](docs/en/advanced_guides/accelerator_intro.md). Below is an example using LMDeploy:
+
+```bash
+# CLI
+opencompass --models hf_internlm2_5_1_8b_chat --datasets demo_gsm8k_chat_gen -a lmdeploy
+
+# Python scripts
+opencompass examples/eval_lmdeploy_demo.py
+```
+
+### Supported Models and Datasets
+
+OpenCompass has predefined configurations for many models and datasets. You can list all available model and dataset configurations using the [tools](./docs/en/tools.md#list-configs).
+
+```bash
+# List all configurations
+python tools/list_configs.py
+# List all configurations related to llama and mmlu
+python tools/list_configs.py llama mmlu
+```
+
+#### Supported Models
+
+If the model is not on the list but supported by Huggingface AutoModel class or encapsulation of inference engine based on OpenAI interface (see [docs](https://opencompass.readthedocs.io/en/latest/advanced_guides/new_model.html) for details), you can also evaluate it with OpenCompass. You are welcome to contribute to the maintenance of the OpenCompass supported model and dataset lists.
+
+```bash
+opencompass --datasets demo_gsm8k_chat_gen --hf-type chat --hf-path internlm/internlm2_5-1_8b-chat
+```
+
+#### Supported Datasets
+
+Currently, OpenCompass have provided standard recommended configurations for datasets. Generally, config files ending with `_gen.py` or `_llm_judge_gen.py` will point to the recommended config we provide for this dataset. You can refer to [docs](https://opencompass.readthedocs.io/en/latest/dataset_statistics.html) for more details.
+
+```bash
+# Recommended Evaluation Config based on Rules
+opencompass --datasets aime2024_gen --models hf_internlm2_5_1_8b_chat
+
+# Recommended Evaluation Config based on LLM Judge
+opencompass --datasets aime2024_llmjudge_gen --models hf_internlm2_5_1_8b_chat
+```
+
+If you want to use multiple GPUs to evaluate the model in data parallel, you can use `--max-num-worker`.
+
+```bash
+CUDA_VISIBLE_DEVICES=0,1 opencompass --datasets demo_gsm8k_chat_gen --hf-type chat --hf-path internlm/internlm2_5-1_8b-chat --max-num-worker 2
+```
+
+> \[!TIP\]
+>
+> `--hf-num-gpus` is used for model parallel(huggingface format), `--max-num-worker` is used for data parallel.
+
+> \[!TIP\]
+>
+> configuration with `_ppl` is designed for base model typically.
+> configuration with `_gen` can be used for both base model and chat model.
+
+Through the command line or configuration files, OpenCompass also supports evaluating APIs or custom models, as well as more diversified evaluation strategies. Please read the [Quick Start](https://opencompass.readthedocs.io/en/latest/get_started/quick_start.html) to learn how to run an evaluation task.
+
+<p align="right"><a href="#top">🔝Back to top</a></p>
+
+## 📣 OpenCompass 2.0
+
+We are thrilled to introduce OpenCompass 2.0, an advanced suite featuring three key components: [CompassKit](https://github.com/open-compass), [CompassHub](https://hub.opencompass.org.cn/home), and [CompassRank](https://rank.opencompass.org.cn/home).
+![oc20](https://github.com/tonysy/opencompass/assets/7881589/90dbe1c0-c323-470a-991e-2b37ab5350b2)
+
+**CompassRank** has been significantly enhanced into the leaderboards that now incorporates both open-source benchmarks and proprietary benchmarks. This upgrade allows for a more comprehensive evaluation of models across the industry.
+
+**CompassHub** presents a pioneering benchmark browser interface, designed to simplify and expedite the exploration and utilization of an extensive array of benchmarks for researchers and practitioners alike. To enhance the visibility of your own benchmark within the community, we warmly invite you to contribute it to CompassHub. You may initiate the submission process by clicking [here](https://hub.opencompass.org.cn/dataset-submit).
+
+**CompassKit** is a powerful collection of evaluation toolkits specifically tailored for Large Language Models and Large Vision-language Models. It provides an extensive set of tools to assess and measure the performance of these complex models effectively. Welcome to try our toolkits for in your research and products.
+
+## ✨ Introduction
+
+![image](https://github.com/open-compass/opencompass/assets/22607038/f45fe125-4aed-4f8c-8fe8-df4efb41a8ea)
+
+OpenCompass is a one-stop platform for large model evaluation, aiming to provide a fair, open, and reproducible benchmark for large model evaluation. Its main features include:
+
+- **Comprehensive support for models and datasets**: Pre-support for 20+ HuggingFace and API models, a model evaluation scheme of 70+ datasets with about 400,000 questions, comprehensively evaluating the capabilities of the models in five dimensions.
+
+- **Efficient distributed evaluation**: One line command to implement task division and distributed evaluation, completing the full evaluation of billion-scale models in just a few hours.
+
+- **Diversified evaluation paradigms**: Support for zero-shot, few-shot, and chain-of-thought evaluations, combined with standard or dialogue-type prompt templates, to easily stimulate the maximum performance of various models.
+
+- **Modular design with high extensibility**: Want to add new models or datasets, customize an advanced task division strategy, or even support a new cluster management system? Everything about OpenCompass can be easily expanded!
+
+- **Experiment management and reporting mechanism**: Use config files to fully record each experiment, and support real-time reporting of results.
+
+## 📖 Dataset Support
+
+We have supported a statistical list of all datasets that can be used on this platform in the documentation on the OpenCompass website.
+
+You can quickly find the dataset you need from the list through sorting, filtering, and searching functions.
+
+In addition, we provide a recommended configuration for each dataset, and some datasets also support LLM Judge-based configurations.
+
+Please refer to the dataset statistics chapter of [docs](https://opencompass.readthedocs.io/en/latest/dataset_statistics.html) for details.
+
+<p align="right"><a href="#top">🔝Back to top</a></p>
+
+## 📖 Model Support
+
+<table align="center">
+  <tbody>
+    <tr align="center" valign="bottom">
+      <td>
+        <b>Open-source Models</b>
+      </td>
+      <td>
+        <b>API Models</b>
+      </td>
+      <!-- <td>
+        <b>Custom Models</b>
+      </td> -->
+    </tr>
+    <tr valign="top">
+      <td>
+
+- [Alpaca](https://github.com/tatsu-lab/stanford_alpaca)
+- [Baichuan](https://github.com/baichuan-inc)
+- [BlueLM](https://github.com/vivo-ai-lab/BlueLM)
+- [ChatGLM2](https://github.com/THUDM/ChatGLM2-6B)
+- [ChatGLM3](https://github.com/THUDM/ChatGLM3-6B)
+- [Gemma](https://huggingface.co/google/gemma-7b)
+- [InternLM](https://github.com/InternLM/InternLM)
+- [LLaMA](https://github.com/facebookresearch/llama)
+- [LLaMA3](https://github.com/meta-llama/llama3)
+- [Qwen](https://github.com/QwenLM/Qwen)
+- [TigerBot](https://github.com/TigerResearch/TigerBot)
+- [Vicuna](https://github.com/lm-sys/FastChat)
+- [WizardLM](https://github.com/nlpxucan/WizardLM)
+- [Yi](https://github.com/01-ai/Yi)
+- ……
+
+</td>
+<td>
+
+- OpenAI
+- Gemini
+- Claude
+- ZhipuAI(ChatGLM)
+- Baichuan
+- ByteDance(YunQue)
+- Huawei(PanGu)
+- 360
+- Baidu(ERNIEBot)
+- MiniMax(ABAB-Chat)
+- SenseTime(nova)
+- Xunfei(Spark)
+- ……
+
+</td>
+
+</tr>
+  </tbody>
+</table>
+
+<p align="right"><a href="#top">🔝Back to top</a></p>
+
+## 🔜 Roadmap
+
+- [x] Subjective Evaluation
+  - [x] Release CompassAreana.
+  - [x] Subjective evaluation.
+- [x] Long-context
+  - [x] Long-context evaluation with extensive datasets.
+  - [ ] Long-context leaderboard.
+- [x] Coding
+  - [ ] Coding evaluation leaderboard.
+  - [x] Non-python language evaluation service.
+- [x] Agent
+  - [ ] Support various agent frameworks.
+  - [x] Evaluation of tool use of the LLMs.
+- [x] Robustness
+  - [x] Support various attack methods.
+
+## 👷‍♂️ Contributing
+
+We appreciate all contributions to improving OpenCompass. Please refer to the [contributing guideline](https://opencompass.readthedocs.io/en/latest/notes/contribution_guide.html) for the best practice.
+
+<!-- Copy-paste in your Readme.md file -->
+
+<!-- Made with [OSS Insight](https://ossinsight.io/) -->
+
+<a href="https://github.com/open-compass/opencompass/graphs/contributors" target="_blank">
+  <table>
+    <tr>
+      <th colspan="2">
+        <br><img src="https://contrib.rocks/image?repo=open-compass/opencompass"><br><br>
+      </th>
+    </tr>
+  </table>
+</a>
+
+## 🤝 Acknowledgements
+
+Some code in this project is cited and modified from [OpenICL](https://github.com/Shark-NLP/OpenICL).
+
+Some datasets and prompt implementations are modified from [chain-of-thought-hub](https://github.com/FranxYao/chain-of-thought-hub) and [instruct-eval](https://github.com/declare-lab/instruct-eval).
+
+## 🖊️ Citation
+
+```bibtex
+@misc{2023opencompass,
+    title={OpenCompass: A Universal Evaluation Platform for Foundation Models},
+    author={OpenCompass Contributors},
+    howpublished = {\url{https://github.com/open-compass/opencompass}},
+    year={2023}
+}
+```
+
+<p align="right"><a href="#top">🔝Back to top</a></p>
+
+[github-contributors-link]: https://github.com/open-compass/opencompass/graphs/contributors
+[github-contributors-shield]: https://img.shields.io/github/contributors/open-compass/opencompass?color=c4f042&labelColor=black&style=flat-square
+[github-forks-link]: https://github.com/open-compass/opencompass/network/members
+[github-forks-shield]: https://img.shields.io/github/forks/open-compass/opencompass?color=8ae8ff&labelColor=black&style=flat-square
+[github-issues-link]: https://github.com/open-compass/opencompass/issues
+[github-issues-shield]: https://img.shields.io/github/issues/open-compass/opencompass?color=ff80eb&labelColor=black&style=flat-square
+[github-license-link]: https://github.com/open-compass/opencompass/blob/main/LICENSE
+[github-license-shield]: https://img.shields.io/github/license/open-compass/opencompass?color=white&labelColor=black&style=flat-square
+[github-release-link]: https://github.com/open-compass/opencompass/releases
+[github-release-shield]: https://img.shields.io/github/v/release/open-compass/opencompass?color=369eff&labelColor=black&logo=github&style=flat-square
+[github-releasedate-link]: https://github.com/open-compass/opencompass/releases
+[github-releasedate-shield]: https://img.shields.io/github/release-date/open-compass/opencompass?labelColor=black&style=flat-square
+[github-stars-link]: https://github.com/open-compass/opencompass/stargazers
+[github-stars-shield]: https://img.shields.io/github/stars/open-compass/opencompass?color=ffcb47&labelColor=black&style=flat-square
+[github-trending-shield]: https://trendshift.io/api/badge/repositories/6630
+[github-trending-url]: https://trendshift.io/repositories/6630

README_zh-CN.md

@@ -0,0 +1,430 @@
+<div align="center">
+  <img src="docs/zh_cn/_static/image/logo.svg" width="500px"/>
+  <br />
+  <br />
+
+[![][github-release-shield]][github-release-link]
+[![][github-releasedate-shield]][github-releasedate-link]
+[![][github-contributors-shield]][github-contributors-link]<br>
+[![][github-forks-shield]][github-forks-link]
+[![][github-stars-shield]][github-stars-link]
+[![][github-issues-shield]][github-issues-link]
+[![][github-license-shield]][github-license-link]
+
+<!-- [![PyPI](https://badge.fury.io/py/opencompass.svg)](https://pypi.org/project/opencompass/) -->
+
+[🌐官方网站](https://opencompass.org.cn/) |
+[📖数据集社区](https://hub.opencompass.org.cn/home) |
+[📊性能榜单](https://rank.opencompass.org.cn/home) |
+[📘文档教程](https://opencompass.readthedocs.io/zh_CN/latest/index.html) |
+[🛠️安装](https://opencompass.readthedocs.io/zh_CN/latest/get_started/installation.html) |
+[🤔报告问题](https://github.com/open-compass/opencompass/issues/new/choose)
+
+[English](/README.md) | 简体中文
+
+[![][github-trending-shield]][github-trending-url]
+
+</div>
+
+<p align="center">
+    👋 加入我们的 <a href="https://discord.gg/KKwfEbFj7U" target="_blank">Discord</a> 和 <a href="https://r.vansin.top/?r=opencompass" target="_blank">微信社区</a>
+</p>
+
+> \[!IMPORTANT\]
+>
+> **收藏项目**，你将能第一时间获取 OpenCompass 的最新动态～⭐️
+
+<details>
+  <summary><kbd>Star History</kbd></summary>
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=open-compass%2Fopencompass&theme=dark&type=Date">
+    <img width="100%" src="https://api.star-history.com/svg?repos=open-compass%2Fopencompass&type=Date">
+  </picture>
+</details>
+
+## 🧭	欢迎
+
+来到**OpenCompass**！
+
+就像指南针在我们的旅程中为我们导航一样，我们希望OpenCompass能够帮助你穿越评估大型语言模型的重重迷雾。OpenCompass提供丰富的算法和功能支持，期待OpenCompass能够帮助社区更便捷地对NLP模型的性能进行公平全面的评估。
+
+🚩🚩🚩 欢迎加入 OpenCompass！我们目前**招聘全职研究人员/工程师和实习生**。如果您对 LLM 和 OpenCompass 充满热情，请随时通过[电子邮件](mailto:zhangsongyang@pjlab.org.cn)与我们联系。我们非常期待与您交流！
+
+🔥🔥🔥 祝贺 **OpenCompass 作为大模型标准测试工具被Meta AI官方推荐**, 点击 Llama 的 [入门文档](https://ai.meta.com/llama/get-started/#validation) 获取更多信息。
+
+> **注意**<br />
+> 重要通知：从 v0.4.0 版本开始，所有位于 ./configs/datasets、./configs/models 和 ./configs/summarizers 目录下的 AMOTIC 配置文件将迁移至 opencompass 包中。请及时更新您的配置文件路径。
+
+## 🚀 最新进展 <a><img width="35" height="20" src="https://user-images.githubusercontent.com/12782558/212848161-5e783dd6-11e8-4fe0-bbba-39ffb77730be.png"></a>
+
+- **\[2025.07.26\]** OpenCompass 现已支持Intern-S1相关的通用及科学评测基准，请参阅[Intern-S1评测教程](https://opencompass.readthedocs.io/zh-cn/latest/user_guides/interns1.html)了解详情！🔥🔥🔥
+- **\[2025.04.01\]** OpenCompass 现已支持 `CascadeEvaluator`，允许多个评估器按顺序工作，可以为更复杂的评估场景创建自定义评估流程，查看[文档](docs/zh_cn/advanced_guides/llm_judge.md)了解具体用法！🔥🔥🔥
+- **\[2025.03.11\]** 现已支持 `SuperGPQA`  覆盖285 个研究生学科的知识能力评测，欢迎尝试！🔥🔥🔥
+- **\[2025.02.28\]** 我们为 `DeepSeek-R1` 系列模型添加了教程，请查看 [评估推理模型](docs/zh_cn/user_guides/deepseek_r1.md) 了解更多详情！🔥🔥🔥
+- **\[2025.02.15\]** 我们新增了两个实用的评测工具：用于LLM作为评判器的`GenericLLMEvaluator`和用于数学推理评估的`MATHVerifyEvaluator`。查看[LLM评判器](docs/zh_cn/advanced_guides/llm_judge.md)和[数学能力评测](docs/zh_cn/advanced_guides/general_math.md)文档了解更多详情！🔥🔥🔥
+- **\[2025.01.16\]** 我们现已支持 [InternLM3-8B-Instruct](https://huggingface.co/internlm/internlm3-8b-instruct) 模型，该模型在推理、知识类任务上取得同量级最优性能，欢迎尝试。
+- **\[2024.12.17\]** 我们提供了12月CompassAcademic学术榜单评估脚本 [CompassAcademic](configs/eval_academic_leaderboard_202412.py)，你可以通过简单地配置复现官方评测结果。
+- **\[2024.10.14\]** 现已支持OpenAI多语言问答数据集[MMMLU](https://huggingface.co/datasets/openai/MMMLU)，欢迎尝试! 🔥🔥🔥
+- **\[2024.09.19\]** 现已支持[Qwen2.5](https://huggingface.co/Qwen)(0.5B to 72B) ，可以使用多种推理后端(huggingface/vllm/lmdeploy), 欢迎尝试! 🔥🔥🔥
+- **\[2024.09.05\]** 现已支持OpenAI o1 模型(`o1-mini-2024-09-12` and `o1-preview-2024-09-12`), 欢迎尝试! 🔥🔥🔥
+- **\[2024.09.05\]** OpenCompass 现在支持通过模型后处理来��行答案提取，以更准确地展示模型的能力。作为此次更新的一部分，我们集成了 [XFinder](https://github.com/IAAR-Shanghai/xFinder) 作为首个后处理模型。具体信息请参阅 [文档](opencompass/utils/postprocessors/xfinder/README.md)，欢迎尝试！ 🔥🔥🔥
+- **\[2024.08.20\]** OpenCompass 现已支持 [SciCode](https://github.com/scicode-bench/SciCode): A Research Coding Benchmark Curated by Scientists。 🔥🔥🔥
+- **\[2024.08.16\]** OpenCompass 现已支持全新的长上下文语言模型评估基准——[RULER](https://arxiv.org/pdf/2404.06654)。RULER 通过灵活的配置，提供了对长上下文包括检索、多跳追踪、聚合和问答等多种任务类型的评测，欢迎访问[RULER](configs/datasets/ruler/README.md)。🔥🔥🔥
+- **\[2024.07.23\]** 我们支持了[Gemma2](https://huggingface.co/collections/google/gemma-2-release-667d6600fd5220e7b967f315)模型，欢迎试用！🔥🔥🔥
+- **\[2024.07.23\]** 我们支持了[ModelScope](www.modelscope.cn)数据集，您可以按需加载，无需事先下载全部数据到本地，欢迎试用！🔥🔥🔥
+- **\[2024.07.17\]** 我们发布了CompassBench-202407榜单的示例数据和评测规则，敬请访问 [CompassBench](https://opencompass.readthedocs.io/zh-cn/latest/advanced_guides/compassbench_intro.html) 获取更多信息。 🔥🔥🔥
+- **\[2024.07.17\]** 我们正式发布 NeedleBench 的[技术报告](http://arxiv.org/abs/2407.11963)。诚邀您访问我们的[帮助文档](https://opencompass.readthedocs.io/zh-cn/latest/advanced_guides/needleinahaystack_eval.html)进行评估。🔥🔥🔥
+- **\[2024.07.04\]** OpenCompass 现已支持 InternLM2.5， 它拥有卓越的推理性能、有效支持百万字超长上下文以及工具调用能力整体升级，欢迎访问[OpenCompass Config](https://github.com/open-compass/opencompass/tree/main/configs/models/hf_internlm) 和 [InternLM](https://github.com/InternLM/InternLM) .🔥🔥🔥.
+- **\[2024.06.20\]** OpenCompass 现已支持一键切换推理加速后端，助力评测过程更加高效。除了默认的HuggingFace推理后端外，还支持了常用的 [LMDeploy](https://github.com/InternLM/lmdeploy) 和 [vLLM](https://github.com/vllm-project/vllm) ，支持命令行一键切换和部署 API 加速服务两种方式，详细使用方法见[文档](docs/zh_cn/advanced_guides/accelerator_intro.md)。欢迎试用！🔥🔥🔥.
+
+> [更多](docs/zh_cn/notes/news.md)
+
+## 📊 性能榜单
+
+我们将陆续提供开源模型和 API 模型的具体性能榜单，请见 [OpenCompass Leaderboard](https://rank.opencompass.org.cn/home) 。如需加入评测，请提供模型仓库地址或标准的 API 接口至邮箱  `opencompass@pjlab.org.cn`.
+
+你也可以参考[学术榜单精度复现教程](https://opencompass.readthedocs.io/zh-cn/latest/academic.html)，快速地复现榜单的结果。
+
+<p align="right"><a href="#top">🔝返回顶部</a></p>
+
+## 🛠️ 安装指南
+
+下面提供了快速安装和数据集准备的步骤。
+
+### 💻 环境搭建
+
+我们强烈建议使用 `conda` 来管理您的 Python 环境。
+
+- #### 创建虚拟环境
+
+  ```bash
+  conda create --name opencompass python=3.10 -y
+  conda activate opencompass
+  ```
+
+- #### 通过pip安装OpenCompass
+
+  ```bash
+  # 支持绝大多数数据集及模型
+  pip install -U opencompass
+
+  # 完整安装（支持更多数据集）
+  # pip install "opencompass[full]"
+
+  # 模型推理后端，由于这些推理后端通常存在依赖冲突，建议使用不同的虚拟环境来管理它们。
+  # pip install "opencompass[lmdeploy]"
+  # pip install "opencompass[vllm]"
+
+  # API 测试（例如 OpenAI、Qwen）
+  # pip install "opencompass[api]"
+  ```
+
+- #### 基于源码安装OpenCompass
+
+  如果希望使用 OpenCompass 的最新功能，也可以从源代码构建它：
+
+  ```bash
+  git clone https://github.com/open-compass/opencompass opencompass
+  cd opencompass
+  pip install -e .
+  # pip install -e ".[full]"
+  # pip install -e ".[vllm]"
+  ```
+
+### 📂 数据准备
+
+#### 提前离线下载
+
+OpenCompass支持使用本地数据集进行评测，数据集的下载和解压可以通过以下命令完成：
+
+```bash
+# 下载数据集到 data/ 处
+wget https://github.com/open-compass/opencompass/releases/download/0.2.2.rc1/OpenCompassData-core-20240207.zip
+unzip OpenCompassData-core-20240207.zip
+```
+
+#### 从 OpenCompass 自动下载
+
+我们已经支持从OpenCompass存储服务器自动下载数据集。您可以通过额外的 `--dry-run` 参数来运行评估以下载这些数据集。
+目前支持的数据集列表在[这里](https://github.com/open-compass/opencompass/blob/main/opencompass/utils/datasets_info.py#L259)。更多数据集将会很快上传。
+
+#### (可选) 使用 ModelScope 自动下载
+
+另外，您还可以使用[ModelScope](www.modelscope.cn)来加载数据集：
+环境准备：
+
+```bash
+pip install modelscope
+export DATASET_SOURCE=ModelScope
+```
+
+配置好环境后，无需下载全部数据，直接提交评测任务即可。目前支持的数据集有：
+
+```bash
+humaneval, triviaqa, commonsenseqa, tydiqa, strategyqa, cmmlu, lambada, piqa, ceval, math, LCSTS, Xsum, winogrande, openbookqa, AGIEval, gsm8k, nq, race, siqa, mbpp, mmlu, hellaswag, ARC, BBH, xstory_cloze, summedits, GAOKAO-BENCH, OCNLI, cmnli
+```
+
+有部分第三方功能,如 Humaneval 以及 Llama,可能需要额外步骤才能正常运行，详细步骤请参考[安装指南](https://opencompass.readthedocs.io/zh_CN/latest/get_started/installation.html)。
+
+<p align="right"><a href="#top">🔝返回顶部</a></p>
+
+## 🏗️ ️评测
+
+在确保按照上述步骤正确安装了 OpenCompass 并准备好了数据集之后，现在您可以开始使用 OpenCompass 进行首次评估！
+
+- ### 首次评测
+
+  OpenCompass 支持通过命令行界面 (CLI) 或 Python 脚本来设置配置。对于简单的评估设置，我们推荐使用 CLI；而对于更复杂的评估，则建议使用脚本方式。你可以在examples文件夹下找到更多脚本示例。
+
+  ```bash
+  # 命令行界面 (CLI)
+  opencompass --models hf_internlm2_5_1_8b_chat --datasets demo_gsm8k_chat_gen
+
+  # Python 脚本
+  opencompass examples/eval_chat_demo.py
+  ```
+
+  你可以在[examples](./examples) 文件夹下找到更多的脚本示例。
+
+- ### API评测
+
+  OpenCompass 在设计上并不区分开源模型与 API 模型。您可以以相同的方式或甚至在同一设置中评估这两种类型的模型。
+
+  ```bash
+  export OPENAI_API_KEY="YOUR_OPEN_API_KEY"
+  # 命令行界面 (CLI)
+  opencompass --models gpt_4o_2024_05_13 --datasets demo_gsm8k_chat_gen
+
+  # Python 脚本
+  opencompass  examples/eval_api_demo.py
+
+
+  # 现已支持 o1_mini_2024_09_12/o1_preview_2024_09_12  模型, 默认情况下 max_completion_tokens=8192.
+  ```
+
+- ### 推理后端
+
+  另外，如果您想使用除 HuggingFace 之外的推理后端来进行加速评估，比如 LMDeploy 或 vLLM，可以通过以下命令进行。请确保您已经为所选的后端安装了必要的软件包，并且您的模型支持该后端的加速推理。更多信息，请参阅关于推理加速后端的文档 [这里](docs/zh_cn/advanced_guides/accelerator_intro.md)。以下是使用 LMDeploy 的示例：
+
+  ```bash
+  opencompass --models hf_internlm2_5_1_8b_chat --datasets demo_gsm8k_chat_gen -a lmdeploy
+  ```
+
+- ### 支持的模型与数据集
+
+  OpenCompass 预定义了许多模型和数据集的配置，你可以通过 [工具](./docs/zh_cn/tools.md#ListConfigs) 列出所有可用的模型和数据集配置。
+
+  ```bash
+  # 列出所有配置
+  python tools/list_configs.py
+  # 列出所有跟 llama 及 mmlu 相关的配置
+  python tools/list_configs.py llama mmlu
+  ```
+
+  #### 支持的模型
+
+  如果模型不在列表中，但支持 Huggingface AutoModel 类或支持针对 OpenAI 接口的推理引擎封装（详见[官方文档](https://opencompass.readthedocs.io/zh-cn/latest/advanced_guides/new_model.html)），您仍然可以使用 OpenCompass 对其进行评估。欢迎您贡献维护 OpenCompass 支持的模型和数据集列表。
+
+  ```bash
+  opencompass --datasets demo_gsm8k_chat_gen --hf-type chat --hf-path internlm/internlm2_5-1_8b-chat
+  ```
+
+  #### 支持的数据集
+
+  目前，OpenCompass针对数据集给出了标准的推荐配置。通常，`_gen.py`或`_llm_judge_gen.py`为结尾的配置文件将指向我们为该数据集提供的推荐配置。您可以参阅[官方文档](https://opencompass.readthedocs.io/zh-cn/latest/dataset_statistics.html) 的数据集统计章节来获取详细信息。
+
+  ```bash
+  # 基于规则的推荐配置
+  opencompass --datasets aime2024_gen --models hf_internlm2_5_1_8b_chat
+
+  # 基于LLM Judge的推荐配置
+  opencompass --datasets aime2024_llmjudge_gen --models hf_internlm2_5_1_8b_chat
+  ```
+
+  此外，如果你想在多块 GPU 上使用模型进行推理，您可以使用 `--max-num-worker` 参数。
+
+  ```bash
+  CUDA_VISIBLE_DEVICES=0,1 opencompass --datasets demo_gsm8k_chat_gen --hf-type chat --hf-path internlm/internlm2_5-1_8b-chat --max-num-worker 2
+  ```
+opencompass --datasets triviaqa --hf-type base --hf-path /mnt/jfzn/msj/train_exp/mask_gdn_1B_hrr-rank4 --max-num-worker 2
+
+opencompass --datasets longbench --models gla-1.3B --max-num-worker 8
+> \[!TIP\]
+>
+> `--hf-num-gpus` 用于 模型并行(huggingface 格式)，`--max-num-worker` 用于数据并行。
+
+> \[!TIP\]
+>
+> configuration with `_ppl` is designed for base model typically.
+> 配置带 `_ppl` 的配置设计给基础模型使用。
+> 配置带 `_gen` 的配置可以同时用于基础模型和对话模型。
+
+通过命令行或配置文件，OpenCompass 还支持评测 API 或自定义模型，以及更多样化的评测策略。请阅读[快速开始](https://opencompass.readthedocs.io/zh_CN/latest/get_started/quick_start.html)了解如何运行一个评测任务。
+
+更多教程请查看我们的[文档](https://opencompass.readthedocs.io/zh_CN/latest/index.html)。
+
+<p align="right"><a href="#top">🔝返回顶部</a></p>
+
+## 📣 OpenCompass 2.0
+
+我们很高兴发布 OpenCompass 司南 2.0 大模型评测体系，它主要由三大核心模块构建而成：[CompassKit](https://github.com/open-compass)、[CompassHub](https://hub.opencompass.org.cn/home)以及[CompassRank](https://rank.opencompass.org.cn/home)。
+
+**CompassRank** 系统进行了重大革新与提升，现已成为一个兼容并蓄的排行榜体系，不仅囊括了开源基准测试项目，还包含了私有基准测试。此番升级极大地拓宽了对行业内各类模型进行全面而深入测评的可能性。
+
+**CompassHub** 创新性地推出了一个基准测试资源导航平台，其设计初衷旨在简化和加快研究人员及行业从业者在多样化的基准测试库中进行搜索与利用的过程。为了让更多独具特色的基准测试成果得以在业内广泛传播和应用，我们热忱欢迎各位将自定义的基准数据贡献至CompassHub平台。只需轻点鼠标，通过访问[这里](https://hub.opencompass.org.cn/dataset-submit)，即可启动提交流程。
+
+**CompassKit** 是一系列专为大型语言模型和大型视觉-语言模型打造的强大评估工具合集，它所提供的全面评测工具集能够有效地对这些复杂模型的功能性能进行精准测量和科学评估。在此，我们诚挚邀请您在学术研究或产品研发过程中积极尝试运用我们的工具包，以助您取得更加丰硕的研究成果和产品优化效果。
+
+## ✨ 介绍
+
+![image](https://github.com/open-compass/opencompass/assets/22607038/30bcb2e2-3969-4ac5-9f29-ad3f4abb4f3b)
+
+OpenCompass 是面向大模型评测的一站式平台。其主要特点如下：
+
+- **开源可复现**：提供公平、公开、可复现的大模型评测方案
+
+- **全面的能力维度**：五大维度设计，提供 70+ 个数据集约 40 万题的的模型评测方案，全面评估模型能力
+
+- **丰富的模型支持**：已支持 20+ HuggingFace 及 API 模型
+
+- **分布式高效评测**：一行命令实现任务分割和分布式评测，数小时即可完成千亿模型全量评测
+
+- **多样化评测范式**：支持零样本、小样本及思维链评测，结合标准型或对话型提示词模板，轻松激发各种模型最大性能
+
+- **灵活化拓展**：想增加新模型或数据集？想要自定义更高级的任务分割策略，甚至接入新的集群管理系统？OpenCompass 的一切均可轻松扩展！
+
+## 📖 数据集支持
+
+我们已经在OpenCompass官网的文档中支持了所有可在本平台上使用的数据集的统计列表。
+
+您可以通过排序、筛选和搜索等功能从列表中快速找到您需要的数据集。
+
+详情请参阅 [官方文档](https://opencompass.readthedocs.io/zh-cn/latest/dataset_statistics.html) 的数据集统计章节。
+
+<p align="right"><a href="#top">🔝返回顶部</a></p>
+
+## 📖 模型支持
+
+<table align="center">
+  <tbody>
+    <tr align="center" valign="bottom">
+      <td>
+        <b>开源模型</b>
+      </td>
+      <td>
+        <b>API 模型</b>
+      </td>
+      <!-- <td>
+        <b>自定义模型</b>
+      </td> -->
+    </tr>
+    <tr valign="top">
+      <td>
+
+- [Alpaca](https://github.com/tatsu-lab/stanford_alpaca)
+- [Baichuan](https://github.com/baichuan-inc)
+- [BlueLM](https://github.com/vivo-ai-lab/BlueLM)
+- [ChatGLM2](https://github.com/THUDM/ChatGLM2-6B)
+- [ChatGLM3](https://github.com/THUDM/ChatGLM3-6B)
+- [Gemma](https://huggingface.co/google/gemma-7b)
+- [InternLM](https://github.com/InternLM/InternLM)
+- [LLaMA](https://github.com/facebookresearch/llama)
+- [LLaMA3](https://github.com/meta-llama/llama3)
+- [Qwen](https://github.com/QwenLM/Qwen)
+- [TigerBot](https://github.com/TigerResearch/TigerBot)
+- [Vicuna](https://github.com/lm-sys/FastChat)
+- [WizardLM](https://github.com/nlpxucan/WizardLM)
+- [Yi](https://github.com/01-ai/Yi)
+- ……
+
+</td>
+<td>
+
+- OpenAI
+- Gemini
+- Claude
+- ZhipuAI(ChatGLM)
+- Baichuan
+- ByteDance(YunQue)
+- Huawei(PanGu)
+- 360
+- Baidu(ERNIEBot)
+- MiniMax(ABAB-Chat)
+- SenseTime(nova)
+- Xunfei(Spark)
+- ……
+
+</td>
+
+</tr>
+  </tbody>
+</table>
+
+<p align="right"><a href="#top">🔝返回顶部</a></p>
+
+## 🔜 路线图
+
+- [x] 主观评测
+  - [x] 发布主观评测榜单
+  - [x] 发布主观评测数据集
+- [x] 长文本
+  - [x] 支持广泛的长文本评测集
+  - [ ] 发布长文本评测榜单
+- [x] 代码能力
+  - [ ] 发布代码能力评测榜单
+  - [x] 提供非Python语言的评测服务
+- [x] 智能体
+  - [ ] 支持丰富的智能体方案
+  - [x] 提供智能体评测榜单
+- [x] 鲁棒性
+  - [x] 支持各类攻击方法
+
+## 👷‍♂️ 贡献
+
+我们感谢所有的贡献者为改进和提升 OpenCompass 所作出的努力。请参考[贡献指南](https://opencompass.readthedocs.io/zh_CN/latest/notes/contribution_guide.html)来了解参与项目贡献的相关指引。
+
+<a href="https://github.com/open-compass/opencompass/graphs/contributors" target="_blank">
+  <table>
+    <tr>
+      <th colspan="2">
+        <br><img src="https://contrib.rocks/image?repo=open-compass/opencompass"><br><br>
+      </th>
+    </tr>
+  </table>
+</a>
+
+## 🤝 致谢
+
+该项目部分的代码引用并修改自 [OpenICL](https://github.com/Shark-NLP/OpenICL)。
+
+该项目部分的数据集和提示词实现修改自 [chain-of-thought-hub](https://github.com/FranxYao/chain-of-thought-hub), [instruct-eval](https://github.com/declare-lab/instruct-eval)
+
+## 🖊️ 引用
+
+```bibtex
+@misc{2023opencompass,
+    title={OpenCompass: A Universal Evaluation Platform for Foundation Models},
+    author={OpenCompass Contributors},
+    howpublished = {\url{https://github.com/open-compass/opencompass}},
+    year={2023}
+}
+```
+
+<p align="right"><a href="#top">🔝返回顶部</a></p>
+
+[github-contributors-link]: https://github.com/open-compass/opencompass/graphs/contributors
+[github-contributors-shield]: https://img.shields.io/github/contributors/open-compass/opencompass?color=c4f042&labelColor=black&style=flat-square
+[github-forks-link]: https://github.com/open-compass/opencompass/network/members
+[github-forks-shield]: https://img.shields.io/github/forks/open-compass/opencompass?color=8ae8ff&labelColor=black&style=flat-square
+[github-issues-link]: https://github.com/open-compass/opencompass/issues
+[github-issues-shield]: https://img.shields.io/github/issues/open-compass/opencompass?color=ff80eb&labelColor=black&style=flat-square
+[github-license-link]: https://github.com/open-compass/opencompass/blob/main/LICENSE
+[github-license-shield]: https://img.shields.io/github/license/open-compass/opencompass?color=white&labelColor=black&style=flat-square
+[github-release-link]: https://github.com/open-compass/opencompass/releases
+[github-release-shield]: https://img.shields.io/github/v/release/open-compass/opencompass?color=369eff&labelColor=black&logo=github&style=flat-square
+[github-releasedate-link]: https://github.com/open-compass/opencompass/releases
+[github-releasedate-shield]: https://img.shields.io/github/release-date/open-compass/opencompass?labelColor=black&style=flat-square
+[github-stars-link]: https://github.com/open-compass/opencompass/stargazers
+[github-stars-shield]: https://img.shields.io/github/stars/open-compass/opencompass?color=ffcb47&labelColor=black&style=flat-square
+[github-trending-shield]: https://trendshift.io/api/badge/repositories/6630
+[github-trending-url]: https://trendshift.io/repositories/6630

a.sh

@@ -0,0 +1,6 @@
+export http_proxy="http://cloudml:gP1dY0uI0o@10.119.176.202:3128"
+export https_proxy="http://cloudml:gP1dY0uI0o@10.119.176.202:3128"
+# opencompass --datasets longbench --models mask_gdn_t-1.3B --max-num-worker 8 
+# opencompass --datasets longbench --models gdn-1.3B --max-num-worker 8
+
+opencompass --datasets needlebench_v2_4k_single --models mask_gdn_t-1.3B --max-num-worker 8  

dataset-index.yml

@@ -0,0 +1,1157 @@
+- ifeval:
+    name: IFEval
+    category: Instruction Following
+    paper: https://arxiv.org/pdf/2311.07911
+    configpath: opencompass/configs/datasets/IFEval/IFEval_gen.py
+    configpath_llmjudge: ''
+- nphard:
+    name: NPHardEval
+    category: Reasoning
+    paper: https://arxiv.org/pdf/2312.14890v2
+    configpath: opencompass/configs/datasets/NPHardEval/NPHardEval_gen.py
+    configpath_llmjudge: ''
+- pmmeval:
+    name: PMMEval
+    category: Language
+    paper: https://arxiv.org/pdf/2411.09116v1
+    configpath: opencompass/configs/datasets/PMMEval/pmmeval_gen.py
+    configpath_llmjudge: ''
+- pi_llm:
+    name: PI-LLM
+    category: Memory
+    paper: https://arxiv.org/abs/2506.08184
+    configpath: opencompass/configs/datasets/PI_LLM/pi_llm_gen.py
+    configpath_llmjudge: ''
+- theoremqa:
+    name: TheroremQA
+    category: Reasoning
+    paper: https://arxiv.org/pdf/2305.12524
+    configpath: opencompass/configs/datasets/TheroremQA/TheoremQA_gen.py
+    configpath_llmjudge: ''
+- agieval:
+    name: AGIEval
+    category: Examination
+    paper: https://arxiv.org/pdf/2304.06364
+    configpath: opencompass/configs/datasets/agieval/agieval_gen.py
+    configpath_llmjudge: ''
+- babilong:
+    name: BABILong
+    category: Long Context
+    paper: https://arxiv.org/pdf/2406.10149
+    configpath: opencompass/configs/datasets/babilong
+    configpath_llmjudge: ''
+- bigcodebench:
+    name: BigCodeBench
+    category: Code
+    paper: https://arxiv.org/pdf/2406.15877
+    configpath: opencompass/configs/datasets/bigcodebench/bigcodebench_gen.py
+    configpath_llmjudge: ''
+- calm:
+    name: CaLM
+    category: Reasoning
+    paper: https://arxiv.org/pdf/2405.00622
+    configpath: opencompass/configs/datasets/calm/calm.py
+    configpath_llmjudge: ''
+- infinitebench:
+    name: InfiniteBench (∞Bench)
+    category: Long Context
+    paper: https://aclanthology.org/2024.acl-long.814.pdf
+    configpath: opencompass/configs/datasets/infinitebench/infinitebench.py
+    configpath_llmjudge: ''
+- korbench:
+    name: KOR-Bench
+    category: Reasoning
+    paper: https://arxiv.org/pdf/2410.06526v1
+    configpath: opencompass/configs/datasets/korbench/korbench_gen.py
+    configpath_llmjudge: opencompass/configs/datasets/korbench/korbench_llm_judge_gen.py
+- lawbench:
+    name: LawBench
+    category: Knowledge / Law
+    paper: https://arxiv.org/pdf/2309.16289
+    configpath:
+      - opencompass/configs/datasets/lawbench/lawbench_zero_shot_gen_002588.py
+      - opencompass/configs/datasets/lawbench/lawbench_one_shot_gen_002588.py
+    configpath_llmjudge: ''
+- leval:
+    name: L-Eval
+    category: Long Context
+    paper: https://arxiv.org/pdf/2307.11088v1
+    configpath: opencompass/configs/datasets/leval/leval.py
+    configpath_llmjudge: ''
+- livecodebench:
+    name: LiveCodeBench
+    category: Code
+    paper: https://arxiv.org/pdf/2403.07974
+    configpath: opencompass/configs/datasets/livecodebench/livecodebench_gen.py
+    configpath_llmjudge: ''
+- livemathbench:
+    name: LiveMathBench
+    category: Math
+    paper: https://arxiv.org/pdf/2412.13147
+    configpath: opencompass/configs/datasets/livemathbench/livemathbench_gen.py
+    configpath_llmjudge: ''
+- livereasonbench:
+    name: LiveReasonBench
+    category: Reasoning
+    paper: ''
+    configpath: opencompass/configs/datasets/livereasonbench/livereasonbench_gen.py
+    configpath_llmjudge: ''
+- longbench:
+    name: LongBench
+    category: Long Context
+    paper: https://github.com/THUDM/LongBench
+    configpath:
+      - opencompass/configs/datasets/longbench/longbench.py
+      - opencompass/configs/datasets/longbenchv2/longbenchv2_gen.py
+    configpath_llmjudge: ''
+- lveval:
+    name: LV-Eval
+    category: Long Context
+    paper: https://arxiv.org/pdf/2402.05136
+    configpath: opencompass/configs/datasets/lveval/lveval.py
+    configpath_llmjudge: ''
+- mastermath2024v1:
+    name: Mastermath2024v1
+    category: Math
+    paper: ''
+    configpath: opencompass/configs/datasets/mastermath2024v1/mastermath2024v1_gen.py
+    configpath_llmjudge: ''
+- matbench:
+    name: matbench
+    category: Science / Material
+    paper: 'https://www.nature.com/articles/s41524-020-00406-3'
+    configpath: opencompass/configs/datasets/matbench/matbench_gen_f71840.py
+    configpath_llmjudge: ''
+- medbench:
+    name: MedBench
+    category: Knowledge / Medicine
+    paper: https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=10778138
+    configpath: opencompass/configs/datasets/MedBench/medbench_gen.py
+    configpath_llmjudge: ''
+- MedCalc_Bench:
+    name: MedCalc_Bench
+    category: Knowledge / Medicine
+    paper: https://arxiv.org/abs/2406.12036
+    configpath: opencompass/configs/datasets/MedCalc_Bench/MedCalcBench_official_gen_a5155f.py
+    configpath_llmjudge: ''
+- MedXpertQA:
+    name: MedQA
+    category: Knowledge / Medicine
+    paper: https://arxiv.org/abs/2009.13081
+    configpath: opencompass/configs/datasets/MedQA/MedQA_gen.py
+    configpath_llmjudge: opencompass/configs/datasets/MedQA/MedQA_llmjudge_gen.py
+- MedXpertQA:
+    name: MedXpertQA
+    category: Knowledge / Medicine
+    paper: https://arxiv.org/abs/2501.18362
+    configpath: opencompass/configs/datasets/MedXpertQA/MedXpertQA_gen.py
+    configpath_llmjudge: opencompass/configs/datasets/MedXpertQA/MedXpertQA_llmjudge_gen.py
+- ClinicBench:
+    name: ClinicBench
+    category: Knowledge / Medicine
+    paper: https://arxiv.org/abs/2405.00716
+    configpath: ''
+    configpath_llmjudge: opencompass/configs/datasets/ClinicBench/ClinicBench_llmjudge_gen.py
+- ScienceQA:
+    name: ScienceQA
+    category: Knowledge / Medicine
+    paper: https://arxiv.org/abs/2209.09513
+    configpath: ''
+    configpath_llmjudge: opencompass/configs/datasets/ScienceQA/ScienceQA_llmjudge_gen.py
+- PubMedQA:
+    name: PubMedQA
+    category: Knowledge / Medicine
+    paper: https://arxiv.org/abs/1909.06146
+    configpath: ''
+    configpath_llmjudge: opencompass/configs/datasets/PubMedQA/PubMedQA_llmjudge_gen.py
+- musr:
+    name: MuSR
+    category: Reasoning
+    paper: https://arxiv.org/pdf/2310.16049
+    configpath: opencompass/configs/datasets/musr/musr_gen.py
+    configpath_llmjudge: opencompass/configs/datasets/musr/musr_llm_judge_gen.py
+- needlebench:
+    name: NeedleBench V1 (Deprecated)
+    category: Long Context
+    paper: https://arxiv.org/abs/2407.11963v1
+    configpath: opencompass/configs/datasets/needlebench
+    configpath_llmjudge: ''
+- needlebench_v2:
+    name: NeedleBench V2
+    category: Long Context
+    paper: https://arxiv.org/abs/2407.11963v2
+    configpath: opencompass/configs/datasets/needlebench_v2
+    configpath_llmjudge: ''
+- ruler:
+    name: RULER
+    category: Long Context
+    paper: https://arxiv.org/pdf/2404.06654
+    configpath: opencompass/configs/datasets/ruler
+    configpath_llmjudge: ''
+- alignment:
+    name: AlignBench
+    category: Subjective / Alignment
+    paper: https://arxiv.org/pdf/2311.18743
+    configpath: opencompass/configs/datasets/subjective/alignbench
+    configpath_llmjudge: ''
+- alpaca:
+    name: AlpacaEval
+    category: Subjective / Instruction Following
+    paper: https://github.com/tatsu-lab/alpaca_eval
+    configpath: opencompass/configs/datasets/subjective/aplaca_eval
+    configpath_llmjudge: ''
+- arenahard:
+    name: Arena-Hard
+    category: Subjective / Chatbot
+    paper: https://lmsys.org/blog/2024-04-19-arena-hard/
+    configpath: opencompass/configs/datasets/subjective/arena_hard
+    configpath_llmjudge: ''
+- flames:
+    name: FLAMES
+    category: Subjective / Alignment
+    paper: https://arxiv.org/pdf/2311.06899
+    configpath: opencompass/configs/datasets/subjective/flames/flames_gen.py
+    configpath_llmjudge: ''
+- fofo:
+    name: FOFO
+    category: Subjective / Format Following
+    paper: https://arxiv.org/pdf/2402.18667
+    configpath: opencompass/configs/datasets/subjective/fofo
+    configpath_llmjudge: ''
+- followbench:
+    name: FollowBench
+    category: Subjective / Instruction Following
+    paper: https://arxiv.org/pdf/2310.20410
+    configpath: opencompass/configs/datasets/subjective/followbench
+    configpath_llmjudge: ''
+- hellobench:
+    name: HelloBench
+    category: Subjective / Long Context
+    paper: https://arxiv.org/pdf/2409.16191
+    configpath: opencompass/configs/datasets/subjective/hellobench
+    configpath_llmjudge: ''
+- judgerbench:
+    name: JudgerBench
+    category: Subjective / Long Context
+    paper: https://arxiv.org/pdf/2410.16256
+    configpath: opencompass/configs/datasets/subjective/judgerbench
+    configpath_llmjudge: ''
+- multiround:
+    name: MT-Bench-101
+    category: Subjective / Multi-Round
+    paper: https://arxiv.org/pdf/2402.14762
+    configpath: opencompass/configs/datasets/subjective/multiround
+    configpath_llmjudge: ''
+- wildbench:
+    name: WildBench
+    category: Subjective / Real Task
+    paper: https://arxiv.org/pdf/2406.04770
+    configpath: opencompass/configs/datasets/subjective/wildbench
+    configpath_llmjudge: ''
+- teval:
+    name: T-Eval
+    category: Tool Utilization
+    paper: https://arxiv.org/pdf/2312.14033
+    configpath:
+      - opencompass/configs/datasets/teval/teval_en_gen.py
+      - opencompass/configs/datasets/teval/teval_zh_gen.py
+    configpath_llmjudge: ''
+- finalceiq:
+    name: FinanceIQ
+    category: Knowledge / Finance
+    paper: https://github.com/Duxiaoman-DI/XuanYuan/tree/main/FinanceIQ
+    configpath: opencompass/configs/datasets/FinanceIQ/FinanceIQ_gen.py
+    configpath_llmjudge: ''
+- gaokaobench:
+    name: GAOKAOBench
+    category: Examination
+    paper: https://arxiv.org/pdf/2305.12474
+    configpath: opencompass/configs/datasets/GaokaoBench/GaokaoBench_gen.py
+    configpath_llmjudge: ''
+- lcbench:
+    name: LCBench
+    category: Code
+    paper: https://github.com/open-compass/CodeBench/
+    configpath: opencompass/configs/datasets/LCBench/lcbench_gen.py
+    configpath_llmjudge: ''
+- MMLUArabic:
+    name: ArabicMMLU
+    category: Language
+    paper: https://arxiv.org/pdf/2402.12840
+    configpath: opencompass/configs/datasets/MMLUArabic/MMLUArabic_gen.py
+    configpath_llmjudge: ''
+- OpenFinData:
+    name: OpenFinData
+    category: Knowledge / Finance
+    paper: https://github.com/open-compass/OpenFinData
+    configpath: opencompass/configs/datasets/OpenFinData/OpenFinData_gen.py
+    configpath_llmjudge: ''
+- QuALITY:
+    name: QuALITY
+    category: Long Context
+    paper: https://arxiv.org/pdf/2112.08608
+    configpath: opencompass/configs/datasets/QuALITY/QuALITY_gen.py
+    configpath_llmjudge: ''
+- advglue:
+    name: Adversarial GLUE
+    category: Safety
+    paper: https://openreview.net/pdf?id=GF9cSKI3A_q
+    configpath:
+    - opencompass/configs/datasets/adv_glue/adv_glue_mnli/adv_glue_mnli_gen.py
+    - opencompass/configs/datasets/adv_glue/adv_glue_mnli_mm/adv_glue_mnli_mm_gen.py
+    - opencompass/configs/datasets/adv_glue/adv_glue_mnli_qnli/adv_glue_qnli_gen.py
+    - opencompass/configs/datasets/adv_glue/adv_glue_mnli_qqp/adv_glue_qqp_gen.py
+    - opencompass/configs/datasets/adv_glue/adv_glue_mnli_rte/adv_glue_rte_gen.py
+    - opencompass/configs/datasets/adv_glue/adv_glue_mnli_sst2/adv_glue_sst2_gen.py
+    configpath_llmjudge: ''
+- afqmcd:
+    name: CLUE / AFQMC
+    category: Language
+    paper: https://arxiv.org/pdf/2004.05986
+    configpath: opencompass/configs/datasets/CLUE_afqmc/CLUE_afqmc_gen.py
+    configpath_llmjudge: ''
+- aime2024:
+    name: AIME2024
+    category: Examination
+    paper: https://huggingface.co/datasets/Maxwell-Jia/AIME_2024
+    configpath: opencompass/configs/datasets/aime2024/aime2024_gen.py
+    configpath_llmjudge: opencompass/configs/datasets/aime2024/aime2024_llmjudge_gen.py
+- anli:
+    name: Adversarial NLI
+    category: Reasoning
+    paper: https://arxiv.org/pdf/1910.14599v2
+    configpath: opencompass/configs/datasets/anli/anli_gen.py
+    configpath_llmjudge: ''
+- anthropics_evals:
+    name: Anthropics Evals
+    category: Safety
+    paper: https://arxiv.org/pdf/2212.09251
+    configpath:
+    - opencompass/configs/datasets/anthropics_evals/airisk_gen.py
+    - opencompass/configs/datasets/anthropics_evals/persona_gen.py
+    - opencompass/configs/datasets/anthropics_evals/sycophancy_gen.py
+    configpath_llmjudge: ''
+- apps:
+    name: APPS
+    category: Code
+    paper: https://arxiv.org/pdf/2105.09938
+    configpath:
+    - opencompass/configs/datasets/apps/apps_gen.py
+    - opencompass/configs/datasets/apps/apps_mini_gen.py
+    configpath_llmjudge: ''
+- arc:
+    name: ARC
+    category: Reasoning
+    paper: https://arxiv.org/pdf/1803.05457
+    configpath:
+    - opencompass/configs/datasets/ARC_c/ARC_c_gen.py
+    - opencompass/configs/datasets/ARC_e/ARC_e_gen.py
+    configpath_llmjudge: ''
+- arc_prize_public_eval:
+    name: ARC Prize
+    category: ARC-AGI
+    paper: https://arcprize.org/guide#private
+    configpath: opencompass/configs/datasets/ARC_Prize_Public_Evaluation/arc_prize_public_evaluation_gen.py
+    configpath_llmjudge: ''
+- ax:
+    name: SuperGLUE / AX
+    category: Reasoning
+    paper: https://proceedings.neurips.cc/paper_files/paper/2019/file/4496bf24afe7fab6f046bf4923da8de6-Paper.pdf
+    configpath:
+    - opencompass/configs/datasets/SuperGLUE_AX_b/SuperGLUE_AX_b_gen.py
+    - opencompass/configs/datasets/SuperGLUE_AX_g/SuperGLUE_AX_g_gen.py
+    configpath_llmjudge: ''
+- bbh:
+    name: BIG-Bench Hard
+    category: Reasoning
+    paper: https://arxiv.org/pdf/2210.09261
+    configpath: opencompass/configs/datasets/bbh/bbh_gen.py
+    configpath_llmjudge: opencompass/configs/datasets/bbh/bbh_llm_judge_gen.py
+- bbeh:
+    name: BIG-Bench Extra Hard
+    category: Reasoning
+    paper: https://arxiv.org/abs/2502.19187
+    configpath: opencompass/configs/datasets/bbeh
+    configpath_llmjudge: ''
+- BoolQ:
+    name: SuperGLUE / BoolQ
+    category: Knowledge
+    paper: https://proceedings.neurips.cc/paper_files/paper/2019/file/4496bf24afe7fab6f046bf4923da8de6-Paper.pdf
+    configpath: opencompass/configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_gen.py
+    configpath_llmjudge: ''
+- c3:
+    name: CLUE / C3 (C³)
+    category: Understanding
+    paper: https://arxiv.org/pdf/2004.05986
+    configpath: opencompass/configs/datasets/CLUE_C3/CLUE_C3_gen.py
+    configpath_llmjudge: ''
+- CARDBiomedBench:
+    name: CARDBiomedBench
+    category: Knowledge / Medicine
+    paper: https://www.biorxiv.org/content/10.1101/2025.01.15.633272v1
+    configpath: opencompass/configs/datasets/CARDBiomedBench
+    configpath_llmjudge: 'opencompass/configs/datasets/CARDBiomedBench/CARDBiomedBench_llmjudge_gen_99a231.py'
+- cb:
+    name: SuperGLUE / CB
+    category: Reasoning
+    paper: https://proceedings.neurips.cc/paper_files/paper/2019/file/4496bf24afe7fab6f046bf4923da8de6-Paper.pdf
+    configpath: opencompass/configs/datasets/SuperGLUE_CB/SuperGLUE_CB_gen.py
+    configpath_llmjudge: ''
+- ceval:
+    name: C-EVAL
+    category: Examination
+    paper: https://arxiv.org/pdf/2305.08322v1
+    configpath: opencompass/configs/datasets/ceval/ceval_gen.py
+    configpath_llmjudge: ''
+- charm:
+    name: CHARM
+    category: Reasoning
+    paper: https://arxiv.org/pdf/2403.14112
+    configpath: opencompass/configs/datasets/CHARM/charm_reason_gen.py
+    configpath_llmjudge: ''
+- chembench:
+    name: ChemBench
+    category: Knowledge / Chemistry
+    paper: https://arxiv.org/pdf/2404.01475
+    configpath: opencompass/configs/datasets/ChemBench/ChemBench_gen.py
+    configpath_llmjudge: ''
+- chid:
+    name: FewCLUE / CHID
+    category: Language
+    paper: https://arxiv.org/pdf/2107.07498
+    configpath: opencompass/configs/datasets/FewCLUE_chid/FewCLUE_chid_gen.py
+    configpath_llmjudge: ''
+- chinese_simpleqa:
+    name: Chinese SimpleQA
+    category: Knowledge
+    paper: https://arxiv.org/pdf/2411.07140
+    configpath: opencompass/configs/datasets/chinese_simpleqa/chinese_simpleqa_gen.py
+    configpath_llmjudge: ''
+- cibench:
+    name: CIBench
+    category: Code
+    paper: https://www.arxiv.org/pdf/2407.10499
+    configpath:
+      - opencompass/configs/datasets/CIBench/CIBench_generation_gen_8ab0dc.py
+      - opencompass/configs/datasets/CIBench/CIBench_template_gen_e6b12a.py
+      - opencompass/configs/datasets/CIBench/CIBench_template_oracle_gen_fecda1.py
+    configpath_llmjudge: ''
+- civilcomments:
+    name: CivilComments
+    category: Safety
+    paper: https://arxiv.org/pdf/1903.04561
+    configpath: opencompass/configs/datasets/civilcomments/civilcomments_clp.py
+    configpath_llmjudge: ''
+- clozeTest_maxmin:
+    name: Cloze Test-max/min
+    category: Code
+    paper: https://arxiv.org/pdf/2102.04664
+    configpath: opencompass/configs/datasets/clozeTest_maxmin/clozeTest_maxmin_gen.py
+    configpath_llmjudge: ''
+- cluewsc:
+    name: FewCLUE / CLUEWSC
+    category: Language / WSC
+    paper: https://arxiv.org/pdf/2107.07498
+    configpath: opencompass/configs/datasets/FewCLUE_cluewsc/FewCLUE_cluewsc_gen.py
+    configpath_llmjudge: ''
+- cmb:
+    name: CMB
+    category: Knowledge / Medicine
+    paper: https://arxiv.org/pdf/2308.08833
+    configpath: opencompass/configs/datasets/cmb/cmb_gen.py
+    configpath_llmjudge: ''
+- cmmlu:
+    name: CMMLU
+    category: Understanding
+    paper: https://arxiv.org/pdf/2306.09212
+    configpath: opencompass/configs/datasets/cmmlu/cmmlu_gen.py
+    configpath_llmjudge: opencompass/configs/datasets/cmmlu/cmmlu_llm_judge_gen.py
+- cmnli:
+    name: CLUE / CMNLI
+    category: Reasoning
+    paper: https://arxiv.org/pdf/2004.05986
+    configpath: opencompass/configs/datasets/CLUE_cmnli/CLUE_cmnli_gen.py
+    configpath_llmjudge: ''
+- cmo_fib:
+    name: cmo_fib
+    category: Examination
+    paper: ''
+    configpath: opencompass/configs/datasets/cmo_fib/cmo_fib_gen.py
+    configpath_llmjudge: ''
+- cmrc:
+    name: CLUE / CMRC
+    category: Understanding
+    paper: https://arxiv.org/pdf/2004.05986
+    configpath: opencompass/configs/datasets/CLUE_CMRC/CLUE_CMRC_gen.py
+    configpath_llmjudge: ''
+- commonsenseqa:
+    name: CommonSenseQA
+    category: Knowledge
+    paper: https://arxiv.org/pdf/1811.00937v2
+    configpath: opencompass/configs/datasets/commonsenseqa/commonsenseqa_gen.py
+    configpath_llmjudge: ''
+- commonsenseqa_cn:
+    name: CommonSenseQA-CN
+    category: Knowledge
+    paper: ''
+    configpath: opencompass/configs/datasets/commonsenseqa_cn/commonsenseqacn_gen.py
+    configpath_llmjudge: ''
+- copa:
+    name: SuperGLUE / COPA
+    category: Reasoning
+    paper: https://proceedings.neurips.cc/paper_files/paper/2019/file/4496bf24afe7fab6f046bf4923da8de6-Paper.pdf
+    configpath: opencompass/configs/datasets/SuperGLUE_COPA/SuperGLUE_COPA_gen.py
+    configpath_llmjudge: ''
+- crowspairs:
+    name: CrowsPairs
+    category: Safety
+    paper: https://arxiv.org/pdf/2010.00133
+    configpath: opencompass/configs/datasets/crowspairs/crowspairs_gen.py
+    configpath_llmjudge: ''
+- crowspairs_cn:
+    name: CrowsPairs-CN
+    category: Safety
+    paper: ''
+    configpath: opencompass/configs/datasets/crowspairs_cn/crowspairscn_gen.py
+    configpath_llmjudge: ''
+- cvalues:
+    name: CVALUES
+    category: Safety
+    paper: http://xdp-expriment.oss-cn-zhangjiakou.aliyuncs.com/shanqi.xgh/release_github/CValues.pdf
+    configpath: opencompass/configs/datasets/cvalues/cvalues_responsibility_gen.py
+    configpath_llmjudge: ''
+- drcd:
+    name: CLUE / DRCD
+    category: Understanding
+    paper: https://arxiv.org/pdf/2004.05986
+    configpath: opencompass/configs/datasets/CLUE_DRCD/CLUE_DRCD_gen.py
+    configpath_llmjudge: ''
+- drop:
+    name: DROP (DROP Simple Eval)
+    category: Understanding
+    paper: https://arxiv.org/pdf/1903.00161
+    configpath: opencompass/configs/datasets/drop/drop_gen.py
+    configpath_llmjudge: opencompass/configs/datasets/drop/drop_llm_judge_gen.py
+- ds1000:
+    name: DS-1000
+    category: Code
+    paper: https://arxiv.org/pdf/2211.11501
+    configpath:
+    - opencompass/configs/datasets/ds1000/ds1000_gen_5c4bec.py
+    configpath_llmjudge: ''
+- eprstmt:
+    name: FewCLUE / EPRSTMT
+    category: Understanding
+    paper: https://arxiv.org/pdf/2107.07498
+    configpath: opencompass/configs/datasets/FewCLUE_eprstmt/FewCLUE_eprstmt_gen.py
+    configpath_llmjudge: ''
+- flores:
+    name: Flores
+    category: Language
+    paper: https://aclanthology.org/D19-1632.pdf
+    configpath: opencompass/configs/datasets/flores/flores_gen.py
+    configpath_llmjudge: ''
+- game24:
+    name: Game24
+    category: Math
+    paper: https://huggingface.co/datasets/nlile/24-game
+    configpath: opencompass/configs/datasets/game24/game24_gen.py
+    configpath_llmjudge: ''
+- govrepcrs:
+    name: Government Report Dataset
+    category: Long Context
+    paper: https://aclanthology.org/2021.naacl-main.112.pdf
+    configpath: opencompass/configs/datasets/govrepcrs/govrepcrs_gen.py
+    configpath_llmjudge: ''
+- gpqa:
+    name: GPQA
+    category: Knowledge
+    paper: https://arxiv.org/pdf/2311.12022v1
+    configpath: opencompass/configs/datasets/gpqa/gpqa_gen.py
+    configpath_llmjudge: opencompass/configs/datasets/gpqa/gpqa_llm_judge_gen.py
+- gsm8k:
+    name: GSM8K
+    category: Math
+    paper: https://arxiv.org/pdf/2110.14168v2
+    configpath: opencompass/configs/datasets/gsm8k/gsm8k_gen.py
+    configpath_llmjudge: ''
+- gsm_hard:
+    name: GSM-Hard
+    category: Math
+    paper: https://proceedings.mlr.press/v202/gao23f/gao23f.pdf
+    configpath: opencompass/configs/datasets/gsm_hard/gsmhard_gen.py
+    configpath_llmjudge: ''
+- hle:
+    name: HLE(Humanity's Last Exam)
+    category: Reasoning
+    paper: https://lastexam.ai/paper
+    configpath: opencompass/configs/datasets/HLE/hle_gen.py
+    configpath_llmjudge: ''
+- hellaswag:
+    name: HellaSwag
+    category: Reasoning
+    paper: https://arxiv.org/pdf/1905.07830
+    configpath: opencompass/configs/datasets/hellaswag/hellaswag_gen.py
+    configpath_llmjudge: opencompass/configs/datasets/hellaswag/hellaswag_llm_judge_gen.py
+- humaneval:
+    name: HumanEval
+    category: Code
+    paper: https://arxiv.org/pdf/2107.03374v2
+    configpath: opencompass/configs/datasets/humaneval/humaneval_gen.py
+    configpath_llmjudge: ''
+- humaneval_cn:
+    name: HumanEval-CN
+    category: Code
+    paper: ''
+    configpath: opencompass/configs/datasets/humaneval_cn/humaneval_cn_gen.py
+    configpath_llmjudge: ''
+- humaneval_multi:
+    name: Multi-HumanEval
+    category: Code
+    paper: https://arxiv.org/pdf/2210.14868
+    configpath: opencompass/configs/datasets/humaneval_multi/humaneval_multi_gen.py
+    configpath_llmjudge: ''
+- humaneval_multi:
+    name: HumanEval+
+    category: Code
+    paper: https://arxiv.org/pdf/2305.01210
+    configpath: opencompass/configs/datasets/humaneval_plus/humaneval_plus_gen.py
+    configpath_llmjudge: ''
+- humanevalx:
+    name: HumanEval-X
+    category: Code
+    paper: https://dl.acm.org/doi/pdf/10.1145/3580305.3599790
+    configpath: opencompass/configs/datasets/humanevalx/humanevalx_gen.py
+    configpath_llmjudge: ''
+- humaneval_pro:
+    name: HumanEval Pro
+    category: Code
+    paper: https://arxiv.org/abs/2412.21199
+    configpath: opencompass/configs/datasets/humaneval_pro/humaneval_pro_gen.py
+    configpath_llmjudge: ''
+- hungarian_math:
+    name: Hungarian_Math
+    category: Math
+    paper: https://huggingface.co/datasets/keirp/hungarian_national_hs_finals_exam
+    configpath: opencompass/configs/datasets/hungarian_exam/hungarian_exam_gen.py
+    configpath_llmjudge: ''
+- iwslt2017:
+    name: IWSLT2017
+    category: Language
+    paper: https://cris.fbk.eu/bitstream/11582/312796/1/iwslt17-overview.pdf
+    configpath: opencompass/configs/datasets/iwslt2017/iwslt2017_gen.py
+    configpath_llmjudge: ''
+- jigsawmultilingual:
+    name: JigsawMultilingual
+    category: Safety
+    paper: https://www.kaggle.com/competitions/jigsaw-multilingual-toxic-comment-classification/data
+    configpath: opencompass/configs/datasets/jigsawmultilingual/jigsawmultilingual_clp.py
+    configpath_llmjudge: ''
+- lambada:
+    name: LAMBADA
+    category: Understanding
+    paper: https://arxiv.org/pdf/1606.06031
+    configpath: opencompass/configs/datasets/lambada/lambada_gen.py
+    configpath_llmjudge: ''
+- lcsts:
+    name: LCSTS
+    category: Understanding
+    paper: https://aclanthology.org/D15-1229.pdf
+    configpath: opencompass/configs/datasets/lcsts/lcsts_gen.py
+    configpath_llmjudge: ''
+- livestembench:
+    name: LiveStemBench
+    category: ''
+    paper: ''
+    configpath: opencompass/configs/datasets/livestembench/livestembench_gen.py
+    configpath_llmjudge: ''
+- llm_compression:
+    name: LLM Compression
+    category: Bits Per Character (BPC)
+    paper: https://arxiv.org/pdf/2404.09937
+    configpath: opencompass/configs/datasets/llm_compression/llm_compression.py
+    configpath_llmjudge: ''
+- math:
+    name: MATH
+    category: Math
+    paper: https://arxiv.org/pdf/2103.03874
+    configpath: opencompass/configs/datasets/math/math_gen.py
+    configpath_llmjudge: opencompass/configs/datasets/math/math_llm_judge_gen.py
+- math500:
+    name: MATH500
+    category: Math
+    paper: https://github.com/openai/prm800k
+    configpath: opencompass/configs/datasets/math/math_prm800k_500_gen.py
+    configpath_llmjudge: opencompass/configs/datasets/math/math_prm800k_500_llm_judge_gen.py
+- math401:
+    name: MATH 401
+    category: Math
+    paper: https://arxiv.org/pdf/2304.02015
+    configpath: opencompass/configs/datasets/math401/math401_gen.py
+    configpath_llmjudge: ''
+- mathbench:
+    name: MathBench
+    category: Math
+    paper: https://arxiv.org/pdf/2405.12209
+    configpath: opencompass/configs/datasets/mathbench/mathbench_gen.py
+    configpath_llmjudge: ''
+- mbpp:
+    name: MBPP
+    category: Code
+    paper: https://arxiv.org/pdf/2108.07732
+    configpath: opencompass/configs/datasets/mbpp/mbpp_gen.py
+    configpath_llmjudge: ''
+- mbpp_cn:
+    name: MBPP-CN
+    category: Code
+    paper: ''
+    configpath: opencompass/configs/datasets/mbpp_cn/mbpp_cn_gen.py
+    configpath_llmjudge: ''
+- mbpp_plus:
+    name: MBPP-PLUS
+    category: Code
+    paper: ''
+    configpath: opencompass/configs/datasets/mbpp_plus/mbpp_plus_gen.py
+    configpath_llmjudge: ''
+- mbpp_pro:
+    name: MBPP Pro
+    category: Code
+    paper: https://arxiv.org/abs/2412.21199
+    configpath: opencompass/configs/datasets/mbpp_pro/mbpp_pro_gen.py
+    configpath_llmjudge: ''
+- mgsm:
+    name: MGSM
+    category: Language / Math
+    paper: https://arxiv.org/pdf/2210.03057
+    configpath: opencompass/configs/datasets/mgsm/mgsm_gen.py
+    configpath_llmjudge: ''
+- mmlu:
+    name: MMLU
+    category: Understanding
+    paper: https://arxiv.org/pdf/2009.03300
+    configpath: opencompass/configs/datasets/mmlu/mmlu_gen.py
+    configpath_llmjudge: opencompass/configs/datasets/mmlu/mmlu_llm_judge_gen.py
+- SciEval:
+    name: SciEval
+    category: Understanding
+    paper: https://arxiv.org/pdf/2308.13149
+    configpath: opencompass/configs/datasets/SciEval/SciEval_gen.py
+    configpath_llmjudge: opencompass/configs/datasets/SciEval/SciEval_llm_judge_gen.py
+- mmlu_cf:
+    name: MMLU-CF
+    category: Understanding
+    paper: https://arxiv.org/pdf/2412.15194
+    configpath: opencompass/configs/datasets/mmlu_cf/mmlu_cf_gen.py
+    configpath_llmjudge: ''
+- mmlu_pro:
+    name: MMLU-Pro
+    category: Understanding
+    paper: https://arxiv.org/pdf/2406.01574
+    configpath: opencompass/configs/datasets/mmlu_pro/mmlu_pro_gen.py
+    configpath_llmjudge: opencompass/configs/datasets/mmlu_pro/mmlu_pro_llm_judge_gen.py
+- mmmlu:
+    name: MMMLU
+    category: Language / Understanding
+    paper: https://huggingface.co/datasets/openai/MMMLU
+    configpath:
+      - opencompass/configs/datasets/mmmlu/mmmlu_gen.py
+      - opencompass/configs/datasets/mmmlu_lite/mmmlu_lite_gen.py
+    configpath_llmjudge: ''
+- multirc:
+    name: SuperGLUE / MultiRC
+    category: Understanding
+    paper: https://proceedings.neurips.cc/paper_files/paper/2019/file/4496bf24afe7fab6f046bf4923da8de6-Paper.pdf
+    configpath: opencompass/configs/datasets/SuperGLUE_MultiRC/SuperGLUE_MultiRC_gen.py
+    configpath_llmjudge: ''
+- multipl_e:
+    name: MultiPL-E
+    category: Code
+    paper: https://arxiv.org/pdf/2210.14868
+    configpath: opencompass/configs/datasets/multipl_e
+    configpath_llmjudge: ''
+- narrativeqa:
+    name: NarrativeQA
+    category: Understanding
+    paper: https://github.com/google-deepmind/narrativeqa
+    configpath: opencompass/configs/datasets/narrativeqa/narrativeqa_gen.py
+    configpath_llmjudge: ''
+- natural_question:
+    name: NaturalQuestions
+    category: Knowledge
+    paper: https://github.com/google-research-datasets/natural-questions
+    configpath: opencompass/configs/datasets/nq/nq_gen.py
+    configpath_llmjudge: ''
+- natural_question_cn:
+    name: NaturalQuestions-CN
+    category: Knowledge
+    paper: ''
+    configpath: opencompass/configs/datasets/nq_cn/nqcn_gen.py
+    configpath_llmjudge: ''
+- obqa:
+    name: OpenBookQA
+    category: Knowledge
+    paper: https://arxiv.org/pdf/1809.02789v1
+    configpath: opencompass/configs/datasets/obqa/obqa_gen.py
+    configpath_llmjudge: ''
+- olymmath:
+    name: OlymMATH
+    category: Math
+    paper: https://arxiv.org/abs/2503.21380
+    configpath: ''
+    configpath_llmjudge: opencompass/configs/datasets/OlymMATH/olymmath_llm_judeg_gen.py
+- piqa:
+    name: OpenBookQA
+    category: Knowledge / Physics
+    paper: https://arxiv.org/pdf/1911.11641v1
+    configpath: opencompass/configs/datasets/piqa/piqa_gen.py
+    configpath_llmjudge: ''
+- ProteinLMBench:
+    name: ProteinLMBench
+    category: Knowledge / Biology (Protein)
+    paper: https://arxiv.org/abs/2406.05540
+    configpath: opencompass/configs/datasets/ProteinLMBench/ProteinLMBench_gen.py
+    configpath_llmjudge: opencompass/configs/datasets/ProteinLMBench/ProteinLMBench_llmjudge_gen.py
+- py150:
+    name: py150
+    category: Code
+    paper: https://github.com/microsoft/CodeXGLUE/tree/main/Code-Code/CodeCompletion-line
+    configpath: opencompass/configs/datasets/py150/py150_gen.py
+    configpath_llmjudge: ''
+- qasper:
+    name: Qasper
+    category: Long Context
+    paper: https://arxiv.org/pdf/2105.03011
+    configpath: opencompass/configs/datasets/qasper/qasper_gen.py
+    configpath_llmjudge: ''
+- qaspercut:
+    name: Qasper-Cut
+    category: Long Context
+    paper: ''
+    configpath: opencompass/configs/datasets/qaspercut/qaspercut_gen.py
+    configpath_llmjudge: ''
+- race:
+    name: RACE
+    category: Examination
+    paper: https://arxiv.org/pdf/1704.04683
+    configpath: opencompass/configs/datasets/race/race_gen.py
+    configpath_llmjudge: ''
+- rbench:
+    name: R-Bench
+    category: Reasoning
+    paper: https://arxiv.org/pdf/2505.02018
+    configpath: opencompass/configs/datasets/R-Bench/rbench_gen_37cbaf8.py
+    configpath_llmjudge: ''
+- realtoxicprompts:
+    name: RealToxicPrompts
+    category: Safety
+    paper: https://arxiv.org/pdf/2009.11462
+    configpath: opencompass/configs/datasets/realtoxicprompts/realtoxicprompts_gen.py
+    configpath_llmjudge: ''
+- record:
+    name: SuperGLUE / ReCoRD
+    category: Understanding
+    paper: https://proceedings.neurips.cc/paper_files/paper/2019/file/4496bf24afe7fab6f046bf4923da8de6-Paper.pdf
+    configpath: opencompass/configs/datasets/SuperGLUE_ReCoRD/SuperGLUE_ReCoRD_gen.py
+    configpath_llmjudge: ''
+- rte:
+    name: SuperGLUE / RTE
+    category: Reasoning
+    paper: https://proceedings.neurips.cc/paper_files/paper/2019/file/4496bf24afe7fab6f046bf4923da8de6-Paper.pdf
+    configpath: opencompass/configs/datasets/SuperGLUE_RTE/SuperGLUE_RTE_gen.py
+    configpath_llmjudge: ''
+- ocnli:
+    name: CLUE / OCNLI
+    category: Reasoning
+    paper: https://arxiv.org/pdf/2004.05986
+    configpath: opencompass/configs/datasets/CLUE_ocnli/CLUE_ocnli_gen.py
+    configpath_llmjudge: ''
+- ocnlifc:
+    name: FewCLUE / OCNLI-FC
+    category: Reasoning
+    paper: https://arxiv.org/pdf/2107.07498
+    configpath: opencompass/configs/datasets/FewCLUE_ocnli_fc/FewCLUE_ocnli_fc_gen.py
+    configpath_llmjudge: ''
+- rolebench:
+    name: RoleBench
+    category: Role Play
+    paper: https://arxiv.org/pdf/2310.00746
+    configpath: opencompass/configs/datasets/rolebench
+    configpath_llmjudge: ''
+- s3eval:
+    name: S3Eval
+    category: Long Context
+    paper: https://aclanthology.org/2024.naacl-long.69.pdf
+    configpath: opencompass/configs/datasets/s3eval/s3eval_gen.py
+    configpath_llmjudge: ''
+- scibench:
+    name: SciBench
+    category: Reasoning
+    paper: https://sxkdz.github.io/files/publications/ICML/SciBench/SciBench.pdf
+    configpath: opencompass/configs/datasets/scibench/scibench_gen.py
+    configpath_llmjudge: ''
+- scicode:
+    name: SciCode
+    category: Code
+    paper: https://arxiv.org/pdf/2407.13168
+    configpath: opencompass/configs/datasets/scicode/scicode_gen.py
+    configpath_llmjudge: ''
+- seedbench:
+    name: SeedBench
+    category: Knowledge
+    paper: 'https://aclanthology.org/2025.acl-long.1516.pdf'
+    configpath: opencompass/configs/datasets/SeedBench/seedbench_gen.py
+    configpath_llmjudge: ''
+- simpleqa:
+    name: SimpleQA
+    category: Knowledge
+    paper: https://arxiv.org/pdf/2411.04368
+    configpath: opencompass/configs/datasets/SimpleQA/simpleqa_gen.py
+    configpath_llmjudge: ''
+- siqa:
+    name: SocialIQA
+    category: Reasoning
+    paper: https://arxiv.org/pdf/1904.09728
+    configpath: opencompass/configs/datasets/siqa/siqa_gen.py
+    configpath_llmjudge: ''
+- squad20:
+    name: SQuAD2.0
+    category: Understanding
+    paper: https://arxiv.org/pdf/1806.03822
+    configpath: opencompass/configs/datasets/squad20/squad20_gen.py
+    configpath_llmjudge: ''
+- storycloze:
+    name: StoryCloze
+    category: Reasoning
+    paper: https://aclanthology.org/2022.emnlp-main.616.pdf
+    configpath: opencompass/configs/datasets/storycloze/storycloze_gen.py
+    configpath_llmjudge: ''
+- strategyqa:
+    name: StrategyQA
+    category: Reasoning
+    paper: https://arxiv.org/pdf/2101.02235
+    configpath: opencompass/configs/datasets/strategyqa/strategyqa_gen.py
+    configpath_llmjudge: ''
+- summedits:
+    name: SummEdits
+    category: Language
+    paper: https://aclanthology.org/2023.emnlp-main.600.pdf
+    configpath: opencompass/configs/datasets/summedits/summedits_gen.py
+    configpath_llmjudge: ''
+- summscreen:
+    name: SummScreen
+    category: Understanding
+    paper: https://arxiv.org/pdf/2104.07091v1
+    configpath: opencompass/configs/datasets/summscreen/summscreen_gen.py
+    configpath_llmjudge: ''
+- svamp:
+    name: SVAMP
+    category: Math
+    paper: https://aclanthology.org/2021.naacl-main.168.pdf
+    configpath: opencompass/configs/datasets/SVAMP/svamp_gen.py
+    configpath_llmjudge: ''
+- tabmwp:
+    name: TabMWP
+    category: Math / Table
+    paper: https://arxiv.org/pdf/2209.14610
+    configpath: opencompass/configs/datasets/TabMWP/TabMWP_gen.py
+    configpath_llmjudge: ''
+- taco:
+    name: TACO
+    category: Code
+    paper: https://arxiv.org/pdf/2312.14852
+    configpath: opencompass/configs/datasets/taco/taco_gen.py
+    configpath_llmjudge: ''
+- tnews:
+    name: FewCLUE / TNEWS
+    category: Understanding
+    paper: https://arxiv.org/pdf/2107.07498
+    configpath: opencompass/configs/datasets/FewCLUE_tnews/FewCLUE_tnews_gen.py
+    configpath_llmjudge: ''
+- bustm:
+    name: FewCLUE / BUSTM
+    category: Reasoning
+    paper: https://arxiv.org/pdf/2107.07498
+    configpath: opencompass/configs/datasets/FewCLUE_bustm/FewCLUE_bustm_gen.py
+    configpath_llmjudge: ''
+- csl:
+    name: FewCLUE / CSL
+    category: Understanding
+    paper: https://arxiv.org/pdf/2107.07498
+    configpath: opencompass/configs/datasets/FewCLUE_csl/FewCLUE_csl_gen.py
+    configpath_llmjudge: ''
+- ocnli_fc:
+    name: FewCLUE / OCNLI-FC
+    category: Reasoning
+    paper: https://arxiv.org/pdf/2107.07498
+    configpath: opencompass/configs/datasets/FewCLUE_ocnli_fc
+    configpath_llmjudge: ''
+- triviaqa:
+    name: TriviaQA
+    category: Knowledge
+    paper: https://arxiv.org/pdf/1705.03551v2
+    configpath: opencompass/configs/datasets/triviaqa/triviaqa_gen.py
+    configpath_llmjudge: ''
+- triviaqarc:
+    name: TriviaQA-RC
+    category: Knowledge / Understanding
+    paper: ''
+    configpath: opencompass/configs/datasets/triviaqarc/triviaqarc_gen.py
+    configpath_llmjudge: ''
+- truthfulqa:
+    name: TruthfulQA
+    category: Safety
+    paper: https://arxiv.org/pdf/2109.07958v2
+    configpath: opencompass/configs/datasets/truthfulqa/truthfulqa_gen.py
+    configpath_llmjudge: ''
+- tydiqa:
+    name: TyDi-QA
+    category: Language
+    paper: https://storage.googleapis.com/tydiqa/tydiqa.pdf
+    configpath: opencompass/configs/datasets/tydiqa/tydiqa_gen.py
+    configpath_llmjudge: ''
+- wic:
+    name: SuperGLUE / WiC
+    category: Language
+    paper: https://proceedings.neurips.cc/paper_files/paper/2019/file/4496bf24afe7fab6f046bf4923da8de6-Paper.pdf
+    configpath: opencompass/configs/datasets/SuperGLUE_WiC/SuperGLUE_WiC_gen.py
+    configpath_llmjudge: ''
+- wsc:
+    name: SuperGLUE / WSC
+    category: Language / WSC
+    paper: https://proceedings.neurips.cc/paper_files/paper/2019/file/4496bf24afe7fab6f046bf4923da8de6-Paper.pdf
+    configpath: opencompass/configs/datasets/SuperGLUE_WSC/SuperGLUE_WSC_gen.py
+    configpath_llmjudge: ''
+- winogrande:
+    name: WinoGrande
+    category: Language / WSC
+    paper: https://arxiv.org/pdf/1907.10641v2
+    configpath: opencompass/configs/datasets/winogrande/winogrande_gen.py
+    configpath_llmjudge: ''
+- xcopa:
+    name: XCOPA
+    category: Language
+    paper: https://arxiv.org/pdf/2005.00333
+    configpath: opencompass/configs/datasets/XCOPA/XCOPA_ppl.py
+    configpath_llmjudge: ''
+- xiezhi:
+    name: Xiezhi
+    category: Knowledge
+    paper: https://arxiv.org/pdf/2306.05783
+    configpath: opencompass/configs/datasets/xiezhi/xiezhi_gen.py
+    configpath_llmjudge: ''
+- xlsum:
+    name: XLSum
+    category: Understanding
+    paper: https://arxiv.org/pdf/2106.13822v1
+    configpath: opencompass/configs/datasets/XLSum/XLSum_gen.py
+    configpath_llmjudge: ''
+- xsum:
+    name: Xsum
+    category: Understanding
+    paper: https://arxiv.org/pdf/1808.08745
+    configpath: opencompass/configs/datasets/Xsum/Xsum_gen.py
+    configpath_llmjudge: ''
+- cola:
+    name: GLUE / CoLA
+    category: Understanding
+    paper: https://arxiv.org/pdf/1804.07461
+    configpath: opencompass/configs/datasets/GLUE_CoLA/GLUE_CoLA_ppl.py
+    configpath_llmjudge: ''
+- mprc:
+    name: GLUE / MPRC
+    category: Understanding
+    paper: https://arxiv.org/pdf/1804.07461
+    configpath: opencompass/configs/datasets/GLUE_MRPC/GLUE_MRPC_ppl.py
+    configpath_llmjudge: ''
+- qqp:
+    name: GLUE / QQP
+    category: Understanding
+    paper: https://arxiv.org/pdf/1804.07461
+    configpath: opencompass/configs/datasets/GLUE_QQP/GLUE_QQP_ppl.py
+    configpath_llmjudge: ''
+- omni_math:
+    name: Omni-MATH
+    category: Math
+    paper: https://omni-math.github.io/
+    configpath: opencompass/configs/datasets/omni_math/omni_math_gen.py
+    configpath_llmjudge: ''
+- wikibench:
+    name: WikiBench
+    category: Knowledge
+    paper: ''
+    configpath: opencompass/configs/datasets/wikibench/wikibench_gen.py
+    configpath_llmjudge: ''
+- supergpqa:
+    name: SuperGPQA
+    category: Knowledge
+    paper: https://arxiv.org/pdf/2502.14739
+    configpath: opencompass/configs/datasets/supergpqa
+    configpath_llmjudge: ''
+- climaqa:
+    name: ClimaQA
+    category: Science
+    paper: https://arxiv.org/pdf/2410.16701
+    configpath: ''
+    configpath_llmjudge:
+        - opencompass/configs/datasets/ClimaQA/ClimaQA_Gold_llm_judge.py
+        - opencompass/configs/datasets/ClimaQA/ClimaQA_Silver_llm_judge.py
+- physics:
+    name: PHYSICS
+    category: Science
+    paper: https://arxiv.org/pdf/2503.21821
+    configpath: ''
+    configpath_llmjudge: opencompass/configs/datasets/PHYSICS/PHYSICS_llm_judge_gen_a133a2.py
+- smolinstruct:
+    name: SmolInstruct
+    category: Science /Chemistry
+    paper: https://arxiv.org/pdf/2402.09391
+    configpath: opencompass/configs/datasets/SmolInstruct/smolinstruct_gen.py
+    configpath_llmjudge: ''
+- SciKnowEval:
+    name: SciKnowEval
+    category: Science
+    paper: https://arxiv.org/abs/2406.09098
+    configpath: opencompass/configs/datasets/SciKnowEval/SciKnowEval_gen_ebe47d.py
+    configpath_llmjudge: opencompass/configs/datasets/SciKnowEval/SciKnowEval_llmjudge_gen_ebe47d.py
+- internsandbox:
+    name: InternSandbox
+    category: Reasoning/Code/Agent
+    paper: ''
+    configpath: opencompass/configs/datasets/internsandbox/internsandbox_gen_44b982.py
+    configpath_llmjudge: ''
+- nejmaibench:
+    name: nejmaibench
+    category: Science /Medicine
+    paper: https://arxiv.org/pdf/2308.04709
+    configpath: opencompass/configs/datasets/nejm_ai_benchmark/nejmaibench_gen.py
+    configpath_llmjudge: opencompass/configs/datasets/nejm_ai_benchmark/nejmaibench_llmjudge_gen.py
+- medbullets:
+    name: Medbullets
+    category: Science /Medicine
+    paper: https://arxiv.org/pdf/2402.18060
+    configpath: opencompass/configs/datasets/Medbullets/medbullets_gen.py
+    configpath_llmjudge: opencompass/configs/datasets/Medbullets/medbullets_llmjudge_gen.py
+- medmcqa:
+    name: medmcqa
+    category: Science /Medicine
+    paper: https://arxiv.org/pdf/2203.14371
+    configpath: opencompass/configs/datasets/medmcqa/medmcqa_gen.py
+    configpath_llmjudge: opencompass/configs/datasets/medmcqa/medmcqa_llmjudge_gen.py
+- phybench:
+    name: PHYBench
+    category: Science /Physics
+    paper: https://arxiv.org/abs/2504.16074
+    configpath: opencompass/configs/datasets/PHYBench/phybench_gen.py
+    configpath_llmjudge: ''
+- beyondaime:
+    name: BeyondAIME
+    category: Math
+    paper: ''
+    configpath: opencompass/configs/datasets/BeyondAIME/beyondaime_gen.py
+    configpath_llmjudge: ''
+- eese:
+    name: EESE
+    category: Science
+    paper: https://arxiv.org/abs/2507.16514
+    configpath: opencompass/configs/datasets/eese/eese_llm_judge_gen.py
+    configpath_llmjudge: opencompass/configs/datasets/eese/eese_llm_judge_gen.py

examples/eval_PMMEval.py

@@ -0,0 +1,29 @@
+from mmengine.config import read_base
+
+from opencompass.models import HuggingFacewithChatTemplate
+
+with read_base():
+    # from opencompass.configs.datasets.PMMEval.flores_gen import PMMEval_flores_datasets
+    # from opencompass.configs.datasets.PMMEval.humanevalxl_gen import PMMEval_HumanEvalXL_datasets
+    # from opencompass.configs.datasets.PMMEval.mgsm_gen import PMMEval_MGSM_datasets
+    # from opencompass.configs.datasets.PMMEval.mhellaswag_gen import PMMEval_MHellaswag_datasets
+    # from opencompass.configs.datasets.PMMEval.mifeval_gen import PMMEval_MIFEval_datasets
+    # from opencompass.configs.datasets.PMMEval.mlogiqa_gen import PMMEval_MLogiQA_datasets
+    # from opencompass.configs.datasets.PMMEval.mmmlu_gen import PMMEval_MMMLU_datasets
+    # from opencompass.configs.datasets.PMMEval.xnli import PMMEval_XNLI_datasets
+    from opencompass.configs.datasets.PMMEval.pmmeval_gen import \
+        PMMEval_datasets
+    from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_7b_instruct import \
+        models
+    from opencompass.configs.summarizers.PMMEval import summarizer
+
+# datasets = PMMEval_flores_datasets
+# datasets = PMMEval_HumanEvalXL_datasets
+# datasets = PMMEval_MGSM_datasets
+# datasets = PMMEval_MHellaswag_datasets
+# datasets = PMMEval_MIFEval_datasets
+# datasets = PMMEval_MLogiQA_datasets
+# datasets = PMMEval_MMMLU_datasets
+# datasets = PMMEval_XNLI_datasets
+
+datasets = PMMEval_datasets

examples/eval_TheoremQA.py

@@ -0,0 +1,21 @@
+from mmengine.config import read_base
+
+with read_base():
+    from opencompass.configs.datasets.TheoremQA.TheoremQA_5shot_gen_6f0af8 import \
+        TheoremQA_datasets as datasets
+    from opencompass.configs.models.hf_internlm.hf_internlm2_20b import \
+        models as hf_internlm2_20b_model
+    from opencompass.configs.models.hf_internlm.hf_internlm2_math_20b import \
+        models as hf_internlm2_math_20b_model
+    from opencompass.configs.models.mistral.hf_mistral_7b_v0_1 import \
+        models as hf_mistral_7b_v0_1_model
+    from opencompass.configs.models.mistral.hf_mistral_7b_v0_2 import \
+        models as hf_mistral_7b_v0_2_model
+
+models = sum([v for k, v in locals().items() if k.endswith('_model')], [])
+
+work_dir = 'outputs/TheoremQA-5shot'
+
+# dataset    version    metric    mode      mistral-7b-v0.1-hf    mistral-7b-v0.2-hf    internlm2-20b-hf    internlm2-math-20b-hf
+# ---------  ---------  --------  ------  --------------------  --------------------  ------------------  -----------------------
+# TheoremQA  6f0af8     score     gen                    18.00                 16.75               25.87                    30.88

examples/eval_academic_leaderboard_202407.py

@@ -0,0 +1,192 @@
+import os.path as osp
+
+from mmengine.config import read_base
+
+from opencompass.partitioners import NaivePartitioner, NumWorkerPartitioner
+from opencompass.runners import LocalRunner
+from opencompass.tasks import OpenICLEvalTask, OpenICLInferTask
+
+#######################################################################
+#                          PART 0  Essential Configs                  #
+#######################################################################
+with read_base():
+    # Datasets Part
+    ## Core Set
+    # ## Examination
+    # ## Reasoning
+    from opencompass.configs.datasets.bbh.bbh_gen_4a31fa import bbh_datasets
+    from opencompass.configs.datasets.cmmlu.cmmlu_0shot_cot_gen_305931 import \
+        cmmlu_datasets
+    from opencompass.configs.datasets.gpqa.gpqa_openai_simple_evals_gen_5aeece import \
+        gpqa_datasets
+    # ## Coding
+    from opencompass.configs.datasets.humaneval.humaneval_gen_8e312c import \
+        humaneval_datasets
+    # ## Instruction Following
+    from opencompass.configs.datasets.IFEval.IFEval_gen_3321a3 import \
+        ifeval_datasets
+    # ## Math
+    from opencompass.configs.datasets.math.math_0shot_gen_393424 import \
+        math_datasets
+    from opencompass.configs.datasets.mmlu.mmlu_openai_simple_evals_gen_b618ea import \
+        mmlu_datasets
+    from opencompass.configs.datasets.mmlu_pro.mmlu_pro_0shot_cot_gen_08c1de import \
+        mmlu_pro_datasets
+    from opencompass.configs.summarizers.groups.bbh import bbh_summary_groups
+    from opencompass.configs.summarizers.groups.cmmlu import \
+        cmmlu_summary_groups
+    # Summarizer
+    from opencompass.configs.summarizers.groups.mmlu import mmlu_summary_groups
+    from opencompass.configs.summarizers.groups.mmlu_pro import \
+        mmlu_pro_summary_groups
+
+    # Model List
+    # from opencompass.configs.models.qwen.lmdeploy_qwen2_1_5b_instruct import models as lmdeploy_qwen2_1_5b_instruct_model
+    # from opencompass.configs.models.hf_internlm.lmdeploy_internlm2_5_7b_chat import models as hf_internlm2_5_7b_chat_model
+    # from opencompass.configs.models.openbmb.hf_minicpm_2b_sft_bf16 import models as hf_minicpm_2b_sft_bf16_model
+    # from opencompass.configs.models.yi.hf_yi_1_5_6b_chat import models as hf_yi_1_5_6b_chat_model
+    # from opencompass.configs.models.gemma.hf_gemma_2b_it import models as hf_gemma_2b_it_model
+    # from opencompass.configs.models.yi.hf_yi_1_5_34b_chat import models as hf_yi_1_5_34b_chat_model
+
+#######################################################################
+#                          PART 1  Datasets List                      #
+#######################################################################
+# datasets list for evaluation
+datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
+
+#######################################################################
+#                       PART 2  Datset Summarizer                     #
+#######################################################################
+# with read_base():
+
+core_summary_groups = [
+    {
+        'name':
+        'core_average',
+        'subsets': [
+            ['mmlu', 'accuracy'],
+            ['mmlu_pro', 'accuracy'],
+            # ['cmmlu', 'naive_average'],
+            ['cmmlu', 'accuracy'],
+            ['bbh', 'score'],
+            ['math', 'accuracy'],
+            ['openai_humaneval', 'humaneval_pass@1'],
+            ['GPQA_diamond', 'accuracy'],
+            ['IFEval', 'Prompt-level-strict-accuracy'],
+        ],
+    },
+]
+
+summarizer = dict(
+    dataset_abbrs=[
+        ['core_average', 'naive_average'],
+        ['mmlu', 'accuracy'],
+        ['mmlu_pro', 'accuracy'],
+        ['cmmlu', 'accuracy'],
+        ['bbh', 'score'],
+        ['math', 'accuracy'],
+        ['openai_humaneval', 'humaneval_pass@1'],
+        ['GPQA_diamond', 'accuracy'],
+        ['IFEval', 'Prompt-level-strict-accuracy'],
+        '',
+        ['mmlu', 'accuracy'],
+        ['mmlu-stem', 'accuracy'],
+        ['mmlu-social-science', 'accuracy'],
+        ['mmlu-humanities', 'accuracy'],
+        ['mmlu-other', 'accuracy'],
+        '',
+        ['mmlu_pro', 'accuracy'],
+        ['mmlu_pro_math', 'accuracy'],
+        ['mmlu_pro_physics', 'accuracy'],
+        ['mmlu_pro_chemistry', 'accuracy'],
+        ['mmlu_pro_law', 'accuracy'],
+        ['mmlu_pro_engineering', 'accuracy'],
+        ['mmlu_pro_other', 'accuracy'],
+        ['mmlu_pro_economics', 'accuracy'],
+        ['mmlu_pro_health', 'accuracy'],
+        ['mmlu_pro_psychology', 'accuracy'],
+        ['mmlu_pro_business', 'accuracy'],
+        ['mmlu_pro_biology', 'accuracy'],
+        ['mmlu_pro_philosophy', 'accuracy'],
+        ['mmlu_pro_computer_science', 'accuracy'],
+        ['mmlu_pro_history', 'accuracy'],
+        '',
+        ['cmmlu', 'accuracy'],
+        ['cmmlu-stem', 'accuracy'],
+        ['cmmlu-social-science', 'accuracy'],
+        ['cmmlu-humanities', 'accuracy'],
+        ['cmmlu-other', 'accuracy'],
+        ['cmmlu-china-specific', 'accuracy'],
+        '',
+        ['bbh', 'extract_rate'],
+        ['math', 'extract_rate'],
+        # ['openai_humaneval', 'extract_rate'],
+        ['GPQA_diamond', 'extract_rate'],
+        # ['IFEval', 'extract_rate'],
+        '',
+        ['mmlu', 'extract_rate'],
+        ['mmlu-stem', 'extract_rate'],
+        ['mmlu-social-science', 'extract_rate'],
+        ['mmlu-humanities', 'extract_rate'],
+        ['mmlu-other', 'extract_rate'],
+        '',
+        ['mmlu_pro', 'extract_rate'],
+        ['mmlu_pro_math', 'extract_rate'],
+        ['mmlu_pro_physics', 'extract_rate'],
+        ['mmlu_pro_chemistry', 'extract_rate'],
+        ['mmlu_pro_law', 'extract_rate'],
+        ['mmlu_pro_engineering', 'extract_rate'],
+        ['mmlu_pro_other', 'extract_rate'],
+        ['mmlu_pro_economics', 'extract_rate'],
+        ['mmlu_pro_health', 'extract_rate'],
+        ['mmlu_pro_psychology', 'extract_rate'],
+        ['mmlu_pro_business', 'extract_rate'],
+        ['mmlu_pro_biology', 'extract_rate'],
+        ['mmlu_pro_philosophy', 'extract_rate'],
+        ['mmlu_pro_computer_science', 'extract_rate'],
+        ['mmlu_pro_history', 'extract_rate'],
+        '',
+        ['cmmlu', 'extract_rate'],
+        ['cmmlu-stem', 'extract_rate'],
+        ['cmmlu-social-science', 'extract_rate'],
+        ['cmmlu-humanities', 'extract_rate'],
+        ['cmmlu-other', 'extract_rate'],
+        ['cmmlu-china-specific', 'extract_rate'],
+    ],
+    summary_groups=sum(
+        [v for k, v in locals().items() if k.endswith('_summary_groups')], []),
+)
+
+#######################################################################
+#                        PART 3  Models  List                         #
+#######################################################################
+
+models = sum([v for k, v in locals().items() if k.endswith('_model')], [])
+
+#######################################################################
+#                 PART 4  Inference/Evaluation Configuaration         #
+#######################################################################
+
+# Local Runner
+infer = dict(
+    partitioner=dict(type=NumWorkerPartitioner, num_worker=8),
+    runner=dict(
+        type=LocalRunner,
+        max_num_workers=16,
+        retry=0,  # Modify if needed
+        task=dict(type=OpenICLInferTask)),
+)
+
+# eval with local runner
+eval = dict(
+    partitioner=dict(type=NaivePartitioner, n=10),
+    runner=dict(type=LocalRunner,
+                max_num_workers=16,
+                task=dict(type=OpenICLEvalTask)),
+)
+
+#######################################################################
+#                      PART 5  Utils Configuaration                   #
+#######################################################################
+base_exp_dir = 'outputs/corebench_v1_9/'
+work_dir = osp.join(base_exp_dir, 'chat_objective')

examples/eval_academic_leaderboard_REALTIME.py

@@ -0,0 +1,131 @@
+# flake8: noqa
+
+from mmengine.config import read_base
+
+from opencompass.partitioners import NaivePartitioner, NumWorkerPartitioner
+from opencompass.runners import LocalRunner
+from opencompass.tasks import OpenICLEvalTask, OpenICLInferTask
+
+
+#######################################################################
+#                          PART 0  Essential Configs                  #
+#######################################################################
+with read_base():
+    # Datasets
+    from opencompass.configs.datasets.aime2025.aime2025_llmjudge_academic import \
+        aime2025_datasets
+    from opencompass.configs.datasets.gpqa.gpqa_cascade_eval_academic import \
+        gpqa_datasets
+    from opencompass.configs.datasets.IFEval.IFEval_gen_353ae7 import \
+        ifeval_datasets
+    from opencompass.configs.datasets.livecodebench.livecodebench_v6_academic import \
+        LCBCodeGeneration_dataset
+    from opencompass.configs.datasets.mmlu_pro.mmlu_pro_0shot_cot_gen_08c1de import \
+        mmlu_pro_datasets
+    from opencompass.configs.datasets.HLE.hle_llmverify_academic import \
+        hle_datasets
+
+    # Summary Groups
+    from opencompass.configs.summarizers.groups.mmlu_pro import \
+        mmlu_pro_summary_groups
+
+    # Models (add your models here)
+    # from opencompass.configs.models.hf_internlm.lmdeploy_internlm2_5_7b_chat import \
+    #     models as hf_internlm2_5_7b_chat_model
+
+#######################################################################
+#                          PART 1  Datasets List                      #
+#######################################################################
+# datasets list for evaluation
+# Only take LCB generation for evaluation
+
+datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')),
+               []) + [LCBCodeGeneration_dataset]
+
+# LLM judge config: using LLM to evaluate predictions
+judge_cfg = dict()
+
+for item in datasets:
+    if 'judge_cfg' in item['eval_cfg']['evaluator']:
+        item['eval_cfg']['evaluator']['judge_cfg'] = judge_cfg
+    if 'llm_evaluator' in item['eval_cfg']['evaluator'].keys() and 'judge_cfg' in item['eval_cfg']['evaluator']['llm_evaluator']:
+        item['eval_cfg']['evaluator']['llm_evaluator']['judge_cfg'] = judge_cfg
+
+
+#######################################################################
+#                       PART 2  Datset Summarizer                     #
+#######################################################################
+
+core_summary_groups = [
+    {
+        'name':
+        'core_average',
+        'subsets': [
+            ['IFEval', 'Prompt-level-strict-accuracy'],
+            ['hle_llmjudge', 'accuracy'],
+            ['aime2025_repeat_32', 'accuracy (32 runs average)'],
+            ['GPQA_diamond_repeat_4', 'accuracy (4 runs average)'],
+            ['mmlu_pro', 'naive_average'],
+            ['lcb_code_generation_repeat_6', 'pass@1 (6 runs average)'],
+        ],
+    },
+]
+
+summarizer = dict(
+    dataset_abbrs=[
+        ['core_average', 'naive_average'],
+        '',
+        'Instruction Following',
+        ['IFEval', 'Prompt-level-strict-accuracy'],
+        '',
+        'General Reasoning',
+        ['hle_llmjudge', 'accuracy'],
+        ['GPQA_diamond_repeat_4', 'accuracy (4 runs average)'],
+        '',
+        'Math Calculation',
+        ['aime2025_repeat_32', 'accuracy (32 runs average)'],
+        '',
+        'Knowledge',
+        ['mmlu_pro', 'naive_average'],
+        '',
+        'Code',
+        ['lcb_code_generation_repeat_6', 'pass@1 (6 runs average)'],
+    ],
+    summary_groups=sum(
+        [v for k, v in locals().items() if k.endswith('_summary_groups')], []),
+)
+
+#######################################################################
+#                        PART 3  Models  List                         #
+#######################################################################
+
+models = sum([v for k, v in locals().items() if k.endswith('_model')], [])
+
+#######################################################################
+#                 PART 4  Inference/Evaluation Configuaration         #
+#######################################################################
+
+# infer with local runner
+infer = dict(
+    partitioner=dict(type=NumWorkerPartitioner, num_worker=8),
+    runner=dict(
+        type=LocalRunner,
+        max_num_workers=16,
+        retry=0,  # Modify if needed
+        task=dict(type=OpenICLInferTask),
+    ),
+)
+
+# eval with local runner
+eval = dict(
+    partitioner=dict(type=NaivePartitioner, n=10),
+    runner=dict(type=LocalRunner,
+                max_num_workers=16,
+                task=dict(type=OpenICLEvalTask)),
+)
+
+#######################################################################
+#                      PART 5  Utils Configuaration                   #
+#######################################################################
+
+work_dir = './outputs/oc_academic_202507'

examples/eval_alaya.py

@@ -0,0 +1,15 @@
+from mmengine.config import read_base
+
+with read_base():
+    from opencompass.configs.datasets.agieval.agieval_gen import \
+        agieval_datasets
+    from opencompass.configs.datasets.bbh.bbh_gen import bbh_datasets
+    from opencompass.configs.datasets.ceval.ceval_gen import ceval_datasets
+    from opencompass.configs.datasets.cmmlu.cmmlu_gen import cmmlu_datasets
+    from opencompass.configs.datasets.mmlu.mmlu_gen import mmlu_datasets
+    from opencompass.configs.models.alaya.alaya import models
+
+datasets = [
+    *bbh_datasets, *ceval_datasets, *cmmlu_datasets, *agieval_datasets,
+    *mmlu_datasets
+]

examples/eval_api_demo.py

@@ -0,0 +1,12 @@
+from mmengine.config import read_base
+
+with read_base():
+    from opencompass.configs.datasets.demo.demo_gsm8k_chat_gen import \
+        gsm8k_datasets
+    from opencompass.configs.datasets.demo.demo_math_chat_gen import \
+        math_datasets
+    from opencompass.configs.models.openai.gpt_4o_2024_05_13 import \
+        models as gpt4
+
+datasets = gsm8k_datasets + math_datasets
+models = gpt4

examples/eval_bluelm_32k_lveval.py

@@ -0,0 +1,14 @@
+from mmengine.config import read_base
+
+with read_base():
+    from opencompass.configs.datasets.lveval.lveval import \
+        LVEval_datasets as datasets
+    from opencompass.configs.models.bluelm.hf_bluelm_7b_chat_32k import models
+    from opencompass.configs.summarizers.lveval import summarizer
+
+models[0]['path'] = '/path/to/your/huggingface_models/BlueLM-7B-Chat-32K'
+models[0][
+    'tokenizer_path'] = '/path/to/your/huggingface_models/BlueLM-7B-Chat-32K'
+models[0]['max_seq_len'] = 32768
+models[0]['generation_kwargs'] = dict(do_sample=False)
+models[0]['mode'] = 'mid'  # truncate in the middle

examples/eval_charm_mem.py

@@ -0,0 +1,89 @@
+from mmengine.config import read_base
+from opencompassopencompass.configs.models import OpenAI
+
+from opencompass.partitioners.sub_size import SubjectiveSizePartitioner
+from opencompass.runners import LocalRunner
+from opencompass.summarizers import CharmMemSummarizer
+from opencompass.tasks.subjective_eval import SubjectiveEvalTask
+
+with read_base():
+    from opencompass.configs.datasets.CHARM.charm_memory_gen_bbbd53 import \
+        charm_memory_datasets as datasets
+
+    # ------>>>>>> https://arxiv.org/abs/2403.14112
+    # from opencompass.configs.models.openai.gpt_3_5_turbo_1106 import models as gpt_3_5_turbo_1106_model
+    # from opencompass.configs.models.openai.gpt_4_1106_preview import models as gpt_4_1106_preview_model
+    # from opencompass.configs.models.hf_llama.hf_llama2_7b_chat import models as llama2_7b_chat_model
+    # from opencompass.configs.models.hf_llama.hf_llama2_13b_chat import models as llama2_13b_chat_model
+    # from opencompass.configs.models.hf_llama.hf_llama2_70b_chat import models as llama2_70b_chat_model
+    # from opencompass.configs.models.vicuna.hf_vicuna_7b_v15_16k import models as vicuna_7b_v15_16k_model
+    # from opencompass.configs.models.vicuna.hf_vicuna_13b_v15_16k import models as vicuna_13b_v15_16k_model
+    # from opencompass.configs.models.chatglm.hf_chatglm3_6b_32k import models as chatglm3_6b_32k_model
+    # from opencompass.configs.models.baichuan.hf_baichuan2_7b_chat import models as baichuan2_7b_chat_model  # need torch 2.1
+    # from opencompass.configs.models.baichuan.hf_baichuan2_13b_chat import models as baichuan2_13b_chat_model  # need torch 2.1
+    # from opencompass.configs.models.hf_internlm.hf_internlm2_chat_7b import models as hf_internlm2_chat_7b_model
+    # from opencompass.configs.models.hf_internlm.hf_internlm2_chat_20b import models as hf_internlm2_chat_20b_model
+    # from opencompass.configs.models.yi.hf_yi_6b_chat import models as yi_6b_chat_model
+    # from opencompass.configs.models.yi.hf_yi_34b_chat import models as yi_34b_chat_model
+    # from opencompass.configs.models.deepseek.hf_deepseek_7b_chat import models as deepseek_7b_chat_model
+    # from opencompass.configs.models.deepseek.hf_deepseek_67b_chat import models as deepseek_67b_chat_model
+    # from opencompass.configs.models.qwen.hf_qwen_7b_chat import models as qwen_7b_chat_model
+    # from opencompass.configs.models.qwen.hf_qwen_14b_chat import models as qwen_14b_chat_model
+    # from opencompass.configs.models.qwen.hf_qwen_72b_chat import models as qwen_72b_chat_model
+    # <<<<<<------ https://arxiv.org/abs/2403.14112
+    # from opencompass.configs.models.openai.gpt_3_5_turbo_0125 import models as gpt_3_5_turbo_0125_model
+    # from opencompass.configs.models.openai.gpt_4o_2024_05_13 import models as gpt_4o_2024_05_13_model
+    # from opencompass.configs.models.gemini.gemini_1_5_flash import models as gemini_1_5_flash_model
+    # from opencompass.configs.models.gemini.gemini_1_5_pro import models as gemini_1_5_pro_model
+    # from opencompass.configs.models.hf_llama.lmdeploy_llama3_8b_instruct import models as lmdeploy_llama3_8b_instruct_model
+    # from opencompass.configs.models.hf_llama.lmdeploy_llama3_70b_instruct import models as lmdeploy_llama3_70b_instruct_model
+    # from opencompass.configs.models.hf_internlm.lmdeploy_internlm2_chat_1_8b import models as lmdeploy_internlm2_chat_1_8b_model
+    # from opencompass.configs.models.hf_internlm.lmdeploy_internlm2_chat_7b import models as lmdeploy_internlm2_chat_7b_model
+    # from opencompass.configs.models.hf_internlm.lmdeploy_internlm2_chat_20b import models as lmdeploy_internlm2_chat_20b_model
+    # from opencompass.configs.models.yi.hf_yi_1_5_6b_chat import models as yi_1_5_6b_chat_model
+    # from opencompass.configs.models.yi.hf_yi_1_5_34b_chat import models as yi_1_5_34b_chat_model
+    # from opencompass.configs.models.deepseek.hf_deepseek_v2_chat import models as deepseek_v2_chat_model
+    # from opencompass.configs.models.qwen.hf_qwen1_5_1_8b_chat import models as qwen1_5_1_8b_chat_model
+    # from opencompass.configs.models.qwen.hf_qwen1_5_7b_chat import models as qwen1_5_7b_chat_model
+    # from opencompass.configs.models.qwen.hf_qwen1_5_14b_chat import models as qwen1_5_14b_chat_model
+    # from opencompass.configs.models.qwen.hf_qwen1_5_72b_chat import models as qwen1_5_72b_chat_model
+
+models = sum([v for k, v in locals().items() if k.endswith('_model')], [])
+
+## ------------- JudgeLLM Configuration
+api_meta_template = dict(round=[
+    dict(role='HUMAN', api_role='HUMAN'),
+    dict(role='BOT', api_role='BOT', generate=True),
+])
+judge_models = [
+    dict(
+        abbr='GPT-3.5-turbo-0125',
+        type=OpenAI,
+        path='gpt-3.5-turbo-0125',
+        key='ENV',
+        meta_template=api_meta_template,
+        query_per_second=16,
+        max_out_len=2048,
+        max_seq_len=2048,
+        batch_size=8,
+        temperature=0,
+    )
+]
+
+## ------------- Evaluation Configuration
+eval = dict(
+    partitioner=dict(
+        type=SubjectiveSizePartitioner,
+        max_task_size=1000,
+        mode='singlescore',
+        models=models,
+        judge_models=judge_models,
+    ),
+    runner=dict(type=LocalRunner,
+                max_num_workers=2,
+                task=dict(type=SubjectiveEvalTask)),
+)
+
+summarizer = dict(type=CharmMemSummarizer)
+
+work_dir = './outputs/CHARM_mem/chat/'

examples/eval_code_passk_repeat_dataset.py

@@ -0,0 +1,58 @@
+# This config is used for pass@k evaluation with dataset repetition
+# That model cannot generate multiple response for single input
+from mmengine.config import read_base
+
+from opencompass.models import HuggingFaceCausalLM
+from opencompass.partitioners import SizePartitioner
+from opencompass.runners import LocalRunner
+from opencompass.tasks import OpenICLInferTask
+
+with read_base():
+    from opencompass.configs.datasets.humaneval.humaneval_repeat10_gen_8e312c import \
+        humaneval_datasets
+    from opencompass.configs.datasets.mbpp.deprecated_mbpp_repeat10_gen_1e1056 import \
+        mbpp_datasets
+    from opencompass.configs.datasets.mbpp.deprecated_sanitized_mbpp_repeat10_gen_1e1056 import \
+        sanitized_mbpp_datasets
+
+datasets = []
+datasets += humaneval_datasets
+datasets += mbpp_datasets
+datasets += sanitized_mbpp_datasets
+
+_meta_template = dict(round=[
+    dict(role='HUMAN', begin='<|User|>:', end='\n'),
+    dict(role='BOT', begin='<|Bot|>:', end='<eoa>\n', generate=True),
+], )
+
+models = [
+    dict(
+        abbr='internlm-chat-7b-hf-v11',
+        type=HuggingFaceCausalLM,
+        path='internlm/internlm-chat-7b-v1_1',
+        tokenizer_path='internlm/internlm-chat-7b-v1_1',
+        tokenizer_kwargs=dict(
+            padding_side='left',
+            truncation_side='left',
+            use_fast=False,
+            trust_remote_code=True,
+        ),
+        max_seq_len=2048,
+        meta_template=_meta_template,
+        model_kwargs=dict(trust_remote_code=True, device_map='auto'),
+        generation_kwargs=dict(
+            do_sample=True,
+            top_p=0.95,
+            temperature=0.8,
+        ),
+        run_cfg=dict(num_gpus=1, num_procs=1),
+        batch_size=8,
+    )
+]
+
+infer = dict(
+    partitioner=dict(type=SizePartitioner, max_task_size=600),
+    runner=dict(type=LocalRunner,
+                max_num_workers=16,
+                task=dict(type=OpenICLInferTask)),
+)

examples/eval_codegeex2.py

@@ -0,0 +1,8 @@
+from mmengine.config import read_base
+
+with read_base():
+    from opencompass.configs.datasets.humanevalx.humanevalx_gen import \
+        humanevalx_datasets
+    from opencompass.configs.models.codegeex2.hf_codegeex2_6b import models
+
+datasets = humanevalx_datasets

examples/eval_compassarena_subjectivebench.py

@@ -0,0 +1,103 @@
+from mmengine.config import read_base
+
+with read_base():
+    from opencompass.configs.datasets.subjective.compass_arena_subjective_bench.singleturn.pairwise_judge import compassarena_subjectivebench_singleturn_datasets
+    from opencompass.configs.datasets.subjective.compass_arena_subjective_bench.multiturn.pairwise_judge import compassarena_subjectivebench_multiturn_datasets
+
+    from opencompass.configs.models.hf_internlm.lmdeploy_internlm2_5_7b_chat import models as lmdeploy_internlm2_5_7b_chat
+    from opencompass.configs.models.hf_internlm.lmdeploy_internlm2_5_20b_chat import models as lmdeploy_internlm2_5_20b_chat
+    from opencompass.configs.models.hf_llama.lmdeploy_llama3_1_8b_instruct import models as lmdeploy_llama3_1_8b_instruct
+    from opencompass.configs.models.hf_llama.lmdeploy_llama3_1_70b_instruct import models as lmdeploy_llama3_1_70b_instruct
+    from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_0_5b_instruct import models as lmdeploy_qwen2_5_0_5b_instruct
+    from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_1_5b_instruct import models as lmdeploy_qwen2_5_1_5b_instruct
+    from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_3b_instruct import models as lmdeploy_qwen2_5_3b_instruct
+    from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_7b_instruct import models as lmdeploy_qwen2_5_7b_instruct
+    from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_14b_instruct import models as lmdeploy_qwen2_5_14b_instruct
+    from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_32b_instruct import models as lmdeploy_qwen2_5_32b_instruct
+    from opencompass.configs.models.qwen2_5.lmdeploy_qwen2_5_72b_instruct import models as lmdeploy_qwen2_5_72b_instruct
+    from opencompass.configs.models.qwen.lmdeploy_qwen2_7b_instruct import models as lmdeploy_qwen2_7b_instruct
+
+from opencompass.models import (HuggingFace, HuggingFaceCausalLM,
+                                HuggingFaceChatGLM3, OpenAI,
+                                TurboMindModelwithChatTemplate)
+from opencompass.partitioners import NaivePartitioner, SizePartitioner
+from opencompass.partitioners.sub_naive import SubjectiveNaivePartitioner
+from opencompass.partitioners.sub_num_worker import \
+    SubjectiveNumWorkerPartitioner
+from opencompass.partitioners.sub_size import SubjectiveSizePartitioner
+from opencompass.runners import LocalRunner, SlurmSequentialRunner
+from opencompass.summarizers import DefaultSubjectiveSummarizer
+from opencompass.tasks import OpenICLInferTask
+from opencompass.tasks.subjective_eval import SubjectiveEvalTask
+
+api_meta_template = dict(round=[
+    dict(role='HUMAN', api_role='HUMAN'),
+    dict(role='BOT', api_role='BOT', generate=True),
+])
+
+# -------------Inference Stage ----------------------------------------
+# For subjective evaluation, we often set do sample for models
+# models = [
+#     dict(
+#         type=TurboMindModelwithChatTemplate,
+#         abbr='CompassJudger-1-7B-Instruct',
+#         path='opencompass/CompassJudger-1-7B-Instruct',
+#         engine_config=dict(session_len=16384, max_batch_size=16, tp=1),
+#         gen_config=dict(top_k=1, temperature=1e-6, top_p=0.9, max_new_tokens=2048),
+#         max_seq_len=16384,
+#         max_out_len=2048,
+#         batch_size=16,
+#         run_cfg=dict(num_gpus=1),
+#     )
+# ]
+
+models = [
+    *lmdeploy_qwen2_5_14b_instruct, *lmdeploy_qwen2_5_32b_instruct,
+    *lmdeploy_qwen2_5_7b_instruct, *lmdeploy_qwen2_7b_instruct
+]
+
+datasets = [
+    *compassarena_subjectivebench_singleturn_datasets,
+    *compassarena_subjectivebench_multiturn_datasets
+]  # add datasets you want
+
+infer = dict(
+    partitioner=dict(type=NaivePartitioner),
+    runner=dict(type=LocalRunner,
+                max_num_workers=16,
+                task=dict(type=OpenICLInferTask)),
+)
+# -------------Evalation Stage ----------------------------------------
+
+## ------------- JudgeLLM Configuration
+judge_models = [
+    dict(
+        type=TurboMindModelwithChatTemplate,
+        abbr='CompassJudger-1-32B-Instruct',
+        path='opencompass/CompassJudger-1-32B-Instruct',
+        engine_config=dict(session_len=16384, max_batch_size=16, tp=4),
+        gen_config=dict(top_k=1,
+                        temperature=1e-6,
+                        top_p=0.9,
+                        max_new_tokens=2048),
+        max_seq_len=16384,
+        max_out_len=2048,
+        batch_size=16,
+        run_cfg=dict(num_gpus=4),
+    )
+]
+
+## ------------- Evaluation Configuration
+eval = dict(
+    partitioner=dict(
+        type=SubjectiveNaivePartitioner,
+        models=models,
+        judge_models=judge_models,
+    ),
+    runner=dict(type=LocalRunner,
+                max_num_workers=16,
+                task=dict(type=SubjectiveEvalTask)),
+)
+
+summarizer = dict(type=DefaultSubjectiveSummarizer, )
+work_dir = 'outputs/subjective/'

examples/eval_hf_llama2.py

@@ -0,0 +1,28 @@
+from mmengine.config import read_base
+
+with read_base():
+    from opencompass.configs.datasets.agieval.agieval_mixed_713d14 import \
+        agieval_datasets
+    from opencompass.configs.datasets.gsm8k.gsm8k_gen_3309bd import \
+        gsm8k_datasets
+    from opencompass.configs.datasets.hellaswag.hellaswag_ppl_a6e128 import \
+        hellaswag_datasets
+    from opencompass.configs.datasets.humaneval.deprecated_humaneval_gen_a82cae import \
+        humaneval_datasets
+    from opencompass.configs.datasets.mmlu.mmlu_ppl_ac766d import mmlu_datasets
+    from opencompass.configs.datasets.nq.nq_open_gen_e93f8a import nq_datasets
+    from opencompass.configs.datasets.obqa.obqa_ppl_6aac9e import obqa_datasets
+    from opencompass.configs.datasets.SuperGLUE_BoolQ.SuperGLUE_BoolQ_ppl_314797 import \
+        BoolQ_datasets
+    from opencompass.configs.datasets.triviaqa.triviaqa_wiki_gen_d18bf4 import \
+        triviaqa_datasets
+    from opencompass.configs.datasets.winogrande.winogrande_ll_c5cf57 import \
+        winogrande_datasets
+    from opencompass.configs.models.hf_llama.hf_llama2_7b import models
+    from opencompass.configs.summarizers.example import summarizer
+
+datasets = sum([
+    v
+    for k, v in locals().items() if k.endswith('_datasets') or k == 'datasets'
+], [])
+work_dir = './outputs/llama2/'

opencompass.egg-info/PKG-INFO

@@ -0,0 +1,745 @@
+Metadata-Version: 2.4
+Name: opencompass
+Version: 0.5.1
+Summary: A comprehensive toolkit for large model evaluation
+Home-page: https://github.com/open-compass/opencompass
+Author: OpenCompass Contributors
+Maintainer: OpenCompass Authors
+License: Apache License 2.0
+Keywords: AI,NLP,in-context learning,large language model,evaluation,benchmark,llm
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: Science/Research
+Requires-Python: >=3.8.0
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: absl-py
+Requires-Dist: accelerate>=0.19.0
+Requires-Dist: cpm_kernels
+Requires-Dist: datasets<4.0.0,>=2.12.0
+Requires-Dist: einops>=0.5.0
+Requires-Dist: evaluate>=0.3.0
+Requires-Dist: func_timeout
+Requires-Dist: fuzzywuzzy
+Requires-Dist: gradio-client
+Requires-Dist: h5py
+Requires-Dist: httpx==0.27.2
+Requires-Dist: huggingface_hub
+Requires-Dist: immutabledict
+Requires-Dist: importlib-metadata
+Requires-Dist: jieba
+Requires-Dist: json5
+Requires-Dist: jsonlines
+Requires-Dist: mmengine-lite
+Requires-Dist: nltk>=3.7
+Requires-Dist: numpy<2.0.0,>=1.23.4
+Requires-Dist: openai
+Requires-Dist: OpenCC
+Requires-Dist: opencv-python-headless
+Requires-Dist: pandas<2.0.0
+Requires-Dist: prettytable
+Requires-Dist: protobuf
+Requires-Dist: python-Levenshtein
+Requires-Dist: rank_bm25==0.2.2
+Requires-Dist: rapidfuzz
+Requires-Dist: requests>=2.31.0
+Requires-Dist: retrying
+Requires-Dist: rich
+Requires-Dist: rouge
+Requires-Dist: rouge_chinese
+Requires-Dist: rouge_score
+Requires-Dist: sacrebleu
+Requires-Dist: scikit_learn==1.5.0
+Requires-Dist: seaborn
+Requires-Dist: sentence_transformers
+Requires-Dist: tabulate
+Requires-Dist: tiktoken
+Requires-Dist: timeout_decorator
+Requires-Dist: tokenizers>=0.13.3
+Requires-Dist: torch>=1.13.1
+Requires-Dist: tqdm>=4.64.1
+Requires-Dist: transformers>=4.29.1
+Requires-Dist: tree-sitter==0.21.3
+Requires-Dist: tree_sitter_languages>=1.10.2
+Requires-Dist: typer
+Provides-Extra: lmdeploy
+Requires-Dist: lmdeploy; extra == "lmdeploy"
+Requires-Dist: absl-py; extra == "lmdeploy"
+Requires-Dist: accelerate>=0.19.0; extra == "lmdeploy"
+Requires-Dist: cpm_kernels; extra == "lmdeploy"
+Requires-Dist: datasets<4.0.0,>=2.12.0; extra == "lmdeploy"
+Requires-Dist: einops>=0.5.0; extra == "lmdeploy"
+Requires-Dist: evaluate>=0.3.0; extra == "lmdeploy"
+Requires-Dist: func_timeout; extra == "lmdeploy"
+Requires-Dist: fuzzywuzzy; extra == "lmdeploy"
+Requires-Dist: gradio-client; extra == "lmdeploy"
+Requires-Dist: h5py; extra == "lmdeploy"
+Requires-Dist: httpx==0.27.2; extra == "lmdeploy"
+Requires-Dist: huggingface_hub; extra == "lmdeploy"
+Requires-Dist: immutabledict; extra == "lmdeploy"
+Requires-Dist: importlib-metadata; extra == "lmdeploy"
+Requires-Dist: jieba; extra == "lmdeploy"
+Requires-Dist: json5; extra == "lmdeploy"
+Requires-Dist: jsonlines; extra == "lmdeploy"
+Requires-Dist: mmengine-lite; extra == "lmdeploy"
+Requires-Dist: nltk>=3.7; extra == "lmdeploy"
+Requires-Dist: numpy<2.0.0,>=1.23.4; extra == "lmdeploy"
+Requires-Dist: openai; extra == "lmdeploy"
+Requires-Dist: OpenCC; extra == "lmdeploy"
+Requires-Dist: opencv-python-headless; extra == "lmdeploy"
+Requires-Dist: pandas<2.0.0; extra == "lmdeploy"
+Requires-Dist: prettytable; extra == "lmdeploy"
+Requires-Dist: protobuf; extra == "lmdeploy"
+Requires-Dist: python-Levenshtein; extra == "lmdeploy"
+Requires-Dist: rank_bm25==0.2.2; extra == "lmdeploy"
+Requires-Dist: rapidfuzz; extra == "lmdeploy"
+Requires-Dist: requests>=2.31.0; extra == "lmdeploy"
+Requires-Dist: retrying; extra == "lmdeploy"
+Requires-Dist: rich; extra == "lmdeploy"
+Requires-Dist: rouge; extra == "lmdeploy"
+Requires-Dist: rouge_chinese; extra == "lmdeploy"
+Requires-Dist: rouge_score; extra == "lmdeploy"
+Requires-Dist: sacrebleu; extra == "lmdeploy"
+Requires-Dist: scikit_learn==1.5.0; extra == "lmdeploy"
+Requires-Dist: seaborn; extra == "lmdeploy"
+Requires-Dist: sentence_transformers; extra == "lmdeploy"
+Requires-Dist: tabulate; extra == "lmdeploy"
+Requires-Dist: tiktoken; extra == "lmdeploy"
+Requires-Dist: timeout_decorator; extra == "lmdeploy"
+Requires-Dist: tokenizers>=0.13.3; extra == "lmdeploy"
+Requires-Dist: torch>=1.13.1; extra == "lmdeploy"
+Requires-Dist: tqdm>=4.64.1; extra == "lmdeploy"
+Requires-Dist: transformers>=4.29.1; extra == "lmdeploy"
+Requires-Dist: tree-sitter==0.21.3; extra == "lmdeploy"
+Requires-Dist: tree_sitter_languages>=1.10.2; extra == "lmdeploy"
+Requires-Dist: typer; extra == "lmdeploy"
+Provides-Extra: vllm
+Requires-Dist: vllm; extra == "vllm"
+Requires-Dist: absl-py; extra == "vllm"
+Requires-Dist: accelerate>=0.19.0; extra == "vllm"
+Requires-Dist: cpm_kernels; extra == "vllm"
+Requires-Dist: datasets<4.0.0,>=2.12.0; extra == "vllm"
+Requires-Dist: einops>=0.5.0; extra == "vllm"
+Requires-Dist: evaluate>=0.3.0; extra == "vllm"
+Requires-Dist: func_timeout; extra == "vllm"
+Requires-Dist: fuzzywuzzy; extra == "vllm"
+Requires-Dist: gradio-client; extra == "vllm"
+Requires-Dist: h5py; extra == "vllm"
+Requires-Dist: httpx==0.27.2; extra == "vllm"
+Requires-Dist: huggingface_hub; extra == "vllm"
+Requires-Dist: immutabledict; extra == "vllm"
+Requires-Dist: importlib-metadata; extra == "vllm"
+Requires-Dist: jieba; extra == "vllm"
+Requires-Dist: json5; extra == "vllm"
+Requires-Dist: jsonlines; extra == "vllm"
+Requires-Dist: mmengine-lite; extra == "vllm"
+Requires-Dist: nltk>=3.7; extra == "vllm"
+Requires-Dist: numpy<2.0.0,>=1.23.4; extra == "vllm"
+Requires-Dist: openai; extra == "vllm"
+Requires-Dist: OpenCC; extra == "vllm"
+Requires-Dist: opencv-python-headless; extra == "vllm"
+Requires-Dist: pandas<2.0.0; extra == "vllm"
+Requires-Dist: prettytable; extra == "vllm"
+Requires-Dist: protobuf; extra == "vllm"
+Requires-Dist: python-Levenshtein; extra == "vllm"
+Requires-Dist: rank_bm25==0.2.2; extra == "vllm"
+Requires-Dist: rapidfuzz; extra == "vllm"
+Requires-Dist: requests>=2.31.0; extra == "vllm"
+Requires-Dist: retrying; extra == "vllm"
+Requires-Dist: rich; extra == "vllm"
+Requires-Dist: rouge; extra == "vllm"
+Requires-Dist: rouge_chinese; extra == "vllm"
+Requires-Dist: rouge_score; extra == "vllm"
+Requires-Dist: sacrebleu; extra == "vllm"
+Requires-Dist: scikit_learn==1.5.0; extra == "vllm"
+Requires-Dist: seaborn; extra == "vllm"
+Requires-Dist: sentence_transformers; extra == "vllm"
+Requires-Dist: tabulate; extra == "vllm"
+Requires-Dist: tiktoken; extra == "vllm"
+Requires-Dist: timeout_decorator; extra == "vllm"
+Requires-Dist: tokenizers>=0.13.3; extra == "vllm"
+Requires-Dist: torch>=1.13.1; extra == "vllm"
+Requires-Dist: tqdm>=4.64.1; extra == "vllm"
+Requires-Dist: transformers>=4.29.1; extra == "vllm"
+Requires-Dist: tree-sitter==0.21.3; extra == "vllm"
+Requires-Dist: tree_sitter_languages>=1.10.2; extra == "vllm"
+Requires-Dist: typer; extra == "vllm"
+Provides-Extra: api
+Requires-Dist: anthropic; extra == "api"
+Requires-Dist: dashscope; extra == "api"
+Requires-Dist: openai; extra == "api"
+Requires-Dist: spark_ai_python; extra == "api"
+Requires-Dist: sseclient-py==1.7.2; extra == "api"
+Requires-Dist: tencentcloud-sdk-python; extra == "api"
+Requires-Dist: volcengine; extra == "api"
+Requires-Dist: volcengine-python-sdk; extra == "api"
+Requires-Dist: websocket-client; extra == "api"
+Requires-Dist: zhipuai; extra == "api"
+Requires-Dist: absl-py; extra == "api"
+Requires-Dist: accelerate>=0.19.0; extra == "api"
+Requires-Dist: cpm_kernels; extra == "api"
+Requires-Dist: datasets<4.0.0,>=2.12.0; extra == "api"
+Requires-Dist: einops>=0.5.0; extra == "api"
+Requires-Dist: evaluate>=0.3.0; extra == "api"
+Requires-Dist: func_timeout; extra == "api"
+Requires-Dist: fuzzywuzzy; extra == "api"
+Requires-Dist: gradio-client; extra == "api"
+Requires-Dist: h5py; extra == "api"
+Requires-Dist: httpx==0.27.2; extra == "api"
+Requires-Dist: huggingface_hub; extra == "api"
+Requires-Dist: immutabledict; extra == "api"
+Requires-Dist: importlib-metadata; extra == "api"
+Requires-Dist: jieba; extra == "api"
+Requires-Dist: json5; extra == "api"
+Requires-Dist: jsonlines; extra == "api"
+Requires-Dist: mmengine-lite; extra == "api"
+Requires-Dist: nltk>=3.7; extra == "api"
+Requires-Dist: numpy<2.0.0,>=1.23.4; extra == "api"
+Requires-Dist: openai; extra == "api"
+Requires-Dist: OpenCC; extra == "api"
+Requires-Dist: opencv-python-headless; extra == "api"
+Requires-Dist: pandas<2.0.0; extra == "api"
+Requires-Dist: prettytable; extra == "api"
+Requires-Dist: protobuf; extra == "api"
+Requires-Dist: python-Levenshtein; extra == "api"
+Requires-Dist: rank_bm25==0.2.2; extra == "api"
+Requires-Dist: rapidfuzz; extra == "api"
+Requires-Dist: requests>=2.31.0; extra == "api"
+Requires-Dist: retrying; extra == "api"
+Requires-Dist: rich; extra == "api"
+Requires-Dist: rouge; extra == "api"
+Requires-Dist: rouge_chinese; extra == "api"
+Requires-Dist: rouge_score; extra == "api"
+Requires-Dist: sacrebleu; extra == "api"
+Requires-Dist: scikit_learn==1.5.0; extra == "api"
+Requires-Dist: seaborn; extra == "api"
+Requires-Dist: sentence_transformers; extra == "api"
+Requires-Dist: tabulate; extra == "api"
+Requires-Dist: tiktoken; extra == "api"
+Requires-Dist: timeout_decorator; extra == "api"
+Requires-Dist: tokenizers>=0.13.3; extra == "api"
+Requires-Dist: torch>=1.13.1; extra == "api"
+Requires-Dist: tqdm>=4.64.1; extra == "api"
+Requires-Dist: transformers>=4.29.1; extra == "api"
+Requires-Dist: tree-sitter==0.21.3; extra == "api"
+Requires-Dist: tree_sitter_languages>=1.10.2; extra == "api"
+Requires-Dist: typer; extra == "api"
+Provides-Extra: full
+Requires-Dist: alpaca-eval==0.6; extra == "full"
+Requires-Dist: antlr4-python3-runtime==4.11; extra == "full"
+Requires-Dist: cn2an; extra == "full"
+Requires-Dist: dingo-python==1.5.0; extra == "full"
+Requires-Dist: faiss_gpu==1.7.2; extra == "full"
+Requires-Dist: human-eval; extra == "full"
+Requires-Dist: langdetect; extra == "full"
+Requires-Dist: ltp; extra == "full"
+Requires-Dist: math-verify[antlr4_11_0]; extra == "full"
+Requires-Dist: pyext; extra == "full"
+Requires-Dist: pypinyin; extra == "full"
+Requires-Dist: rdkit; extra == "full"
+Requires-Dist: wonderwords; extra == "full"
+Requires-Dist: absl-py; extra == "full"
+Requires-Dist: accelerate>=0.19.0; extra == "full"
+Requires-Dist: cpm_kernels; extra == "full"
+Requires-Dist: datasets<4.0.0,>=2.12.0; extra == "full"
+Requires-Dist: einops>=0.5.0; extra == "full"
+Requires-Dist: evaluate>=0.3.0; extra == "full"
+Requires-Dist: func_timeout; extra == "full"
+Requires-Dist: fuzzywuzzy; extra == "full"
+Requires-Dist: gradio-client; extra == "full"
+Requires-Dist: h5py; extra == "full"
+Requires-Dist: httpx==0.27.2; extra == "full"
+Requires-Dist: huggingface_hub; extra == "full"
+Requires-Dist: immutabledict; extra == "full"
+Requires-Dist: importlib-metadata; extra == "full"
+Requires-Dist: jieba; extra == "full"
+Requires-Dist: json5; extra == "full"
+Requires-Dist: jsonlines; extra == "full"
+Requires-Dist: mmengine-lite; extra == "full"
+Requires-Dist: nltk>=3.7; extra == "full"
+Requires-Dist: numpy<2.0.0,>=1.23.4; extra == "full"
+Requires-Dist: openai; extra == "full"
+Requires-Dist: OpenCC; extra == "full"
+Requires-Dist: opencv-python-headless; extra == "full"
+Requires-Dist: pandas<2.0.0; extra == "full"
+Requires-Dist: prettytable; extra == "full"
+Requires-Dist: protobuf; extra == "full"
+Requires-Dist: python-Levenshtein; extra == "full"
+Requires-Dist: rank_bm25==0.2.2; extra == "full"
+Requires-Dist: rapidfuzz; extra == "full"
+Requires-Dist: requests>=2.31.0; extra == "full"
+Requires-Dist: retrying; extra == "full"
+Requires-Dist: rich; extra == "full"
+Requires-Dist: rouge; extra == "full"
+Requires-Dist: rouge_chinese; extra == "full"
+Requires-Dist: rouge_score; extra == "full"
+Requires-Dist: sacrebleu; extra == "full"
+Requires-Dist: scikit_learn==1.5.0; extra == "full"
+Requires-Dist: seaborn; extra == "full"
+Requires-Dist: sentence_transformers; extra == "full"
+Requires-Dist: tabulate; extra == "full"
+Requires-Dist: tiktoken; extra == "full"
+Requires-Dist: timeout_decorator; extra == "full"
+Requires-Dist: tokenizers>=0.13.3; extra == "full"
+Requires-Dist: torch>=1.13.1; extra == "full"
+Requires-Dist: tqdm>=4.64.1; extra == "full"
+Requires-Dist: transformers>=4.29.1; extra == "full"
+Requires-Dist: tree-sitter==0.21.3; extra == "full"
+Requires-Dist: tree_sitter_languages>=1.10.2; extra == "full"
+Requires-Dist: typer; extra == "full"
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: maintainer
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+<div align="center">
+  <img src="docs/en/_static/image/logo.svg" width="500px"/>
+  <br />
+  <br />
+
+[![][github-release-shield]][github-release-link]
+[![][github-releasedate-shield]][github-releasedate-link]
+[![][github-contributors-shield]][github-contributors-link]<br>
+[![][github-forks-shield]][github-forks-link]
+[![][github-stars-shield]][github-stars-link]
+[![][github-issues-shield]][github-issues-link]
+[![][github-license-shield]][github-license-link]
+
+<!-- [![PyPI](https://badge.fury.io/py/opencompass.svg)](https://pypi.org/project/opencompass/) -->
+
+[🌐Website](https://opencompass.org.cn/) |
+[📖CompassHub](https://hub.opencompass.org.cn/home) |
+[📊CompassRank](https://rank.opencompass.org.cn/home) |
+[📘Documentation](https://opencompass.readthedocs.io/en/latest/) |
+[🛠️Installation](https://opencompass.readthedocs.io/en/latest/get_started/installation.html) |
+[🤔Reporting Issues](https://github.com/open-compass/opencompass/issues/new/choose)
+
+English | [简体中文](README_zh-CN.md)
+
+[![][github-trending-shield]][github-trending-url]
+
+</div>
+
+<p align="center">
+    👋 join us on <a href="https://discord.gg/KKwfEbFj7U" target="_blank">Discord</a> and <a href="https://r.vansin.top/?r=opencompass" target="_blank">WeChat</a>
+</p>
+
+> \[!IMPORTANT\]
+>
+> **Star Us**, You will receive all release notifications from GitHub without any delay ~ ⭐️
+
+<details>
+  <summary><kbd>Star History</kbd></summary>
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=open-compass%2Fopencompass&theme=dark&type=Date">
+    <img width="100%" src="https://api.star-history.com/svg?repos=open-compass%2Fopencompass&type=Date">
+  </picture>
+</details>
+
+## 🧭	Welcome
+
+to **OpenCompass**!
+
+Just like a compass guides us on our journey, OpenCompass will guide you through the complex landscape of evaluating large language models. With its powerful algorithms and intuitive interface, OpenCompass makes it easy to assess the quality and effectiveness of your NLP models.
+
+🚩🚩🚩 Explore opportunities at OpenCompass! We're currently **hiring full-time researchers/engineers and interns**. If you're passionate about LLM and OpenCompass, don't hesitate to reach out to us via [email](mailto:zhangsongyang@pjlab.org.cn). We'd love to hear from you!
+
+🔥🔥🔥 We are delighted to announce that **the OpenCompass has been recommended by the Meta AI**, click [Get Started](https://ai.meta.com/llama/get-started/#validation) of Llama for more information.
+
+> **Attention**<br />
+> Breaking Change Notice: In version 0.4.0, we are consolidating all AMOTIC configuration files (previously located in ./configs/datasets, ./configs/models, and ./configs/summarizers) into the opencompass package. Users are advised to update their configuration references to reflect this structural change.
+
+## 🚀 What's New <a><img width="35" height="20" src="https://user-images.githubusercontent.com/12782558/212848161-5e783dd6-11e8-4fe0-bbba-39ffb77730be.png"></a>
+
+- **\[2025.07.26\]** OpenCompass now supports Intern-S1 related general and scientific evaluation benchmarks. Please check [Tutorial for Evaluating Intern-S1](https://opencompass.readthedocs.io/en/latest/user_guides/interns1.html) for more details! 🔥🔥🔥
+- **\[2025.04.01\]** OpenCompass now supports `CascadeEvaluator`, a flexible evaluation mechanism that allows multiple evaluators to work in sequence. This enables creating customized evaluation pipelines for complex assessment scenarios. Check out the [documentation](docs/en/advanced_guides/llm_judge.md) for more details! 🔥🔥🔥
+- **\[2025.03.11\]** We have supported evaluation for `SuperGPQA` which is a great benchmark for measuring LLM knowledge ability 🔥🔥🔥
+- **\[2025.02.28\]** We have added a tutorial for `DeepSeek-R1` series model, please check [Evaluating Reasoning Model](docs/en/user_guides/deepseek_r1.md) for more details! 🔥🔥🔥
+- **\[2025.02.15\]** We have added two powerful evaluation tools: `GenericLLMEvaluator` for LLM-as-judge evaluations and `MATHVerifyEvaluator` for mathematical reasoning assessments. Check out the documentation for [LLM Judge](docs/en/advanced_guides/llm_judge.md) and [Math Evaluation](docs/en/advanced_guides/general_math.md) for more details! 🔥🔥🔥
+- **\[2025.01.16\]** We now support the [InternLM3-8B-Instruct](https://huggingface.co/internlm/internlm3-8b-instruct) model which has enhanced performance on reasoning and knowledge-intensive tasks.
+- **\[2024.12.17\]** We have provided the evaluation script for the December [CompassAcademic](examples/eval_academic_leaderboard_202412.py), which allows users to easily reproduce the official evaluation results by configuring it.
+- **\[2024.11.14\]** OpenCompass now offers support for a sophisticated benchmark designed to evaluate complex reasoning skills — [MuSR](https://arxiv.org/pdf/2310.16049). Check out the [demo](examples/eval_musr.py) and give it a spin! 🔥🔥🔥
+- **\[2024.11.14\]** OpenCompass now supports the brand new long-context language model evaluation benchmark — [BABILong](https://arxiv.org/pdf/2406.10149). Have a look at the [demo](examples/eval_babilong.py) and give it a try! 🔥🔥🔥
+- **\[2024.10.14\]** We now support the OpenAI multilingual QA dataset [MMMLU](https://huggingface.co/datasets/openai/MMMLU). Feel free to give it a try! 🔥🔥🔥
+- **\[2024.09.19\]** We now support [Qwen2.5](https://huggingface.co/Qwen)(0.5B to 72B) with multiple backend(huggingface/vllm/lmdeploy). Feel free to give them a try! 🔥🔥🔥
+- **\[2024.09.17\]** We now support OpenAI o1(`o1-mini-2024-09-12` and `o1-preview-2024-09-12`). Feel free to give them a try! 🔥🔥🔥
+- **\[2024.09.05\]** We now support answer extraction through model post-processing to provide a more accurate representation of the model's capabilities. As part of this update, we have integrated [XFinder](https://github.com/IAAR-Shanghai/xFinder) as our first post-processing model. For more detailed information, please refer to the [documentation](opencompass/utils/postprocessors/xfinder/README.md), and give it a try! 🔥🔥🔥
+- **\[2024.08.20\]** OpenCompass now supports the [SciCode](https://github.com/scicode-bench/SciCode): A Research Coding Benchmark Curated by Scientists. 🔥🔥🔥
+- **\[2024.08.16\]** OpenCompass now supports the brand new long-context language model evaluation benchmark — [RULER](https://arxiv.org/pdf/2404.06654). RULER provides an evaluation of long-context including retrieval, multi-hop tracing, aggregation, and question answering through flexible configurations. Check out the [RULER](configs/datasets/ruler/README.md) evaluation config now! 🔥🔥🔥
+- **\[2024.08.09\]** We have released the example data and configuration for the CompassBench-202408, welcome to [CompassBench](https://opencompass.readthedocs.io/zh-cn/latest/advanced_guides/compassbench_intro.html) for more details. 🔥🔥🔥
+- **\[2024.08.01\]** We supported the [Gemma2](https://huggingface.co/collections/google/gemma-2-release-667d6600fd5220e7b967f315) models. Welcome to try! 🔥🔥🔥
+- **\[2024.07.23\]** We supported the [ModelScope](www.modelscope.cn) datasets, you can load them on demand without downloading all the data to your local disk. Welcome to try! 🔥🔥🔥
+- **\[2024.07.17\]** We are excited to announce the release of NeedleBench's [technical report](http://arxiv.org/abs/2407.11963). We invite you to visit our [support documentation](https://opencompass.readthedocs.io/en/latest/advanced_guides/needleinahaystack_eval.html) for detailed evaluation guidelines. 🔥🔥🔥
+- **\[2024.07.04\]** OpenCompass now supports InternLM2.5, which has **outstanding reasoning capability**, **1M Context window and** and **stronger tool use**, you can try the models in [OpenCompass Config](https://github.com/open-compass/opencompass/tree/main/configs/models/hf_internlm) and [InternLM](https://github.com/InternLM/InternLM) .🔥🔥🔥.
+- **\[2024.06.20\]** OpenCompass now supports one-click switching between inference acceleration backends, enhancing the efficiency of the evaluation process. In addition to the default HuggingFace inference backend, it now also supports popular backends [LMDeploy](https://github.com/InternLM/lmdeploy) and [vLLM](https://github.com/vllm-project/vllm). This feature is available via a simple command-line switch and through deployment APIs. For detailed usage, see the [documentation](docs/en/advanced_guides/accelerator_intro.md).🔥🔥🔥.
+
+> [More](docs/en/notes/news.md)
+
+## 📊 Leaderboard
+
+We provide [OpenCompass Leaderboard](https://rank.opencompass.org.cn/home) for the community to rank all public models and API models. If you would like to join the evaluation, please provide the model repository URL or a standard API interface to the email address `opencompass@pjlab.org.cn`.
+
+You can also refer to [Guide to Reproducing CompassAcademic Leaderboard Results](https://opencompass.readthedocs.io/zh-cn/latest/academic.html) to quickly reproduce the leaderboard results.
+
+<p align="right"><a href="#top">🔝Back to top</a></p>
+
+## 🛠️ Installation
+
+Below are the steps for quick installation and datasets preparation.
+
+### 💻 Environment Setup
+
+We highly recommend using conda to manage your python environment.
+
+- #### Create your virtual environment
+
+  ```bash
+  conda create --name opencompass python=3.10 -y
+  conda activate opencompass
+  ```
+
+- #### Install OpenCompass via pip
+
+  ```bash
+    pip install -U opencompass
+
+    ## Full installation (with support for more datasets)
+    # pip install "opencompass[full]"
+
+    ## Environment with model acceleration frameworks
+    ## Manage different acceleration frameworks using virtual environments
+    ## since they usually have dependency conflicts with each other.
+    # pip install "opencompass[lmdeploy]"
+    # pip install "opencompass[vllm]"
+
+    ## API evaluation (i.e. Openai, Qwen)
+    # pip install "opencompass[api]"
+  ```
+
+- #### Install OpenCompass from source
+
+  If you want to use opencompass's latest features, or develop new features, you can also build it from source
+
+  ```bash
+    git clone https://github.com/open-compass/opencompass opencompass
+    cd opencompass
+    pip install -e .
+    # pip install -e ".[full]"
+    # pip install -e ".[vllm]"
+  ```
+
+### 📂 Data Preparation
+
+You can choose one for the following method to prepare datasets.
+
+#### Offline Preparation
+
+You can download and extract the datasets with the following commands:
+
+```bash
+# Download dataset to data/ folder
+wget https://github.com/open-compass/opencompass/releases/download/0.2.2.rc1/OpenCompassData-core-20240207.zip
+unzip OpenCompassData-core-20240207.zip
+```
+
+#### Automatic Download from OpenCompass
+
+We have supported download datasets automatic from the OpenCompass storage server. You can run the evaluation with extra `--dry-run` to download these datasets.
+Currently, the supported datasets are listed in [here](https://github.com/open-compass/opencompass/blob/main/opencompass/utils/datasets_info.py#L259). More datasets will be uploaded recently.
+
+#### (Optional) Automatic Download with ModelScope
+
+Also you can use the [ModelScope](www.modelscope.cn) to load the datasets on demand.
+
+Installation:
+
+```bash
+pip install modelscope[framework]
+export DATASET_SOURCE=ModelScope
+```
+
+Then submit the evaluation task without downloading all the data to your local disk. Available datasets include:
+
+```bash
+humaneval, triviaqa, commonsenseqa, tydiqa, strategyqa, cmmlu, lambada, piqa, ceval, math, LCSTS, Xsum, winogrande, openbookqa, AGIEval, gsm8k, nq, race, siqa, mbpp, mmlu, hellaswag, ARC, BBH, xstory_cloze, summedits, GAOKAO-BENCH, OCNLI, cmnli
+```
+
+Some third-party features, like Humaneval and Llama, may require additional steps to work properly, for detailed steps please refer to the [Installation Guide](https://opencompass.readthedocs.io/en/latest/get_started/installation.html).
+
+<p align="right"><a href="#top">🔝Back to top</a></p>
+
+## 🏗️ ️Evaluation
+
+After ensuring that OpenCompass is installed correctly according to the above steps and the datasets are prepared. Now you can start your first evaluation using OpenCompass!
+
+### Your first evaluation with OpenCompass!
+
+OpenCompass support setting your configs via CLI or a python script. For simple evaluation settings we recommend using CLI, for more complex evaluation, it is suggested using the script way. You can find more example scripts under the configs folder.
+
+```bash
+# CLI
+opencompass --models hf_internlm2_5_1_8b_chat --datasets demo_gsm8k_chat_gen
+
+# Python scripts
+opencompass examples/eval_chat_demo.py
+```
+
+You can find more script examples under [examples](./examples) folder.
+
+### API evaluation
+
+OpenCompass, by its design, does not really discriminate between open-source models and API models. You can evaluate both model types in the same way or even in one settings.
+
+```bash
+export OPENAI_API_KEY="YOUR_OPEN_API_KEY"
+# CLI
+opencompass --models gpt_4o_2024_05_13 --datasets demo_gsm8k_chat_gen
+
+# Python scripts
+opencompass examples/eval_api_demo.py
+
+# You can use o1_mini_2024_09_12/o1_preview_2024_09_12  for o1 models, we set max_completion_tokens=8192 as default.
+```
+
+### Accelerated Evaluation
+
+Additionally, if you want to use an inference backend other than HuggingFace for accelerated evaluation, such as LMDeploy or vLLM, you can do so with the command below. Please ensure that you have installed the necessary packages for the chosen backend and that your model supports accelerated inference with it. For more information, see the documentation on inference acceleration backends [here](docs/en/advanced_guides/accelerator_intro.md). Below is an example using LMDeploy:
+
+```bash
+# CLI
+opencompass --models hf_internlm2_5_1_8b_chat --datasets demo_gsm8k_chat_gen -a lmdeploy
+
+# Python scripts
+opencompass examples/eval_lmdeploy_demo.py
+```
+
+### Supported Models and Datasets
+
+OpenCompass has predefined configurations for many models and datasets. You can list all available model and dataset configurations using the [tools](./docs/en/tools.md#list-configs).
+
+```bash
+# List all configurations
+python tools/list_configs.py
+# List all configurations related to llama and mmlu
+python tools/list_configs.py llama mmlu
+```
+
+#### Supported Models
+
+If the model is not on the list but supported by Huggingface AutoModel class or encapsulation of inference engine based on OpenAI interface (see [docs](https://opencompass.readthedocs.io/en/latest/advanced_guides/new_model.html) for details), you can also evaluate it with OpenCompass. You are welcome to contribute to the maintenance of the OpenCompass supported model and dataset lists.
+
+```bash
+opencompass --datasets demo_gsm8k_chat_gen --hf-type chat --hf-path internlm/internlm2_5-1_8b-chat
+```
+
+#### Supported Datasets
+
+Currently, OpenCompass have provided standard recommended configurations for datasets. Generally, config files ending with `_gen.py` or `_llm_judge_gen.py` will point to the recommended config we provide for this dataset. You can refer to [docs](https://opencompass.readthedocs.io/en/latest/dataset_statistics.html) for more details.
+
+```bash
+# Recommended Evaluation Config based on Rules
+opencompass --datasets aime2024_gen --models hf_internlm2_5_1_8b_chat
+
+# Recommended Evaluation Config based on LLM Judge
+opencompass --datasets aime2024_llmjudge_gen --models hf_internlm2_5_1_8b_chat
+```
+
+If you want to use multiple GPUs to evaluate the model in data parallel, you can use `--max-num-worker`.
+
+```bash
+CUDA_VISIBLE_DEVICES=0,1 opencompass --datasets demo_gsm8k_chat_gen --hf-type chat --hf-path internlm/internlm2_5-1_8b-chat --max-num-worker 2
+```
+
+> \[!TIP\]
+>
+> `--hf-num-gpus` is used for model parallel(huggingface format), `--max-num-worker` is used for data parallel.
+
+> \[!TIP\]
+>
+> configuration with `_ppl` is designed for base model typically.
+> configuration with `_gen` can be used for both base model and chat model.
+
+Through the command line or configuration files, OpenCompass also supports evaluating APIs or custom models, as well as more diversified evaluation strategies. Please read the [Quick Start](https://opencompass.readthedocs.io/en/latest/get_started/quick_start.html) to learn how to run an evaluation task.
+
+<p align="right"><a href="#top">🔝Back to top</a></p>
+
+## 📣 OpenCompass 2.0
+
+We are thrilled to introduce OpenCompass 2.0, an advanced suite featuring three key components: [CompassKit](https://github.com/open-compass), [CompassHub](https://hub.opencompass.org.cn/home), and [CompassRank](https://rank.opencompass.org.cn/home).
+![oc20](https://github.com/tonysy/opencompass/assets/7881589/90dbe1c0-c323-470a-991e-2b37ab5350b2)
+
+**CompassRank** has been significantly enhanced into the leaderboards that now incorporates both open-source benchmarks and proprietary benchmarks. This upgrade allows for a more comprehensive evaluation of models across the industry.
+
+**CompassHub** presents a pioneering benchmark browser interface, designed to simplify and expedite the exploration and utilization of an extensive array of benchmarks for researchers and practitioners alike. To enhance the visibility of your own benchmark within the community, we warmly invite you to contribute it to CompassHub. You may initiate the submission process by clicking [here](https://hub.opencompass.org.cn/dataset-submit).
+
+**CompassKit** is a powerful collection of evaluation toolkits specifically tailored for Large Language Models and Large Vision-language Models. It provides an extensive set of tools to assess and measure the performance of these complex models effectively. Welcome to try our toolkits for in your research and products.
+
+## ✨ Introduction
+
+![image](https://github.com/open-compass/opencompass/assets/22607038/f45fe125-4aed-4f8c-8fe8-df4efb41a8ea)
+
+OpenCompass is a one-stop platform for large model evaluation, aiming to provide a fair, open, and reproducible benchmark for large model evaluation. Its main features include:
+
+- **Comprehensive support for models and datasets**: Pre-support for 20+ HuggingFace and API models, a model evaluation scheme of 70+ datasets with about 400,000 questions, comprehensively evaluating the capabilities of the models in five dimensions.
+
+- **Efficient distributed evaluation**: One line command to implement task division and distributed evaluation, completing the full evaluation of billion-scale models in just a few hours.
+
+- **Diversified evaluation paradigms**: Support for zero-shot, few-shot, and chain-of-thought evaluations, combined with standard or dialogue-type prompt templates, to easily stimulate the maximum performance of various models.
+
+- **Modular design with high extensibility**: Want to add new models or datasets, customize an advanced task division strategy, or even support a new cluster management system? Everything about OpenCompass can be easily expanded!
+
+- **Experiment management and reporting mechanism**: Use config files to fully record each experiment, and support real-time reporting of results.
+
+## 📖 Dataset Support
+
+We have supported a statistical list of all datasets that can be used on this platform in the documentation on the OpenCompass website.
+
+You can quickly find the dataset you need from the list through sorting, filtering, and searching functions.
+
+In addition, we provide a recommended configuration for each dataset, and some datasets also support LLM Judge-based configurations.
+
+Please refer to the dataset statistics chapter of [docs](https://opencompass.readthedocs.io/en/latest/dataset_statistics.html) for details.
+
+<p align="right"><a href="#top">🔝Back to top</a></p>
+
+## 📖 Model Support
+
+<table align="center">
+  <tbody>
+    <tr align="center" valign="bottom">
+      <td>
+        <b>Open-source Models</b>
+      </td>
+      <td>
+        <b>API Models</b>
+      </td>
+      <!-- <td>
+        <b>Custom Models</b>
+      </td> -->
+    </tr>
+    <tr valign="top">
+      <td>
+
+- [Alpaca](https://github.com/tatsu-lab/stanford_alpaca)
+- [Baichuan](https://github.com/baichuan-inc)
+- [BlueLM](https://github.com/vivo-ai-lab/BlueLM)
+- [ChatGLM2](https://github.com/THUDM/ChatGLM2-6B)
+- [ChatGLM3](https://github.com/THUDM/ChatGLM3-6B)
+- [Gemma](https://huggingface.co/google/gemma-7b)
+- [InternLM](https://github.com/InternLM/InternLM)
+- [LLaMA](https://github.com/facebookresearch/llama)
+- [LLaMA3](https://github.com/meta-llama/llama3)
+- [Qwen](https://github.com/QwenLM/Qwen)
+- [TigerBot](https://github.com/TigerResearch/TigerBot)
+- [Vicuna](https://github.com/lm-sys/FastChat)
+- [WizardLM](https://github.com/nlpxucan/WizardLM)
+- [Yi](https://github.com/01-ai/Yi)
+- ……
+
+</td>
+<td>
+
+- OpenAI
+- Gemini
+- Claude
+- ZhipuAI(ChatGLM)
+- Baichuan
+- ByteDance(YunQue)
+- Huawei(PanGu)
+- 360
+- Baidu(ERNIEBot)
+- MiniMax(ABAB-Chat)
+- SenseTime(nova)
+- Xunfei(Spark)
+- ……
+
+</td>
+
+</tr>
+  </tbody>
+</table>
+
+<p align="right"><a href="#top">🔝Back to top</a></p>
+
+## 🔜 Roadmap
+
+- [x] Subjective Evaluation
+  - [x] Release CompassAreana.
+  - [x] Subjective evaluation.
+- [x] Long-context
+  - [x] Long-context evaluation with extensive datasets.
+  - [ ] Long-context leaderboard.
+- [x] Coding
+  - [ ] Coding evaluation leaderboard.
+  - [x] Non-python language evaluation service.
+- [x] Agent
+  - [ ] Support various agent frameworks.
+  - [x] Evaluation of tool use of the LLMs.
+- [x] Robustness
+  - [x] Support various attack methods.
+
+## 👷‍♂️ Contributing
+
+We appreciate all contributions to improving OpenCompass. Please refer to the [contributing guideline](https://opencompass.readthedocs.io/en/latest/notes/contribution_guide.html) for the best practice.
+
+<!-- Copy-paste in your Readme.md file -->
+
+<!-- Made with [OSS Insight](https://ossinsight.io/) -->
+
+<a href="https://github.com/open-compass/opencompass/graphs/contributors" target="_blank">
+  <table>
+    <tr>
+      <th colspan="2">
+        <br><img src="https://contrib.rocks/image?repo=open-compass/opencompass"><br><br>
+      </th>
+    </tr>
+  </table>
+</a>
+
+## 🤝 Acknowledgements
+
+Some code in this project is cited and modified from [OpenICL](https://github.com/Shark-NLP/OpenICL).
+
+Some datasets and prompt implementations are modified from [chain-of-thought-hub](https://github.com/FranxYao/chain-of-thought-hub) and [instruct-eval](https://github.com/declare-lab/instruct-eval).
+
+## 🖊️ Citation
+
+```bibtex
+@misc{2023opencompass,
+    title={OpenCompass: A Universal Evaluation Platform for Foundation Models},
+    author={OpenCompass Contributors},
+    howpublished = {\url{https://github.com/open-compass/opencompass}},
+    year={2023}
+}
+```
+
+<p align="right"><a href="#top">🔝Back to top</a></p>
+
+[github-contributors-link]: https://github.com/open-compass/opencompass/graphs/contributors
+[github-contributors-shield]: https://img.shields.io/github/contributors/open-compass/opencompass?color=c4f042&labelColor=black&style=flat-square
+[github-forks-link]: https://github.com/open-compass/opencompass/network/members
+[github-forks-shield]: https://img.shields.io/github/forks/open-compass/opencompass?color=8ae8ff&labelColor=black&style=flat-square
+[github-issues-link]: https://github.com/open-compass/opencompass/issues
+[github-issues-shield]: https://img.shields.io/github/issues/open-compass/opencompass?color=ff80eb&labelColor=black&style=flat-square
+[github-license-link]: https://github.com/open-compass/opencompass/blob/main/LICENSE
+[github-license-shield]: https://img.shields.io/github/license/open-compass/opencompass?color=white&labelColor=black&style=flat-square
+[github-release-link]: https://github.com/open-compass/opencompass/releases
+[github-release-shield]: https://img.shields.io/github/v/release/open-compass/opencompass?color=369eff&labelColor=black&logo=github&style=flat-square
+[github-releasedate-link]: https://github.com/open-compass/opencompass/releases
+[github-releasedate-shield]: https://img.shields.io/github/release-date/open-compass/opencompass?labelColor=black&style=flat-square
+[github-stars-link]: https://github.com/open-compass/opencompass/stargazers
+[github-stars-shield]: https://img.shields.io/github/stars/open-compass/opencompass?color=ffcb47&labelColor=black&style=flat-square
+[github-trending-shield]: https://trendshift.io/api/badge/repositories/6630
+[github-trending-url]: https://trendshift.io/repositories/6630

opencompass.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

opencompass.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+opencompass = opencompass.cli.main:main

opencompass.egg-info/requires.txt

@@ -0,0 +1,277 @@
+absl-py
+accelerate>=0.19.0
+cpm_kernels
+datasets<4.0.0,>=2.12.0
+einops>=0.5.0
+evaluate>=0.3.0
+func_timeout
+fuzzywuzzy
+gradio-client
+h5py
+httpx==0.27.2
+huggingface_hub
+immutabledict
+importlib-metadata
+jieba
+json5
+jsonlines
+mmengine-lite
+nltk>=3.7
+numpy<2.0.0,>=1.23.4
+openai
+OpenCC
+opencv-python-headless
+pandas<2.0.0
+prettytable
+protobuf
+python-Levenshtein
+rank_bm25==0.2.2
+rapidfuzz
+requests>=2.31.0
+retrying
+rich
+rouge
+rouge_chinese
+rouge_score
+sacrebleu
+scikit_learn==1.5.0
+seaborn
+sentence_transformers
+tabulate
+tiktoken
+timeout_decorator
+tokenizers>=0.13.3
+torch>=1.13.1
+tqdm>=4.64.1
+transformers>=4.29.1
+tree-sitter==0.21.3
+tree_sitter_languages>=1.10.2
+typer
+
+[api]
+anthropic
+dashscope
+openai
+spark_ai_python
+sseclient-py==1.7.2
+tencentcloud-sdk-python
+volcengine
+volcengine-python-sdk
+websocket-client
+zhipuai
+absl-py
+accelerate>=0.19.0
+cpm_kernels
+datasets<4.0.0,>=2.12.0
+einops>=0.5.0
+evaluate>=0.3.0
+func_timeout
+fuzzywuzzy
+gradio-client
+h5py
+httpx==0.27.2
+huggingface_hub
+immutabledict
+importlib-metadata
+jieba
+json5
+jsonlines
+mmengine-lite
+nltk>=3.7
+numpy<2.0.0,>=1.23.4
+OpenCC
+opencv-python-headless
+pandas<2.0.0
+prettytable
+protobuf
+python-Levenshtein
+rank_bm25==0.2.2
+rapidfuzz
+requests>=2.31.0
+retrying
+rich
+rouge
+rouge_chinese
+rouge_score
+sacrebleu
+scikit_learn==1.5.0
+seaborn
+sentence_transformers
+tabulate
+tiktoken
+timeout_decorator
+tokenizers>=0.13.3
+torch>=1.13.1
+tqdm>=4.64.1
+transformers>=4.29.1
+tree-sitter==0.21.3
+tree_sitter_languages>=1.10.2
+typer
+
+[full]
+alpaca-eval==0.6
+antlr4-python3-runtime==4.11
+cn2an
+dingo-python==1.5.0
+faiss_gpu==1.7.2
+human-eval
+langdetect
+ltp
+math-verify[antlr4_11_0]
+pyext
+pypinyin
+rdkit
+wonderwords
+absl-py
+accelerate>=0.19.0
+cpm_kernels
+datasets<4.0.0,>=2.12.0
+einops>=0.5.0
+evaluate>=0.3.0
+func_timeout
+fuzzywuzzy
+gradio-client
+h5py
+httpx==0.27.2
+huggingface_hub
+immutabledict
+importlib-metadata
+jieba
+json5
+jsonlines
+mmengine-lite
+nltk>=3.7
+numpy<2.0.0,>=1.23.4
+openai
+OpenCC
+opencv-python-headless
+pandas<2.0.0
+prettytable
+protobuf
+python-Levenshtein
+rank_bm25==0.2.2
+rapidfuzz
+requests>=2.31.0
+retrying
+rich
+rouge
+rouge_chinese
+rouge_score
+sacrebleu
+scikit_learn==1.5.0
+seaborn
+sentence_transformers
+tabulate
+tiktoken
+timeout_decorator
+tokenizers>=0.13.3
+torch>=1.13.1
+tqdm>=4.64.1
+transformers>=4.29.1
+tree-sitter==0.21.3
+tree_sitter_languages>=1.10.2
+typer
+
+[lmdeploy]
+lmdeploy
+absl-py
+accelerate>=0.19.0
+cpm_kernels
+datasets<4.0.0,>=2.12.0
+einops>=0.5.0
+evaluate>=0.3.0
+func_timeout
+fuzzywuzzy
+gradio-client
+h5py
+httpx==0.27.2
+huggingface_hub
+immutabledict
+importlib-metadata
+jieba
+json5
+jsonlines
+mmengine-lite
+nltk>=3.7
+numpy<2.0.0,>=1.23.4
+openai
+OpenCC
+opencv-python-headless
+pandas<2.0.0
+prettytable
+protobuf
+python-Levenshtein
+rank_bm25==0.2.2
+rapidfuzz
+requests>=2.31.0
+retrying
+rich
+rouge
+rouge_chinese
+rouge_score
+sacrebleu
+scikit_learn==1.5.0
+seaborn
+sentence_transformers
+tabulate
+tiktoken
+timeout_decorator
+tokenizers>=0.13.3
+torch>=1.13.1
+tqdm>=4.64.1
+transformers>=4.29.1
+tree-sitter==0.21.3
+tree_sitter_languages>=1.10.2
+typer
+
+[vllm]
+vllm
+absl-py
+accelerate>=0.19.0
+cpm_kernels
+datasets<4.0.0,>=2.12.0
+einops>=0.5.0
+evaluate>=0.3.0
+func_timeout
+fuzzywuzzy
+gradio-client
+h5py
+httpx==0.27.2
+huggingface_hub
+immutabledict
+importlib-metadata
+jieba
+json5
+jsonlines
+mmengine-lite
+nltk>=3.7
+numpy<2.0.0,>=1.23.4
+openai
+OpenCC
+opencv-python-headless
+pandas<2.0.0
+prettytable
+protobuf
+python-Levenshtein
+rank_bm25==0.2.2
+rapidfuzz
+requests>=2.31.0
+retrying
+rich
+rouge
+rouge_chinese
+rouge_score
+sacrebleu
+scikit_learn==1.5.0
+seaborn
+sentence_transformers
+tabulate
+tiktoken
+timeout_decorator
+tokenizers>=0.13.3
+torch>=1.13.1
+tqdm>=4.64.1
+transformers>=4.29.1
+tree-sitter==0.21.3
+tree_sitter_languages>=1.10.2
+typer

opencompass.egg-info/top_level.txt

@@ -0,0 +1 @@
+opencompass

opencompass/__init__.py

@@ -0,0 +1 @@
+__version__ = '0.5.1'

opencompass/registry.py

@@ -0,0 +1,55 @@
+from typing import Callable, List, Optional, Type, Union
+
+from mmengine.registry import METRICS as MMENGINE_METRICS
+from mmengine.registry import Registry as OriginalRegistry
+
+
+class Registry(OriginalRegistry):
+
+    # override the default force behavior
+    def register_module(
+            self,
+            name: Optional[Union[str, List[str]]] = None,
+            force: bool = True,
+            module: Optional[Type] = None) -> Union[type, Callable]:
+        return super().register_module(name, force, module)
+
+
+PARTITIONERS = Registry('partitioner', locations=['opencompass.partitioners'])
+RUNNERS = Registry('runner', locations=['opencompass.runners'])
+TASKS = Registry('task', locations=['opencompass.tasks'])
+MODELS = Registry('model', locations=['opencompass.models'])
+# TODO: LOAD_DATASET -> DATASETS
+LOAD_DATASET = Registry('load_dataset', locations=['opencompass.datasets'])
+TEXT_POSTPROCESSORS = Registry(
+    'text_postprocessors', locations=['opencompass.utils.text_postprocessors'])
+DICT_POSTPROCESSORS = Registry('dict_postprocessors',
+                               locations=[
+                                   'opencompass.utils.dict_postprocessors',
+                                   'opencompass.datasets.generic'
+                               ])
+
+EVALUATORS = Registry('evaluators', locations=['opencompass.evaluators'])
+
+ICL_INFERENCERS = Registry('icl_inferencers',
+                           locations=['opencompass.openicl.icl_inferencer'])
+ICL_RETRIEVERS = Registry('icl_retrievers',
+                          locations=['opencompass.openicl.icl_retriever'])
+ICL_DATASET_READERS = Registry(
+    'icl_dataset_readers',
+    locations=['opencompass.openicl.icl_dataset_reader'])
+ICL_PROMPT_TEMPLATES = Registry(
+    'icl_prompt_templates',
+    locations=['opencompass.openicl.icl_prompt_template'])
+ICL_EVALUATORS = Registry(
+    'icl_evaluators',
+    locations=['opencompass.openicl.icl_evaluator', 'opencompass.evaluator'])
+METRICS = Registry('metric',
+                   parent=MMENGINE_METRICS,
+                   locations=['opencompass.metrics'])
+TOT_WRAPPER = Registry('tot_wrapper', locations=['opencompass.datasets'])
+
+
+def build_from_cfg(cfg):
+    """A helper function that builds object with MMEngine's new config."""
+    return PARTITIONERS.build(cfg)

requirements.txt

@@ -0,0 +1 @@
+-r requirements/runtime.txt

requirements/agent.txt

@@ -0,0 +1,13 @@
+antlr4-python3-runtime==4.11
+ipykernel
+ipython
+json5
+jupyter
+jupyter_client
+jupytext
+-e git+https://github.com/open-compass/lagent-cibench.git#egg=lagent
+lightgbm==4.1.0
+networkx
+scikit-image
+sympy==1.12
+tensorflow==2.14.0

requirements/api.txt

@@ -0,0 +1,17 @@
+# claude
+anthropic
+# Qwen
+dashscope
+# openai
+openai
+ # xunfei
+spark_ai_python
+sseclient-py==1.7.2
+# tecent
+tencentcloud-sdk-python
+# bytedance
+volcengine
+volcengine-python-sdk
+websocket-client
+# zhipu
+zhipuai

requirements/docs.txt

@@ -0,0 +1,11 @@
+docutils==0.18.1
+modelindex
+myst-parser
+-e git+https://github.com/open-compass/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme
+sphinx==6.1.3
+sphinx-copybutton
+sphinx-design
+sphinx-notfound-page
+sphinx-tabs
+sphinxcontrib-jquery
+tabulate

requirements/extra.txt

@@ -0,0 +1,25 @@
+# Alpaca-eval
+alpaca-eval==0.6
+# OlympiadBench
+antlr4-python3-runtime==4.11
+cn2an
+# Dingo
+dingo-python==1.5.0
+# Icl topk retriever
+faiss_gpu==1.7.2
+# Humaneval, Humaneval X
+-e git+https://github.com/open-compass/human-eval.git#egg=human-eval
+# IFEval
+langdetect
+# Lawbench, leval
+ltp
+# Math
+math-verify[antlr4_11_0]
+# Taco, apps Dataset
+pyext
+# Law Bench
+pypinyin
+# Smolinstruct
+rdkit
+# RULER
+wonderwords

requirements/lmdeploy.txt

@@ -0,0 +1 @@
+lmdeploy

requirements/runtime.txt

@@ -0,0 +1,50 @@
+absl-py
+accelerate>=0.19.0
+cpm_kernels
+datasets>=2.12.0,<4.0.0
+einops>=0.5.0
+evaluate>=0.3.0
+func_timeout
+fuzzywuzzy
+gradio-client
+h5py
+httpx==0.27.2
+huggingface_hub
+immutabledict
+importlib-metadata
+jieba
+json5
+jsonlines
+mmengine-lite
+nltk>=3.7
+numpy>=1.23.4,<2.0.0
+openai
+OpenCC
+opencv-python-headless
+pandas<2.0.0
+prettytable
+protobuf
+# pyext
+python-Levenshtein
+rank_bm25==0.2.2
+rapidfuzz
+requests>=2.31.0
+retrying
+rich
+rouge
+-e git+https://github.com/Isaac-JL-Chen/rouge_chinese.git@master#egg=rouge_chinese
+rouge_score
+sacrebleu
+scikit_learn==1.5.0
+seaborn
+sentence_transformers
+tabulate
+tiktoken
+timeout_decorator
+tokenizers>=0.13.3
+torch>=1.13.1
+tqdm>=4.64.1
+transformers>=4.29.1
+tree-sitter==0.21.3
+tree_sitter_languages>=1.10.2
+typer

run.py

@@ -0,0 +1,4 @@
+from opencompass.cli.main import main
+
+if __name__ == '__main__':
+    main()

setup.py

@@ -0,0 +1,164 @@
+from setuptools import find_packages, setup
+from setuptools.command.install import install
+
+
+class DownloadNLTK(install):
+
+    def run(self):
+        self.do_egg_install()
+        import nltk
+
+        nltk.download('punkt')
+
+
+def readme():
+    with open('README.md', encoding='utf-8') as f:
+        content = f.read()
+    return content
+
+
+def parse_requirements(fname='requirements.txt', with_version=True):
+    """Parse the package dependencies listed in a requirements file but strips
+    specific versioning information.
+
+    Args:
+        fname (str): path to requirements file
+        with_version (bool, default=False): if True include version specs
+
+    Returns:
+        List[str]: list of requirements items
+
+    CommandLine:
+        python -c "import setup; print(setup.parse_requirements())"
+    """
+    import re
+    import sys
+    from os.path import exists
+
+    require_fpath = fname
+
+    def parse_line(line):
+        """Parse information from a line in a requirements text file."""
+        if line.startswith('-r '):
+            # Allow specifying requirements in other files
+            target = line.split(' ')[1]
+            for info in parse_require_file(target):
+                yield info
+        else:
+            info = {'line': line}
+            if line.startswith('-e '):
+                info['package'] = line.split('#egg=')[1]
+            else:
+                # Remove versioning from the package
+                pat = '(' + '|'.join(['>=', '==', '>']) + ')'
+                parts = re.split(pat, line, maxsplit=1)
+                parts = [p.strip() for p in parts]
+
+                info['package'] = parts[0]
+                if len(parts) > 1:
+                    op, rest = parts[1:]
+                    if ';' in rest:
+                        # Handle platform specific dependencies
+                        # http://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-platform-specific-dependencies
+                        version, platform_deps = map(str.strip,
+                                                     rest.split(';'))
+                        info['platform_deps'] = platform_deps
+                    else:
+                        version = rest  # NOQA
+                    if '--' in version:
+                        # the `extras_require` doesn't accept options.
+                        version = version.split('--')[0].strip()
+                    info['version'] = (op, version)
+            yield info
+
+    def parse_require_file(fpath):
+        with open(fpath, 'r') as f:
+            for line in f.readlines():
+                line = line.strip()
+                if line and not line.startswith('#'):
+                    for info in parse_line(line):
+                        yield info
+
+    def gen_packages_items():
+        if exists(require_fpath):
+            for info in parse_require_file(require_fpath):
+                parts = [info['package']]
+                if with_version and 'version' in info:
+                    parts.extend(info['version'])
+                if not sys.version.startswith('3.4'):
+                    # apparently package_deps are broken in 3.4
+                    platform_deps = info.get('platform_deps')
+                    if platform_deps is not None:
+                        parts.append(';' + platform_deps)
+                item = ''.join(parts)
+                yield item
+
+    packages = list(gen_packages_items())
+    return packages
+
+
+def get_version():
+    version_file = 'opencompass/__init__.py'
+    with open(version_file, 'r', encoding='utf-8') as f:
+        exec(compile(f.read(), version_file, 'exec'))
+    return locals()['__version__']
+
+
+def do_setup():
+    setup(
+        name='opencompass',
+        author='OpenCompass Contributors',
+        version=get_version(),
+        description='A comprehensive toolkit for large model evaluation',
+        url='https://github.com/open-compass/opencompass',
+        long_description=readme(),
+        long_description_content_type='text/markdown',
+        maintainer='OpenCompass Authors',
+        cmdclass={'download_nltk': DownloadNLTK},
+        setup_requires=['nltk==3.8'],
+        python_requires='>=3.8.0',
+        install_requires=parse_requirements('requirements/runtime.txt'),
+        extras_require={
+            'lmdeploy':
+            parse_requirements('requirements/lmdeploy.txt') +
+            parse_requirements('requirements/runtime.txt'),
+            'vllm':
+            parse_requirements('requirements/vllm.txt') +
+            parse_requirements('requirements/runtime.txt'),
+            'api':
+            parse_requirements('requirements/api.txt') +
+            parse_requirements('requirements/runtime.txt'),
+            'full':
+            parse_requirements('requirements/extra.txt') +
+            parse_requirements('requirements/runtime.txt'),
+        },
+        license='Apache License 2.0',
+        include_package_data=True,
+        packages=find_packages(),
+        keywords=[
+            'AI',
+            'NLP',
+            'in-context learning',
+            'large language model',
+            'evaluation',
+            'benchmark',
+            'llm',
+        ],
+        classifiers=[
+            'Programming Language :: Python :: 3.8',
+            'Programming Language :: Python :: 3.9',
+            'Programming Language :: Python :: 3.10',
+            'Intended Audience :: Developers',
+            'Intended Audience :: Education',
+            'Intended Audience :: Science/Research',
+        ],
+        entry_points={
+            'console_scripts': [
+                'opencompass = opencompass.cli.main:main',
+            ],
+        },
+    )
+
+
+if __name__ == '__main__':
+    do_setup()

test.py

@@ -0,0 +1,49 @@
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer,AutoConfig
+
+def load_tokenizer(model_name: str, is_hf: bool=False):
+    if not is_hf:
+        tokenizer = AutoTokenizer.from_pretrained("gpt2")
+        tokenizer.model_max_length = 2048
+    else:
+        if "mamba" in model_name or "mpt" in model_name:
+            tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
+        else:
+            tokenizer = AutoTokenizer.from_pretrained(model_name)
+    tokenizer.pad_token = tokenizer.eos_token
+    tokenizer.pad_token_id = tokenizer.eos_token_id
+    return tokenizer
+
+from fla.models import DeltaNetConfig, DeltaNetForCausalLM, DeltaNetModel
+print(DeltaNetConfig.model_type)
+AutoConfig.register("delta_net",DeltaNetConfig)
+AutoModelForCausalLM.register(DeltaNetConfig,DeltaNetForCausalLM)
+
+from opencompass.models.fla2.models import mask_deltanetConfig,mask_deltanetForCausalLM
+print(mask_deltanetConfig.model_type)
+AutoConfig.register("mask_deltanet",mask_deltanetConfig)
+AutoModelForCausalLM.register(mask_deltanetConfig,mask_deltanetForCausalLM)
+# model_path = "/mnt/jfzn/msj/delta_net-1.3B-100B"
+model_path =  "/mnt/jfzn/msj/train_exp/mask_deltanet_1B_rank4"
+# 注意：DeltaNet 必须开 trust_remote_code!!!
+# tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
+
+model = AutoModelForCausalLM.from_pretrained(
+    model_path,
+    torch_dtype=torch.bfloat16,   # 推荐 fp16，加速且稳定
+    trust_remote_code=True,      # <<=== 你的代码缺了它！
+    device_map="cuda",
+)
+tokenizer = load_tokenizer(model_path, is_hf=True)
+prompt = "What is the official language of China?"
+inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
+
+outputs = model.generate(
+    **inputs,
+    max_new_tokens=100,
+    do_sample=False,
+    pad_token_id=tokenizer.eos_token_id,
+    eos_token_id=tokenizer.eos_token_id,
+)
+
+print(tokenizer.decode(outputs[0], skip_special_tokens=True))

tmp/06837656-e189-4cad-8d53-f175cf579961_params.py

@@ -0,0 +1,1424 @@
+datasets = [
+    [
+        dict(
+            abbr='LongBench_2wikimqa_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='2wikimqa',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBench2wikimqaDataset'),
+        dict(
+            abbr='LongBench_hotpotqa_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='hotpotqa',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchhotpotqaDataset'),
+        dict(
+            abbr='LongBench_musique_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='musique',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchmusiqueDataset'),
+        dict(
+            abbr='LongBench_multifieldqa_en_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Read the following text and answer briefly.\n\n{context}\n\nNow, answer the following question based on the above text, only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='multifieldqa_en',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[114:133]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchmultifieldqa_enDataset'),
+        dict(
+            abbr='LongBench_multifieldqa_zh_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    language='zh',
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            '阅读以下文字并用中文简短回答：\n\n{context}\n\n现在请基于上面的文章回答下面的问题，只告诉我答案，不要输出任何其他字词。\n\n问题：{input}\n回答：',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='multifieldqa_zh',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchmultifieldqa_zhDataset'),
+        dict(
+            abbr='LongBench_narrativeqa_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=128,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'You are given a story, which can be either a novel or a movie script, and a question. Answer the question as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nStory: {context}\n\nNow, answer the question based on the story as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nQuestion: {input}\n\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='narrativeqa',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchnarrativeqaDataset'),
+        dict(
+            abbr='LongBench_qasper_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='qasper',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchqasperDataset'),
+        dict(
+            abbr='LongBench_triviaqa_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_postprocessor=dict(
+                    type='opencompass.datasets.triviaqa_postprocess'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Answer the question based on the given passage. Only give me the answer and do not output any other words. The following are some examples.\n\n{context}\n\n{input}',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='triviaqa',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchtriviaqaDataset'),
+        dict(
+            abbr='LongBench_gov_report_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchRougeEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=512,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'You are given a report by a government agency. Write a one-page summary of the report.\n\nReport:\n{context}\n\nNow, write a one-page summary of the report.\n\nSummary:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='gov_report',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchgov_reportDataset'),
+        dict(
+            abbr='LongBench_qmsum_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchRougeEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=512,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'You are given a meeting transcript and a query containing a question or instruction. Answer the query in one or more sentences.\n\nTranscript:\n{context}\n\nNow, answer the query based on the above meeting transcript in one or more sentences.\n\nQuery: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='qmsum',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchqmsumDataset'),
+        dict(
+            abbr='LongBench_vcsum_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    language='zh',
+                    type='opencompass.datasets.LongBenchRougeEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=512,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            '下面有一段会议记录，请你阅读后，写一段总结，总结会议的内容。\n会议记录：\n{context}\n\n会议总结：',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='vcsum',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchvcsumDataset'),
+        dict(
+            abbr='LongBench_dureader_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    language='zh',
+                    type='opencompass.datasets.LongBenchRougeEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=128,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            '请基于给定的文章回答下述问题。\n\n文章：{context}\n\n请基于上述文章回答下面的问题。\n\n问题：{input}\n回答：',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='dureader',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchdureaderDataset'),
+        dict(
+            abbr='LongBench_lcc_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchCodeSimEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Please complete the code given below. \n{context}Next line of code:\n',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='lcc',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                ],
+                output_column='answers',
+                test_range='[378:441]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchlccDataset'),
+        dict(
+            abbr='LongBench_repobench-p_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchCodeSimEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Please complete the code given below. \n{context}{input}Next line of code:\n',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='repobench-p',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[378:441]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchrepobenchDataset'),
+        dict(
+            abbr='LongBench_passage_retrieval_en_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchRetrievalEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Here are 30 paragraphs from Wikipedia, along with an abstract. Please determine which paragraph the abstract is from.\n\n{context}\n\nThe following is an abstract.\n\n{input}\n\nPlease enter the number of the paragraph that the abstract is from. The answer format must be like "Paragraph 1", "Paragraph 2", etc.\n\nThe answer is: ',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='passage_retrieval_en',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchpassage_retrieval_enDataset'),
+        dict(
+            abbr='LongBench_passage_retrieval_zh_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    language='zh',
+                    type='opencompass.datasets.LongBenchRetrievalEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            '以下是若干段落文字，以及其中一个段落的摘要。请确定给定的摘要出自哪一段。\n\n{context}\n\n下面是一个摘要\n\n{input}\n\n请输入摘要所属段落的编号。答案格式必须是"段落1"，"段落2"等格式\n\n答案是：',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='passage_retrieval_zh',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchpassage_retrieval_zhDataset'),
+        dict(
+            abbr='LongBench_passage_count_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchCountEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'There are some paragraphs below sourced from Wikipedia. Some of them may be duplicates. Please carefully read these paragraphs and determine how many unique paragraphs there are after removing duplicates. In other words, how many non-repeating paragraphs are there in total?\n\n{context}\n\nPlease enter the final count of unique paragraphs after removing duplicates. The output format should only contain the number, such as 1, 2, 3, and so on.\n\nThe final answer is: ',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='passage_count',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchpassage_countDataset'),
+        dict(
+            abbr='LongBench_trec_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchClassificationEvaluator'
+                ),
+                pred_postprocessor=dict(
+                    type='opencompass.datasets.trec_postprocess'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Please determine the type of the question below. Here are some examples of questions.\n\n{context}\n{input}',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='trec',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='all_labels',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchtrecDataset'),
+        dict(
+            abbr='LongBench_lsht_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchClassificationEvaluator'
+                ),
+                pred_postprocessor=dict(
+                    type='opencompass.datasets.lsht_postprocess'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt='请判断给定新闻的类别，下面是一些例子。\n\n{context}\n{input}',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='lsht',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='all_labels',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchlshtDataset'),
+        dict(
+            abbr='LongBench_multi_news_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchRougeEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=512,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'You are given several news passages. Write a one-page summary of all news. \n\nNews:\n{context}\n\nNow, write a one-page summary of all the news.\n\nSummary:\n',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='multi_news',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchmulti_newsDataset'),
+        dict(
+            abbr='LongBench_samsum_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchRougeEvaluator'),
+                pred_postprocessor=dict(
+                    type='opencompass.datasets.samsum_postprocess'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=128,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Summarize the dialogue into a few short sentences. The following are some examples.\n\n{context}\n\n{input}',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='samsum',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchsamsumDataset'),
+        dict(
+            abbr='LongBench_2wikimqa_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='2wikimqa',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBench2wikimqaDataset'),
+        dict(
+            abbr='LongBench_hotpotqa_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='hotpotqa',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchhotpotqaDataset'),
+        dict(
+            abbr='LongBench_musique_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='musique',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchmusiqueDataset'),
+        dict(
+            abbr='LongBench_multifieldqa_en_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Read the following text and answer briefly.\n\n{context}\n\nNow, answer the following question based on the above text, only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='multifieldqa_en',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[114:133]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchmultifieldqa_enDataset'),
+        dict(
+            abbr='LongBench_multifieldqa_zh_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    language='zh',
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            '阅读以下文字并用中文简短回答：\n\n{context}\n\n现在请基于上面的文章回答下面的问题，只告诉我答案，不要输出任何其他字词。\n\n问题：{input}\n回答：',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='multifieldqa_zh',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchmultifieldqa_zhDataset'),
+        dict(
+            abbr='LongBench_narrativeqa_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=128,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'You are given a story, which can be either a novel or a movie script, and a question. Answer the question as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nStory: {context}\n\nNow, answer the question based on the story as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nQuestion: {input}\n\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='narrativeqa',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchnarrativeqaDataset'),
+        dict(
+            abbr='LongBench_qasper_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='qasper',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchqasperDataset'),
+        dict(
+            abbr='LongBench_triviaqa_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_postprocessor=dict(
+                    type='opencompass.datasets.triviaqa_postprocess'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Answer the question based on the given passage. Only give me the answer and do not output any other words. The following are some examples.\n\n{context}\n\n{input}',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='triviaqa',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchtriviaqaDataset'),
+        dict(
+            abbr='LongBench_gov_report_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchRougeEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=512,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'You are given a report by a government agency. Write a one-page summary of the report.\n\nReport:\n{context}\n\nNow, write a one-page summary of the report.\n\nSummary:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='gov_report',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchgov_reportDataset'),
+        dict(
+            abbr='LongBench_qmsum_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchRougeEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=512,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'You are given a meeting transcript and a query containing a question or instruction. Answer the query in one or more sentences.\n\nTranscript:\n{context}\n\nNow, answer the query based on the above meeting transcript in one or more sentences.\n\nQuery: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='qmsum',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchqmsumDataset'),
+        dict(
+            abbr='LongBench_vcsum_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    language='zh',
+                    type='opencompass.datasets.LongBenchRougeEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=512,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            '下面有一段会议记录，请你阅读后，写一段总结，总结会议的内容。\n会议记录：\n{context}\n\n会议总结：',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='vcsum',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchvcsumDataset'),
+        dict(
+            abbr='LongBench_dureader_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    language='zh',
+                    type='opencompass.datasets.LongBenchRougeEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=128,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            '请基于给定的文章回答下述问题。\n\n文章：{context}\n\n请基于上述文章回答下面的问题。\n\n问题：{input}\n回答：',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='dureader',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchdureaderDataset'),
+        dict(
+            abbr='LongBench_lcc_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchCodeSimEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Please complete the code given below. \n{context}Next line of code:\n',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='lcc',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                ],
+                output_column='answers',
+                test_range='[378:441]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchlccDataset'),
+        dict(
+            abbr='LongBench_repobench-p_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchCodeSimEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Please complete the code given below. \n{context}{input}Next line of code:\n',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='repobench-p',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[378:441]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchrepobenchDataset'),
+        dict(
+            abbr='LongBench_passage_retrieval_en_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchRetrievalEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Here are 30 paragraphs from Wikipedia, along with an abstract. Please determine which paragraph the abstract is from.\n\n{context}\n\nThe following is an abstract.\n\n{input}\n\nPlease enter the number of the paragraph that the abstract is from. The answer format must be like "Paragraph 1", "Paragraph 2", etc.\n\nThe answer is: ',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='passage_retrieval_en',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchpassage_retrieval_enDataset'),
+        dict(
+            abbr='LongBench_passage_retrieval_zh_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    language='zh',
+                    type='opencompass.datasets.LongBenchRetrievalEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            '以下是若干段落文字，以及其中一个段落的摘要。请确定给定的摘要出自哪一段。\n\n{context}\n\n下面是一个摘要\n\n{input}\n\n请输入摘要所属段落的编号。答案格式必须是"段落1"，"段落2"等格式\n\n答案是：',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='passage_retrieval_zh',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchpassage_retrieval_zhDataset'),
+        dict(
+            abbr='LongBench_passage_count_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchCountEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'There are some paragraphs below sourced from Wikipedia. Some of them may be duplicates. Please carefully read these paragraphs and determine how many unique paragraphs there are after removing duplicates. In other words, how many non-repeating paragraphs are there in total?\n\n{context}\n\nPlease enter the final count of unique paragraphs after removing duplicates. The output format should only contain the number, such as 1, 2, 3, and so on.\n\nThe final answer is: ',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='passage_count',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchpassage_countDataset'),
+        dict(
+            abbr='LongBench_trec_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchClassificationEvaluator'
+                ),
+                pred_postprocessor=dict(
+                    type='opencompass.datasets.trec_postprocess'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Please determine the type of the question below. Here are some examples of questions.\n\n{context}\n{input}',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='trec',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='all_labels',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchtrecDataset'),
+        dict(
+            abbr='LongBench_lsht_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchClassificationEvaluator'
+                ),
+                pred_postprocessor=dict(
+                    type='opencompass.datasets.lsht_postprocess'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt='请判断给定新闻的类别，下面是一些例子。\n\n{context}\n{input}',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='lsht',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='all_labels',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchlshtDataset'),
+        dict(
+            abbr='LongBench_multi_news_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchRougeEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=512,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'You are given several news passages. Write a one-page summary of all news. \n\nNews:\n{context}\n\nNow, write a one-page summary of all the news.\n\nSummary:\n',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='multi_news',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchmulti_newsDataset'),
+        dict(
+            abbr='LongBench_samsum_6',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchRougeEvaluator'),
+                pred_postprocessor=dict(
+                    type='opencompass.datasets.samsum_postprocess'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=128,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Summarize the dialogue into a few short sentences. The following are some examples.\n\n{context}\n\n{input}',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='samsum',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[150:175]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchsamsumDataset'),
+    ],
+]
+models = [
+    dict(
+        abbr='delta_net',
+        batch_size=128,
+        max_seq_len=2048,
+        model_kwargs=dict(
+            device_map='auto',
+            torch_dtype='torch.bfloat16',
+            trust_remote_code=True),
+        path='/mnt/jfzn/msj/delta_net-1.3B-100B',
+        run_cfg=dict(num_gpus=1),
+        tokenizer_kwargs=dict(padding_side='left', truncation_side='left'),
+        tokenizer_path='/mnt/jfzn/msj/delta_net-1.3B-100B',
+        type='opencompass.models.HuggingFaceBaseModel'),
+]
+work_dir = 'outputs/default/20251127_223020'

tmp/072e98a5-13a4-44fd-91a0-f552368807b5_params.py

@@ -0,0 +1,53 @@
+datasets = [
+    [
+        dict(
+            abbr='LongBench_multifieldqa_en',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Read the following text and answer briefly.\n\n{context}\n\nNow, answer the following question based on the above text, only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='multifieldqa_en',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchmultifieldqa_enDataset'),
+    ],
+]
+eval = dict(runner=dict(task=dict(dump_details=True)))
+models = [
+    dict(
+        abbr='retnet',
+        batch_size=128,
+        max_seq_len=2048,
+        model_kwargs=dict(
+            device_map='auto',
+            torch_dtype='torch.bfloat16',
+            trust_remote_code=True),
+        path='/mnt/jfzn/msj/retnet-1.3B-100B',
+        run_cfg=dict(num_gpus=1),
+        tokenizer_kwargs=dict(padding_side='left', truncation_side='left'),
+        tokenizer_path='/mnt/jfzn/msj/retnet-1.3B-100B',
+        type='opencompass.models.HuggingFaceBaseModel'),
+]
+work_dir = 'outputs/default/20251207_223306'

tmp/07b289dd-41fd-4f58-8c9b-e55ce7391d79_params.py

@@ -0,0 +1,50 @@
+datasets = [
+    [
+        dict(
+            abbr='LongBench_narrativeqa',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=128,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'You are given a story, which can be either a novel or a movie script, and a question. Answer the question as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nStory: {context}\n\nNow, answer the question based on the story as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nQuestion: {input}\n\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='narrativeqa',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchnarrativeqaDataset'),
+    ],
+]
+eval = dict(runner=dict(task=dict(dump_details=True)))
+models = [
+    dict(
+        abbr='mask_gdn-1.3B',
+        batch_padding=False,
+        batch_size=16,
+        max_out_len=100,
+        max_seq_len=16384,
+        path='/mnt/jfzn/msj/train_exp/mask_gdn_1B_hrr-rank4',
+        run_cfg=dict(num_gpus=1),
+        tokenizer_path='/mnt/jfzn/msj/train_exp/mask_gdn_1B_hrr-rank4',
+        type='opencompass.models.HuggingFaceCausalLM'),
+]
+work_dir = 'outputs/default/20251127_164548'

tmp/07f59276-08b3-4e03-ba78-cdcd5f4d49d1_params.py

@@ -0,0 +1,1421 @@
+datasets = [
+    [
+        dict(
+            abbr='LongBench_2wikimqa_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='2wikimqa',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBench2wikimqaDataset'),
+        dict(
+            abbr='LongBench_hotpotqa_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='hotpotqa',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchhotpotqaDataset'),
+        dict(
+            abbr='LongBench_musique_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='musique',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchmusiqueDataset'),
+        dict(
+            abbr='LongBench_multifieldqa_en_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Read the following text and answer briefly.\n\n{context}\n\nNow, answer the following question based on the above text, only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='multifieldqa_en',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[133:152]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchmultifieldqa_enDataset'),
+        dict(
+            abbr='LongBench_multifieldqa_zh_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    language='zh',
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            '阅读以下文字并用中文简短回答：\n\n{context}\n\n现在请基于上面的文章回答下面的问题，只告诉我答案，不要输出任何其他字词。\n\n问题：{input}\n回答：',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='multifieldqa_zh',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchmultifieldqa_zhDataset'),
+        dict(
+            abbr='LongBench_narrativeqa_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=128,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'You are given a story, which can be either a novel or a movie script, and a question. Answer the question as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nStory: {context}\n\nNow, answer the question based on the story as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nQuestion: {input}\n\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='narrativeqa',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchnarrativeqaDataset'),
+        dict(
+            abbr='LongBench_qasper_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='qasper',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchqasperDataset'),
+        dict(
+            abbr='LongBench_triviaqa_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_postprocessor=dict(
+                    type='opencompass.datasets.triviaqa_postprocess'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Answer the question based on the given passage. Only give me the answer and do not output any other words. The following are some examples.\n\n{context}\n\n{input}',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='triviaqa',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchtriviaqaDataset'),
+        dict(
+            abbr='LongBench_gov_report_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchRougeEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=512,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'You are given a report by a government agency. Write a one-page summary of the report.\n\nReport:\n{context}\n\nNow, write a one-page summary of the report.\n\nSummary:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='gov_report',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchgov_reportDataset'),
+        dict(
+            abbr='LongBench_qmsum_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchRougeEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=512,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'You are given a meeting transcript and a query containing a question or instruction. Answer the query in one or more sentences.\n\nTranscript:\n{context}\n\nNow, answer the query based on the above meeting transcript in one or more sentences.\n\nQuery: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='qmsum',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchqmsumDataset'),
+        dict(
+            abbr='LongBench_vcsum_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    language='zh',
+                    type='opencompass.datasets.LongBenchRougeEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=512,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            '下面有一段会议记录，请你阅读后，写一段总结，总结会议的内容。\n会议记录：\n{context}\n\n会议总结：',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='vcsum',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchvcsumDataset'),
+        dict(
+            abbr='LongBench_dureader_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    language='zh',
+                    type='opencompass.datasets.LongBenchRougeEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=128,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            '请基于给定的文章回答下述问题。\n\n文章：{context}\n\n请基于上述文章回答下面的问题。\n\n问题：{input}\n回答：',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='dureader',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchdureaderDataset'),
+        dict(
+            abbr='LongBench_lcc_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchCodeSimEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Please complete the code given below. \n{context}Next line of code:\n',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='lcc',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                ],
+                output_column='answers',
+                test_range='[441:504]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchlccDataset'),
+        dict(
+            abbr='LongBench_repobench-p_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchCodeSimEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Please complete the code given below. \n{context}{input}Next line of code:\n',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='repobench-p',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[441:504]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchrepobenchDataset'),
+        dict(
+            abbr='LongBench_passage_retrieval_en_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchRetrievalEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Here are 30 paragraphs from Wikipedia, along with an abstract. Please determine which paragraph the abstract is from.\n\n{context}\n\nThe following is an abstract.\n\n{input}\n\nPlease enter the number of the paragraph that the abstract is from. The answer format must be like "Paragraph 1", "Paragraph 2", etc.\n\nThe answer is: ',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='passage_retrieval_en',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchpassage_retrieval_enDataset'),
+        dict(
+            abbr='LongBench_passage_retrieval_zh_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    language='zh',
+                    type='opencompass.datasets.LongBenchRetrievalEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            '以下是若干段落文字，以及其中一个段落的摘要。请确定给定的摘要出自哪一段。\n\n{context}\n\n下面是一个摘要\n\n{input}\n\n请输入摘要所属段落的编号。答案格式必须是"段落1"，"段落2"等格式\n\n答案是：',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='passage_retrieval_zh',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchpassage_retrieval_zhDataset'),
+        dict(
+            abbr='LongBench_passage_count_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchCountEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'There are some paragraphs below sourced from Wikipedia. Some of them may be duplicates. Please carefully read these paragraphs and determine how many unique paragraphs there are after removing duplicates. In other words, how many non-repeating paragraphs are there in total?\n\n{context}\n\nPlease enter the final count of unique paragraphs after removing duplicates. The output format should only contain the number, such as 1, 2, 3, and so on.\n\nThe final answer is: ',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='passage_count',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchpassage_countDataset'),
+        dict(
+            abbr='LongBench_trec_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchClassificationEvaluator'
+                ),
+                pred_postprocessor=dict(
+                    type='opencompass.datasets.trec_postprocess'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Please determine the type of the question below. Here are some examples of questions.\n\n{context}\n{input}',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='trec',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='all_labels',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchtrecDataset'),
+        dict(
+            abbr='LongBench_lsht_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchClassificationEvaluator'
+                ),
+                pred_postprocessor=dict(
+                    type='opencompass.datasets.lsht_postprocess'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt='请判断给定新闻的类别，下面是一些例子。\n\n{context}\n{input}',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='lsht',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='all_labels',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchlshtDataset'),
+        dict(
+            abbr='LongBench_multi_news_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchRougeEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=512,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'You are given several news passages. Write a one-page summary of all news. \n\nNews:\n{context}\n\nNow, write a one-page summary of all the news.\n\nSummary:\n',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='multi_news',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchmulti_newsDataset'),
+        dict(
+            abbr='LongBench_samsum_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchRougeEvaluator'),
+                pred_postprocessor=dict(
+                    type='opencompass.datasets.samsum_postprocess'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=128,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Summarize the dialogue into a few short sentences. The following are some examples.\n\n{context}\n\n{input}',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='samsum',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchsamsumDataset'),
+        dict(
+            abbr='LongBench_2wikimqa_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='2wikimqa',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBench2wikimqaDataset'),
+        dict(
+            abbr='LongBench_hotpotqa_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='hotpotqa',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchhotpotqaDataset'),
+        dict(
+            abbr='LongBench_musique_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='musique',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchmusiqueDataset'),
+        dict(
+            abbr='LongBench_multifieldqa_en_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Read the following text and answer briefly.\n\n{context}\n\nNow, answer the following question based on the above text, only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='multifieldqa_en',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[133:152]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchmultifieldqa_enDataset'),
+        dict(
+            abbr='LongBench_multifieldqa_zh_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    language='zh',
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            '阅读以下文字并用中文简短回答：\n\n{context}\n\n现在请基于上面的文章回答下面的问题，只告诉我答案，不要输出任何其他字词。\n\n问题：{input}\n回答：',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='multifieldqa_zh',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchmultifieldqa_zhDataset'),
+        dict(
+            abbr='LongBench_narrativeqa_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=128,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'You are given a story, which can be either a novel or a movie script, and a question. Answer the question as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nStory: {context}\n\nNow, answer the question based on the story as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nQuestion: {input}\n\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='narrativeqa',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchnarrativeqaDataset'),
+        dict(
+            abbr='LongBench_qasper_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{context}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='qasper',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchqasperDataset'),
+        dict(
+            abbr='LongBench_triviaqa_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchF1Evaluator'),
+                pred_postprocessor=dict(
+                    type='opencompass.datasets.triviaqa_postprocess'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Answer the question based on the given passage. Only give me the answer and do not output any other words. The following are some examples.\n\n{context}\n\n{input}',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='triviaqa',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchtriviaqaDataset'),
+        dict(
+            abbr='LongBench_gov_report_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchRougeEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=512,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'You are given a report by a government agency. Write a one-page summary of the report.\n\nReport:\n{context}\n\nNow, write a one-page summary of the report.\n\nSummary:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='gov_report',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchgov_reportDataset'),
+        dict(
+            abbr='LongBench_qmsum_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchRougeEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=512,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'You are given a meeting transcript and a query containing a question or instruction. Answer the query in one or more sentences.\n\nTranscript:\n{context}\n\nNow, answer the query based on the above meeting transcript in one or more sentences.\n\nQuery: {input}\nAnswer:',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='qmsum',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchqmsumDataset'),
+        dict(
+            abbr='LongBench_vcsum_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    language='zh',
+                    type='opencompass.datasets.LongBenchRougeEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=512,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            '下面有一段会议记录，请你阅读后，写一段总结，总结会议的内容。\n会议记录：\n{context}\n\n会议总结：',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='vcsum',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchvcsumDataset'),
+        dict(
+            abbr='LongBench_dureader_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    language='zh',
+                    type='opencompass.datasets.LongBenchRougeEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=128,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            '请基于给定的文章回答下述问题。\n\n文章：{context}\n\n请基于上述文章回答下面的问题。\n\n问题：{input}\n回答：',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='dureader',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchdureaderDataset'),
+        dict(
+            abbr='LongBench_lcc_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchCodeSimEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Please complete the code given below. \n{context}Next line of code:\n',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='lcc',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                ],
+                output_column='answers',
+                test_range='[441:504]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchlccDataset'),
+        dict(
+            abbr='LongBench_repobench-p_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchCodeSimEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Please complete the code given below. \n{context}{input}Next line of code:\n',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='repobench-p',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[441:504]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchrepobenchDataset'),
+        dict(
+            abbr='LongBench_passage_retrieval_en_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchRetrievalEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Here are 30 paragraphs from Wikipedia, along with an abstract. Please determine which paragraph the abstract is from.\n\n{context}\n\nThe following is an abstract.\n\n{input}\n\nPlease enter the number of the paragraph that the abstract is from. The answer format must be like "Paragraph 1", "Paragraph 2", etc.\n\nThe answer is: ',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='passage_retrieval_en',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchpassage_retrieval_enDataset'),
+        dict(
+            abbr='LongBench_passage_retrieval_zh_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    language='zh',
+                    type='opencompass.datasets.LongBenchRetrievalEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            '以下是若干段落文字，以及其中一个段落的摘要。请确定给定的摘要出自哪一段。\n\n{context}\n\n下面是一个摘要\n\n{input}\n\n请输入摘要所属段落的编号。答案格式必须是"段落1"，"段落2"等格式\n\n答案是：',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='passage_retrieval_zh',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchpassage_retrieval_zhDataset'),
+        dict(
+            abbr='LongBench_passage_count_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchCountEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=32,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'There are some paragraphs below sourced from Wikipedia. Some of them may be duplicates. Please carefully read these paragraphs and determine how many unique paragraphs there are after removing duplicates. In other words, how many non-repeating paragraphs are there in total?\n\n{context}\n\nPlease enter the final count of unique paragraphs after removing duplicates. The output format should only contain the number, such as 1, 2, 3, and so on.\n\nThe final answer is: ',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='passage_count',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchpassage_countDataset'),
+        dict(
+            abbr='LongBench_trec_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchClassificationEvaluator'
+                ),
+                pred_postprocessor=dict(
+                    type='opencompass.datasets.trec_postprocess'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Please determine the type of the question below. Here are some examples of questions.\n\n{context}\n{input}',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='trec',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='all_labels',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchtrecDataset'),
+        dict(
+            abbr='LongBench_lsht_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchClassificationEvaluator'
+                ),
+                pred_postprocessor=dict(
+                    type='opencompass.datasets.lsht_postprocess'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=64,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt='请判断给定新闻的类别，下面是一些例子。\n\n{context}\n{input}',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='lsht',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='all_labels',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchlshtDataset'),
+        dict(
+            abbr='LongBench_multi_news_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchRougeEvaluator'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=512,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'You are given several news passages. Write a one-page summary of all news. \n\nNews:\n{context}\n\nNow, write a one-page summary of all the news.\n\nSummary:\n',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='multi_news',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchmulti_newsDataset'),
+        dict(
+            abbr='LongBench_samsum_7',
+            eval_cfg=dict(
+                evaluator=dict(
+                    type='opencompass.datasets.LongBenchRougeEvaluator'),
+                pred_postprocessor=dict(
+                    type='opencompass.datasets.samsum_postprocess'),
+                pred_role='BOT'),
+            infer_cfg=dict(
+                inferencer=dict(
+                    max_out_len=128,
+                    type='opencompass.openicl.icl_inferencer.GenInferencer'),
+                prompt_template=dict(
+                    template=dict(round=[
+                        dict(
+                            prompt=
+                            'Summarize the dialogue into a few short sentences. The following are some examples.\n\n{context}\n\n{input}',
+                            role='HUMAN'),
+                    ]),
+                    type=
+                    'opencompass.openicl.icl_prompt_template.PromptTemplate'),
+                retriever=dict(
+                    type='opencompass.openicl.icl_retriever.ZeroRetriever')),
+            name='samsum',
+            path='opencompass/Longbench',
+            reader_cfg=dict(
+                input_columns=[
+                    'context',
+                    'input',
+                ],
+                output_column='answers',
+                test_range='[175:200]',
+                test_split='test',
+                train_split='test'),
+            type='opencompass.datasets.LongBenchsamsumDataset'),
+    ],
+]
+models = [
+    dict(
+        abbr='mask_gdn-1.3B',
+        batch_padding=False,
+        batch_size=16,
+        max_out_len=100,
+        max_seq_len=16384,
+        path='/mnt/jfzn/msj/train_exp/mask_gdn_1B_hrr-rank4',
+        run_cfg=dict(num_gpus=1),
+        tokenizer_path='/mnt/jfzn/msj/train_exp/mask_gdn_1B_hrr-rank4',
+        type='opencompass.models.HuggingFaceCausalLM'),
+]
+work_dir = 'outputs/default/20251127_164744'