| from typing import List, Optional |
|
|
| from mmengine import ConfigDict |
|
|
| from opencompass.utils import dataset_abbr_from_cfg |
| from opencompass.utils.prompt import get_prompt_hash |
|
|
| from .default import DefaultSummarizer |
|
|
|
|
| class CircularSummarizer(DefaultSummarizer): |
|
|
| def __init__(self, |
| config: ConfigDict, |
| dataset_abbrs: Optional[List[str]] = None, |
| summary_groups: List = [], |
| prompt_db=None, |
| metric_types=None) -> None: |
| super().__init__(config, dataset_abbrs, summary_groups, prompt_db) |
| self.metric_types = metric_types |
|
|
| def _format_table(self, parsed_results, dataset_metrics, |
| dataset_eval_mode): |
| prompt_version = { |
| dataset_abbr_from_cfg(d): get_prompt_hash(d)[:6] |
| for d in self.dataset_cfgs |
| } |
|
|
| table = [] |
| header1 = ['dataset', 'version', 'mode'] + sum( |
| [[model_abbr] + ['-' for _ in range(len(self.metric_types) - 1)] |
| for model_abbr in self.model_abbrs], []) |
| table.append(header1) |
| header2 = ['-', '-', '-'] + sum( |
| [self.metric_types for _ in self.model_abbrs], []) |
| table.append(header2) |
| for dataset_abbr in self.dataset_abbrs: |
| if dataset_abbr not in dataset_metrics: |
| table.append([dataset_abbr, '-', '-'] + ['-'] * |
| len(self.model_abbrs) * len(self.metric_types)) |
| continue |
| row = [ |
| dataset_abbr, |
| prompt_version.get(dataset_abbr, '-'), |
| dataset_eval_mode.get(dataset_abbr, '-') |
| ] |
| for model_abbr in self.model_abbrs: |
| for metric in self.metric_types: |
| if dataset_abbr in parsed_results[ |
| model_abbr] and metric in parsed_results[ |
| model_abbr][dataset_abbr]: |
| row.append('{:.02f}'.format( |
| parsed_results[model_abbr][dataset_abbr][metric])) |
| else: |
| row.append('-') |
| table.append(row) |
| return table |
|
|
