Spaces:
Running
Running
| /** | |
| * | |
| * Copyright 2023-2025 InspectorRAGet Team | |
| * | |
| * Licensed under the Apache License, Version 2.0 (the "License"); | |
| * you may not use this file except in compliance with the License. | |
| * You may obtain a copy of the License at | |
| * | |
| * http://www.apache.org/licenses/LICENSE-2.0 | |
| * | |
| * Unless required by applicable law or agreed to in writing, software | |
| * distributed under the License is distributed on an "AS IS" BASIS, | |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| * See the License for the specific language governing permissions and | |
| * limitations under the License. | |
| * | |
| **/ | |
| 'use client'; | |
| import { groupBy, isEmpty } from 'lodash'; | |
| import cx from 'classnames'; | |
| import { useState, useEffect, useMemo } from 'react'; | |
| import { | |
| DataTable, | |
| TableContainer, | |
| Table, | |
| TableHead, | |
| TableRow, | |
| TableHeader, | |
| TableBody, | |
| TableCell, | |
| } from '@carbon/react'; | |
| import { WarningAlt } from '@carbon/icons-react'; | |
| import { | |
| SimpleBarChart, | |
| RadarChart, | |
| GroupedBarChart, | |
| } from '@carbon/charts-react'; | |
| import { Alignments, ScaleTypes } from '@carbon/charts'; | |
| import { useTheme } from '@/src/theme'; | |
| import { | |
| AggregationConfidenceLevels, | |
| AggregationStatistics, | |
| Aggregator, | |
| Metric, | |
| Model, | |
| TaskEvaluation, | |
| } from '@/src/types'; | |
| import { | |
| extractMetricDisplayName, | |
| castToNumber, | |
| } from '@/src/utilities/metrics'; | |
| import { | |
| meanAggregator, | |
| medianAggregator, | |
| majorityAggregator, | |
| } from '@/src/utilities/aggregators'; | |
| import { areObjectsIntersecting } from '@/src/utilities/objects'; | |
| import { getModelColorPalette } from '@/src/utilities/colors'; | |
| import AggregatorSelector from '@/src/components/selectors/AggregatorSelector'; | |
| import Filters from '@/src/components/filters/Filters'; | |
| import HidePanel from '@/src/views/performance-overview/Hide'; | |
| import '@carbon/charts-react/styles.css'; | |
| import classes from './PerformanceOverview.module.scss'; | |
| // =================================================================================== | |
| // TYPES | |
| // =================================================================================== | |
| interface Props { | |
| evaluationsPerMetric: { [key: string]: TaskEvaluation[] }; | |
| models: Model[]; | |
| metrics: Metric[]; | |
| filters: { [key: string]: string[] }; | |
| numTasks: number; | |
| } | |
| // =================================================================================== | |
| // COMPUTE FUNCTIONS | |
| // =================================================================================== | |
| function calculateRanks( | |
| data: { | |
| model: string; | |
| metric: string; | |
| score: number; | |
| rank: number; | |
| std?: number; | |
| order?: 'ascending' | 'descending'; | |
| levels: { low: number; medium: number; high: number }; | |
| }[], | |
| ) { | |
| const peformancePerMetric: { | |
| [key: string]: { | |
| model: string; | |
| score: number; | |
| rank: number; | |
| std?: number; | |
| }[]; | |
| } = {}; | |
| const order: { [key: string]: 'ascending' | 'descending' } = {}; | |
| for (const entry of data) { | |
| if (peformancePerMetric.hasOwnProperty(entry.metric)) { | |
| peformancePerMetric[entry.metric].push(entry); | |
| } else { | |
| peformancePerMetric[entry.metric] = [entry]; | |
| } | |
| if (!order.hasOwnProperty(entry.metric)) { | |
| order[entry.metric] = entry.order ? entry.order : 'ascending'; | |
| } | |
| } | |
| for (const [metric, performance] of Object.entries(peformancePerMetric)) { | |
| performance.sort((a, b) => { | |
| if (order[metric] == 'ascending') { | |
| return a.score > b.score ? -1 : 1; | |
| } else { | |
| return a.score > b.score ? 1 : -1; | |
| } | |
| }); | |
| let rank = 0; | |
| performance.forEach((entry, idx) => { | |
| if (idx !== 0 && entry.score === performance[idx - 1].score) { | |
| entry['rank'] = rank; | |
| } else { | |
| entry['rank'] = rank + 1; | |
| rank += 1; | |
| } | |
| }); | |
| } | |
| } | |
| // =================================================================================== | |
| // RENDER FUNCTIONS | |
| // =================================================================================== | |
| function sparkline( | |
| distribution: { [key: string]: number } | undefined, | |
| theme?: string, | |
| ) { | |
| if (distribution === undefined) { | |
| return null; | |
| } else { | |
| return ( | |
| <SimpleBarChart | |
| data={Object.entries(distribution).map(([value, count]) => { | |
| return { group: value, value: count }; | |
| })} | |
| options={{ | |
| color: { | |
| scale: { low: '#fa4d56', medium: '#f1c21b', high: '#42be65' }, | |
| }, | |
| axes: { | |
| left: { | |
| mapsTo: 'value', | |
| visible: false, | |
| scaleType: ScaleTypes.LINEAR, | |
| }, | |
| bottom: { | |
| mapsTo: 'group', | |
| visible: false, | |
| scaleType: ScaleTypes.LABELS, | |
| }, | |
| }, | |
| grid: { | |
| y: { | |
| enabled: false, | |
| }, | |
| x: { | |
| enabled: false, | |
| }, | |
| }, | |
| legend: { | |
| enabled: false, | |
| }, | |
| toolbar: { | |
| enabled: false, | |
| }, | |
| theme: theme, | |
| height: '24px', | |
| width: '48px', | |
| }} | |
| ></SimpleBarChart> | |
| ); | |
| } | |
| } | |
| function drawTable( | |
| data: { | |
| model: string; | |
| metric: string; | |
| score: number; | |
| rank: number; | |
| std?: number; | |
| levels: { low: number; medium: number; high: number }; | |
| }[], | |
| metrics: string[], | |
| plot: boolean = false, | |
| theme?: string, | |
| ) { | |
| // Step 1: Define headers | |
| const headers = [ | |
| { key: 'model', header: 'Model' }, | |
| ...metrics.map((metric) => { | |
| return { key: metric, header: metric }; | |
| }), | |
| { key: 'rank', header: 'Rank' }, | |
| ]; | |
| // Step 2: Group data per model | |
| const dataPerModel: { | |
| [key: string]: { | |
| model: string; | |
| metric: string; | |
| score: number; | |
| rank: number; | |
| std?: number; | |
| }[]; | |
| } = groupBy(data, (entry) => entry.model); | |
| // Step 3: Compute overall rank | |
| const overallRank: [string, number][] = Object.entries(dataPerModel).map( | |
| ([model, entry]) => [model, entry.reduce((n, { rank }) => n + rank, 0)], | |
| ); | |
| // Step 4: Sort based on overall rank, if necessary | |
| if (overallRank.length > 1) { | |
| overallRank.sort((a, b) => { | |
| return a[1] - b[1]; | |
| }); | |
| } | |
| // Step 5: Define distribution map | |
| const distributions = new Map( | |
| data.map((entry) => [`${entry.model}:${entry.metric}`, entry.levels]), | |
| ); | |
| // Step 5: Define rows | |
| const rows: { [key: string]: string }[] = []; | |
| overallRank.forEach(([model, sum], index) => { | |
| rows.push({ | |
| id: model, | |
| model: model, | |
| ...Object.fromEntries( | |
| dataPerModel[model].map((record) => [ | |
| record.metric, | |
| record.std | |
| ? `${record.score} ± ${record.std} (${record.rank})` | |
| : `${record.score} (${record.rank})`, | |
| ]), | |
| ), | |
| rank: `${sum.toLocaleString()} (${(index + 1).toLocaleString()})`, | |
| }); | |
| }); | |
| // Step 6: Draw table | |
| return ( | |
| <DataTable rows={rows} headers={headers} isSortable> | |
| {({ rows, headers, getTableProps, getHeaderProps, getRowProps }) => ( | |
| <TableContainer> | |
| <Table {...getTableProps()}> | |
| <TableHead> | |
| <TableRow> | |
| {headers.map((header, index) => ( | |
| <TableHeader | |
| key={'header--' + index} | |
| {...getHeaderProps({ header })} | |
| > | |
| {header.key === 'rank' ? ( | |
| <span>Σ {header.header}</span> | |
| ) : ( | |
| header.header | |
| )} | |
| </TableHeader> | |
| ))} | |
| </TableRow> | |
| </TableHead> | |
| <TableBody> | |
| {rows.map((row, index) => ( | |
| <TableRow key={'row--' + index} {...getRowProps({ row })}> | |
| {row.cells.map((cell) => ( | |
| <TableCell key={cell.id}> | |
| <div className={classes.tableCell}> | |
| {cell.value ? ( | |
| cell.value.includes('(1)') ? ( | |
| <strong>{cell.value}</strong> | |
| ) : ( | |
| cell.value | |
| ) | |
| ) : ( | |
| '-' | |
| )} | |
| {plot && metrics.includes(cell.info.header) | |
| ? sparkline(distributions.get(cell.id), theme) | |
| : null} | |
| </div> | |
| </TableCell> | |
| ))} | |
| </TableRow> | |
| ))} | |
| </TableBody> | |
| </Table> | |
| </TableContainer> | |
| )} | |
| </DataTable> | |
| ); | |
| } | |
| function disclaimers({ | |
| std = false, | |
| spakline = false, | |
| theme, | |
| }: { | |
| std?: boolean; | |
| spakline?: boolean; | |
| theme?: string; | |
| }) { | |
| return ( | |
| <div className={classes.disclaimers}> | |
| <span> | |
| <sup>*</sup> | |
| <strong>(rank)</strong> indicates model's comparative position w.r.t to | |
| other models for a given metric | |
| </span> | |
| {std && ( | |
| <span> | |
| <sup>*</sup> | |
| <strong>value±std</strong> shows averages of aggregate values and | |
| standard deviation across all tasks | |
| </span> | |
| )} | |
| {spakline && ( | |
| <div className={classes.disclaimerSparkline}> | |
| <span> | |
| <sup>*</sup> | |
| </span> | |
| {sparkline({ low: 5, medium: 10, high: 15 }, theme)} | |
| <span>reflects confidence level on the aggregate values. </span> | |
| <div className={classes.legendLow}>■</div> | |
| <span> | |
| # of tasks where where minority rating is far from majority | |
| rating, | |
| </span> | |
| <div className={classes.legendMedium}>■</div> | |
| <span> | |
| # of tasks where where minority rating is similar to majority | |
| rating and | |
| </span> | |
| <div className={classes.legendHigh}>■</div> | |
| <span> | |
| # of tasks where where all annotators chose same rating | |
| </span> | |
| </div> | |
| )} | |
| </div> | |
| ); | |
| } | |
| // =================================================================================== | |
| // MAIN FUNCTION | |
| // =================================================================================== | |
| export default function PerformanceOverview({ | |
| evaluationsPerMetric, | |
| models, | |
| metrics, | |
| filters, | |
| numTasks, | |
| }: Props) { | |
| // Step 1: Initialize state and necessary variables | |
| const [WindowWidth, setWindowWidth] = useState<number>( | |
| global?.window && window.innerWidth, | |
| ); | |
| const [WindowHeight, setWindowHeight] = useState<number>( | |
| global?.window && window.innerHeight, | |
| ); | |
| const aggregators: Aggregator[] = [ | |
| meanAggregator, | |
| medianAggregator, | |
| majorityAggregator, | |
| ]; | |
| const [selectedAggregators, setSelectedAggregators] = useState<{ | |
| [key: string]: Aggregator; | |
| }>( | |
| Object.fromEntries( | |
| metrics | |
| .filter((metric) => metric.author === 'human') | |
| .map((metric) => [ | |
| metric.name, | |
| metric.aggregator === 'majority' | |
| ? majorityAggregator | |
| : metric.aggregator === 'median' | |
| ? medianAggregator | |
| : meanAggregator, | |
| ]), | |
| ), | |
| ); | |
| const [selectedFilters, setSelectedFilters] = useState<{ | |
| [key: string]: string[]; | |
| }>({}); | |
| const [modelColors, modelOrder] = getModelColorPalette(models); | |
| const [hiddenModels, setHiddenModels] = useState<Model[]>([]); | |
| const [hiddenMetrics, setHiddenMetrics] = useState<Metric[]>([]); | |
| // Step 2: Run effects | |
| // Step 2.a: Adjust graph width & heigh based on window size | |
| useEffect(() => { | |
| const handleWindowResize = () => { | |
| setWindowWidth(window.innerWidth); | |
| setWindowHeight(window.innerHeight); | |
| }; | |
| // Step: Add event listener | |
| window.addEventListener('resize', handleWindowResize); | |
| // Step: Cleanup to remove event listener | |
| return () => { | |
| window.removeEventListener('resize', handleWindowResize); | |
| }; | |
| }, []); | |
| // Step 2.a: Fetch theme | |
| const { theme } = useTheme(); | |
| // Step 2.c: Generate performance data for human and algorithmic metrics | |
| const [humanMetricsData, algorithmicMetricsData, numSelectedTasks] = | |
| useMemo(() => { | |
| // Eligible metrics | |
| const eligibleMetrics = Object.fromEntries( | |
| metrics.map((metric) => [metric.name, metric]), | |
| ); | |
| let hData: { | |
| model: string; | |
| metric: string; | |
| score: number; | |
| std: number; | |
| rank: number; | |
| size: number; | |
| levels: { low: number; medium: number; high: number }; | |
| order?: 'ascending' | 'descending'; | |
| }[] = []; | |
| let aData: { | |
| model: string; | |
| metric: string; | |
| score: number; | |
| std?: number; | |
| rank: number; | |
| size: number; | |
| levels: { low: number; medium: number; high: number }; | |
| order?: 'ascending' | 'descending'; | |
| }[] = []; | |
| const performancePerModel: { | |
| [key: string]: { | |
| [key: string]: { | |
| value: number; | |
| std: number; | |
| levels: { low: number; medium: number; high: number }; | |
| }; | |
| }; | |
| } = {}; | |
| const eligibleEvaluationsPerModel: { | |
| [key: string]: { [key: string]: number }; | |
| } = {}; | |
| // Step 1: Calculate model performance across entire dataset | |
| let selectedTasksCount; | |
| for (const [metric, evaluations] of Object.entries( | |
| evaluationsPerMetric, | |
| )) { | |
| const aggregator = selectedAggregators[metric] || meanAggregator; | |
| // Select evaluations based on selected filters | |
| const selectedEvaluations = !isEmpty(selectedFilters) | |
| ? evaluations.filter((e) => { | |
| return areObjectsIntersecting(selectedFilters, e); | |
| }) | |
| : evaluations; | |
| // Calculate selected tasks count | |
| selectedTasksCount = selectedEvaluations.length / models.length; | |
| selectedEvaluations.forEach((evaluation) => { | |
| // Step 1.a: Calcuate aggregated value | |
| const aggregateStatistics: AggregationStatistics = aggregator.apply( | |
| Object.values(evaluation.annotations[`${metric}`]).map( | |
| (entry) => entry.value, | |
| ), | |
| eligibleMetrics[metric].values, | |
| ); | |
| // Step 1.b: Skip evaluations, if no majority aggrement exist | |
| if ( | |
| aggregator.name === 'majority' && | |
| aggregateStatistics.value === 'Indeterminate' | |
| ) { | |
| return; | |
| } | |
| // Step 1.c: Cast to numeric value for further processing | |
| const aggregateValue = castToNumber( | |
| aggregateStatistics.value, | |
| eligibleMetrics[metric].values, | |
| ); | |
| // Step 1.d: Translate model id to model name | |
| const modelName = | |
| models.find((model) => model.modelId === evaluation.modelId) | |
| ?.name || evaluation.modelId; | |
| // Step 1.d: Update performance per model object | |
| if (performancePerModel.hasOwnProperty(modelName)) { | |
| if (performancePerModel[modelName].hasOwnProperty(metric)) { | |
| performancePerModel[modelName][metric].value += aggregateValue; | |
| performancePerModel[modelName][metric].std += | |
| aggregateStatistics.std; | |
| if ( | |
| aggregateStatistics.confidence === | |
| AggregationConfidenceLevels.LOW | |
| ) { | |
| performancePerModel[modelName][metric].levels.low += 1; | |
| } | |
| if ( | |
| aggregateStatistics.confidence === | |
| AggregationConfidenceLevels.MEDIUM | |
| ) { | |
| performancePerModel[modelName][metric].levels.medium += 1; | |
| } | |
| if ( | |
| aggregateStatistics.confidence === | |
| AggregationConfidenceLevels.HIGH | |
| ) { | |
| performancePerModel[modelName][metric].levels.high += 1; | |
| } | |
| } else { | |
| performancePerModel[modelName][metric] = { | |
| value: aggregateValue, | |
| std: aggregateStatistics.std, | |
| levels: { | |
| low: | |
| aggregateStatistics.confidence === | |
| AggregationConfidenceLevels.LOW | |
| ? 1 | |
| : 0, | |
| medium: | |
| aggregateStatistics.confidence === | |
| AggregationConfidenceLevels.MEDIUM | |
| ? 1 | |
| : 0, | |
| high: | |
| aggregateStatistics.confidence === | |
| AggregationConfidenceLevels.HIGH | |
| ? 1 | |
| : 0, | |
| }, | |
| }; | |
| } | |
| } else { | |
| performancePerModel[modelName] = { | |
| [metric]: { | |
| value: aggregateValue, | |
| std: aggregateStatistics.std, | |
| levels: { | |
| low: | |
| aggregateStatistics.confidence === | |
| AggregationConfidenceLevels.LOW | |
| ? 1 | |
| : 0, | |
| medium: | |
| aggregateStatistics.confidence === | |
| AggregationConfidenceLevels.MEDIUM | |
| ? 1 | |
| : 0, | |
| high: | |
| aggregateStatistics.confidence === | |
| AggregationConfidenceLevels.HIGH | |
| ? 1 | |
| : 0, | |
| }, | |
| }, | |
| }; | |
| } | |
| // Step 1.e: Update eligible evaluations per model object | |
| if (eligibleEvaluationsPerModel.hasOwnProperty(modelName)) { | |
| if (eligibleEvaluationsPerModel[modelName].hasOwnProperty(metric)) { | |
| eligibleEvaluationsPerModel[modelName][metric] += 1; | |
| } else { | |
| eligibleEvaluationsPerModel[modelName][metric] = 1; | |
| } | |
| } else { | |
| eligibleEvaluationsPerModel[modelName] = { | |
| [metric]: 1, | |
| }; | |
| } | |
| }); | |
| } | |
| // Step 2: Add raw performance data | |
| for (const [model, performance] of Object.entries(performancePerModel)) { | |
| for (const [metric, statistics] of Object.entries(performance)) { | |
| if (eligibleMetrics.hasOwnProperty(metric)) { | |
| if (eligibleMetrics[metric].author === 'human') { | |
| hData.push({ | |
| model: model, | |
| metric: extractMetricDisplayName(eligibleMetrics[metric]), | |
| score: parseFloat( | |
| (statistics.value / selectedTasksCount).toFixed(2), | |
| ), | |
| rank: -1, | |
| size: eligibleEvaluationsPerModel[model][metric], | |
| std: parseFloat( | |
| (statistics.std / selectedTasksCount).toFixed(2), | |
| ), | |
| levels: statistics.levels, | |
| ...(eligibleMetrics[metric].order && { | |
| order: eligibleMetrics[metric].order, | |
| }), | |
| }); | |
| } else if (eligibleMetrics[metric].author === 'algorithm') { | |
| aData.push({ | |
| model: model, | |
| metric: extractMetricDisplayName(eligibleMetrics[metric]), | |
| score: parseFloat( | |
| (statistics.value / selectedTasksCount).toFixed(2), | |
| ), | |
| rank: -1, | |
| size: eligibleEvaluationsPerModel[model][metric], | |
| std: parseFloat( | |
| (statistics.std / selectedTasksCount).toFixed(2), | |
| ), | |
| levels: statistics.levels, | |
| ...(eligibleMetrics[metric].order && { | |
| order: eligibleMetrics[metric].order, | |
| }), | |
| }); | |
| } | |
| } | |
| } | |
| } | |
| // Step 3: Filter hidden metrics data | |
| const hiddenMetricNames = hiddenMetrics.map((metric) => | |
| extractMetricDisplayName(metric), | |
| ); | |
| // Step 3.a: Human metrics | |
| if (Array.isArray(hData)) { | |
| hData = hData.filter( | |
| (entry) => !hiddenMetricNames.includes(entry.metric), | |
| ); | |
| } | |
| // Step 3.b: Algorithmic metrics | |
| if (Array.isArray(aData)) { | |
| aData = aData.filter( | |
| (entry) => !hiddenMetricNames.includes(entry.metric), | |
| ); | |
| } | |
| // Step 4: Filter hidden models data | |
| const hiddenModelNames = hiddenModels.map((model) => | |
| model.name ? model.name : model.modelId, | |
| ); | |
| // Step 4.a: Human metrics | |
| if (Array.isArray(hData)) { | |
| hData = hData.filter( | |
| (entry) => !hiddenModelNames.includes(entry.model), | |
| ); | |
| } | |
| // Step 4.b: Algorithmic metrics | |
| if (Array.isArray(aData)) { | |
| aData = aData.filter( | |
| (entry) => !hiddenModelNames.includes(entry.model), | |
| ); | |
| } | |
| // Step 5: Generate add rank information | |
| // Step 5.a: Human metrics | |
| if (Array.isArray(hData)) { | |
| calculateRanks(hData); | |
| } | |
| // Step 5.b: Algorithmic metrics | |
| if (Array.isArray(aData)) { | |
| calculateRanks(aData); | |
| } | |
| return [hData, aData, selectedTasksCount]; | |
| }, [ | |
| evaluationsPerMetric, | |
| metrics, | |
| models, | |
| selectedAggregators, | |
| selectedFilters, | |
| hiddenModels, | |
| hiddenMetrics, | |
| ]); | |
| const humanMetricsInData = new Set( | |
| humanMetricsData.map((entry) => entry.metric), | |
| ); | |
| const algorithmicmetricsInData = new Set( | |
| algorithmicMetricsData.map((entry) => entry.metric), | |
| ); | |
| // Step 3: Render | |
| return ( | |
| <div className={classes.page}> | |
| <div className={classes.selectors}> | |
| {Object.entries(selectedAggregators).map(([metricName, aggregator]) => { | |
| const metric = metrics.find((entry) => entry.name === metricName); | |
| return ( | |
| <div | |
| key={`${metricName}-aggregator`} | |
| className={classes.aggregatorSelector} | |
| > | |
| <h5> | |
| {metric | |
| ? extractMetricDisplayName(metric) | |
| : metricName.charAt(0).toUpperCase() + | |
| metricName.slice(1).toLowerCase()} | |
| </h5> | |
| <AggregatorSelector | |
| aggregators={aggregators} | |
| defaultValue={aggregator} | |
| onSelect={(selection: Aggregator) => { | |
| setSelectedAggregators({ | |
| ...selectedAggregators, | |
| [metricName]: selection, | |
| }); | |
| }} | |
| warn={aggregator.name === 'majority'} | |
| warnText={ | |
| aggregator.name === 'majority' | |
| ? 'Caution: Denominator might vary for categorical metrics.' | |
| : 'You must select an aggregator to view results.' | |
| } | |
| ></AggregatorSelector> | |
| </div> | |
| ); | |
| })} | |
| </div> | |
| {!isEmpty(filters) ? ( | |
| <Filters | |
| keyPrefix="PerformanceOverview" | |
| filters={filters} | |
| selectedFilters={selectedFilters} | |
| setSelectedFilters={setSelectedFilters} | |
| /> | |
| ) : null} | |
| <HidePanel | |
| models={models} | |
| metrics={metrics} | |
| hiddenModels={hiddenModels} | |
| hiddenMetrics={hiddenMetrics} | |
| setHiddenModels={setHiddenModels} | |
| setHiddenMetrics={setHiddenMetrics} | |
| /> | |
| <div | |
| className={cx( | |
| classes.row, | |
| humanMetricsInData.size == 0 || algorithmicmetricsInData.size == 0 | |
| ? classes.center | |
| : null, | |
| )} | |
| > | |
| {humanMetricsInData.size ? ( | |
| <div | |
| className={cx( | |
| classes.column, | |
| algorithmicmetricsInData.size === 0 ? classes.expand : null, | |
| )} | |
| > | |
| <h4> | |
| Human Evaluations ({numSelectedTasks}/{numTasks}) | |
| </h4> | |
| <div className={classes.performanceTable}> | |
| {drawTable( | |
| humanMetricsData, | |
| Array.from(humanMetricsInData), | |
| true, | |
| theme, | |
| )} | |
| {disclaimers({ std: true, spakline: true, theme: theme })} | |
| </div> | |
| <div className={classes.row}> | |
| {humanMetricsInData.size < 3 ? ( | |
| <> | |
| <GroupedBarChart | |
| data={humanMetricsData | |
| .sort((a, b) => (a.model > b.model ? -1 : 1)) | |
| .map((entry) => { | |
| return { | |
| group: entry.model, | |
| key: entry.metric, | |
| value: entry.score, | |
| }; | |
| })} | |
| options={{ | |
| axes: { | |
| left: { | |
| mapsTo: 'value', | |
| }, | |
| bottom: { | |
| mapsTo: 'key', | |
| scaleType: ScaleTypes.LABELS, | |
| }, | |
| }, | |
| width: `${Math.round(WindowWidth * 0.45)}px`, | |
| height: `${Math.round(WindowHeight * 0.5)}px`, | |
| toolbar: { | |
| enabled: false, | |
| }, | |
| color: { | |
| scale: modelColors, | |
| }, | |
| legend: { | |
| order: modelOrder, | |
| }, | |
| theme: theme, | |
| }} | |
| ></GroupedBarChart> | |
| </> | |
| ) : ( | |
| <> | |
| <RadarChart | |
| data={humanMetricsData.map((entry) => { | |
| // Step 1: Find metric under consideration | |
| const metric = metrics.find( | |
| (m) => m.displayName === entry.metric, | |
| ); | |
| // Step 2: Calculate normalized score | |
| let normalizedScore = entry.score; | |
| if ( | |
| metric?.minValue !== undefined && | |
| metric.maxValue !== undefined | |
| ) { | |
| // Step 2.a: Fetch minimum value | |
| const minValue = | |
| typeof metric.minValue === 'number' | |
| ? metric.minValue | |
| : castToNumber( | |
| metric.minValue?.value, | |
| metric.values, | |
| ); | |
| // Step 2.b: Fetch maximum value | |
| const maxValue = | |
| typeof metric.maxValue === 'number' | |
| ? metric.maxValue | |
| : castToNumber( | |
| metric.maxValue?.value, | |
| metric.values, | |
| ); | |
| // Step 2.c: Calculate min-max normalized score | |
| normalizedScore = | |
| Math.round( | |
| ((entry.score - minValue) / (maxValue - minValue)) * | |
| 100, | |
| ) / 100; | |
| } | |
| // Step 3: Return | |
| return { | |
| model: entry.model, | |
| metric: entry.metric, | |
| score: normalizedScore, | |
| }; | |
| })} | |
| options={{ | |
| radar: { | |
| alignment: Alignments.CENTER, | |
| axes: { | |
| angle: 'metric', | |
| value: 'score', | |
| }, | |
| }, | |
| data: { | |
| groupMapsTo: 'model', | |
| }, | |
| color: { | |
| scale: modelColors, | |
| }, | |
| legend: { | |
| alignment: Alignments.CENTER, | |
| order: modelOrder, | |
| }, | |
| width: `${Math.round(WindowWidth * 0.45)}px`, | |
| height: `${Math.round(WindowHeight * 0.5)}px`, | |
| toolbar: { | |
| enabled: false, | |
| }, | |
| theme: theme, | |
| }} | |
| ></RadarChart> | |
| </> | |
| )} | |
| </div> | |
| </div> | |
| ) : null} | |
| {humanMetricsInData.size && algorithmicmetricsInData.size ? ( | |
| <div className={classes.seperator}></div> | |
| ) : null} | |
| {algorithmicmetricsInData.size ? ( | |
| <div | |
| className={cx( | |
| classes.column, | |
| humanMetricsInData.size === 0 ? classes.expand : null, | |
| )} | |
| > | |
| <h4> | |
| Algorithmic Evaluations ({numSelectedTasks}/{numTasks}) | |
| </h4> | |
| <div className={classes.performanceTable}> | |
| {drawTable( | |
| algorithmicMetricsData, | |
| Array.from(algorithmicmetricsInData), | |
| )} | |
| {disclaimers({})} | |
| </div> | |
| <div className={classes.row}> | |
| {algorithmicmetricsInData.size < 3 ? ( | |
| <> | |
| <GroupedBarChart | |
| data={algorithmicMetricsData | |
| .sort((a, b) => (a.model > b.model ? -1 : 1)) | |
| .map((entry) => { | |
| // Step 1: Find metric under consideration | |
| const metric = metrics.find( | |
| (m) => m.displayName === entry.metric, | |
| ); | |
| // Step 2: Calculate normalized score | |
| let normalizedScore = entry.score; | |
| if ( | |
| metric?.minValue !== undefined && | |
| metric.maxValue !== undefined | |
| ) { | |
| // Step 2.a: Fetch minimum value | |
| const minValue = | |
| typeof metric.minValue === 'number' | |
| ? metric.minValue | |
| : castToNumber( | |
| metric.minValue?.value, | |
| metric.values, | |
| ); | |
| // Step 2.b: Fetch maximum value | |
| const maxValue = | |
| typeof metric.maxValue === 'number' | |
| ? metric.maxValue | |
| : castToNumber( | |
| metric.maxValue?.value, | |
| metric.values, | |
| ); | |
| // Step 2.c: Calculate min-max normalized score | |
| normalizedScore = | |
| Math.round( | |
| ((entry.score - minValue) / | |
| (maxValue - minValue)) * | |
| 100, | |
| ) / 100; | |
| } | |
| // Step 3: Return | |
| return { | |
| group: entry.model, | |
| key: entry.metric, | |
| value: normalizedScore, | |
| }; | |
| })} | |
| options={{ | |
| axes: { | |
| left: { | |
| mapsTo: 'value', | |
| }, | |
| bottom: { | |
| mapsTo: 'key', | |
| scaleType: ScaleTypes.LABELS, | |
| }, | |
| }, | |
| width: `${Math.round(WindowWidth * 0.45)}px`, | |
| height: `${Math.round(WindowHeight * 0.5)}px`, | |
| toolbar: { | |
| enabled: false, | |
| }, | |
| color: { | |
| scale: modelColors, | |
| }, | |
| legend: { | |
| order: modelOrder, | |
| }, | |
| theme: theme, | |
| }} | |
| ></GroupedBarChart> | |
| </> | |
| ) : ( | |
| <> | |
| <RadarChart | |
| data={algorithmicMetricsData.map((entry) => { | |
| const metric = metrics.find( | |
| (m) => m.displayName === entry.metric, | |
| ); | |
| return { | |
| model: entry.model, | |
| metric: entry.metric, | |
| score: | |
| metric && metric.maxValue | |
| ? Math.round( | |
| (entry.score / | |
| (typeof metric.maxValue === 'number' | |
| ? metric.maxValue | |
| : castToNumber( | |
| metric.maxValue?.value, | |
| metric.values, | |
| ))) * | |
| 100, | |
| ) / 100 | |
| : entry.score, | |
| }; | |
| })} | |
| options={{ | |
| radar: { | |
| alignment: Alignments.CENTER, | |
| axes: { | |
| angle: 'metric', | |
| value: 'score', | |
| }, | |
| }, | |
| data: { | |
| groupMapsTo: 'model', | |
| }, | |
| color: { | |
| scale: modelColors, | |
| }, | |
| legend: { | |
| alignment: Alignments.CENTER, | |
| order: modelOrder, | |
| }, | |
| width: `${Math.round(WindowWidth * 0.45)}px`, | |
| height: `${Math.round(WindowHeight * 0.5)}px`, | |
| toolbar: { | |
| enabled: false, | |
| }, | |
| theme: theme, | |
| }} | |
| ></RadarChart> | |
| </> | |
| )} | |
| </div> | |
| </div> | |
| ) : null} | |
| {humanMetricsInData.size === 0 && | |
| algorithmicmetricsInData.size === 0 ? ( | |
| <div className={classes.warningContainer}> | |
| <WarningAlt | |
| height={'32px'} | |
| width={'32px'} | |
| className={classes.warningContainerIcon} | |
| /> | |
| <span className={classes.warningContainerText}> | |
| {`No matching evaluations found. ${!isEmpty(selectedFilters) ? 'Please try again by removing one or more additional filters.' : ''}`} | |
| </span> | |
| </div> | |
| ) : null} | |
| </div> | |
| </div> | |
| ); | |
| } | |