Spaces:
Running
Running
Download scripts/import_benchmark_data.py from TabBioMed/leaderboard: direct link, hf CLI and curl.
- Browser
- Download file 31.3 kB
-
https://huggingface.co/spaces/TabBioMed/leaderboard/resolve/main/scripts/import_benchmark_data.py
- Command line
-
hf download hf://spaces/TabBioMed/leaderboard/scripts/import_benchmark_data.py
-
curl -L -o import_benchmark_data.py https://huggingface.co/spaces/TabBioMed/leaderboard/resolve/main/scripts/import_benchmark_data.py
31.3 kB
| #!/usr/bin/env python3 | |
| """Build the static leaderboard payload using TabBioMed's paper analysis package.""" | |
| import argparse | |
| from collections import Counter | |
| from concurrent.futures import ProcessPoolExecutor | |
| from datetime import datetime, timezone | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| from scipy.stats import rankdata | |
| from tabbiomed.analysis.common import PRIMARY_METRIC, load_benchmark_models_config | |
| from tabbiomed.analysis.datasets import load_benchmark, load_arrays | |
| from tabbiomed.analysis.elo.missing import apply_missing_policy | |
| from tabbiomed.analysis.elo.splitwise import rate | |
| from tabbiomed.analysis.figures.elo_domains import ( | |
| MIN_BOOTSTRAP_GROUPS, | |
| cluster_rate, | |
| summarize as summarize_domains, | |
| ) | |
| from tabbiomed.analysis.figures.pareto import collect_timings, summarize_costs | |
| from tabbiomed.analysis.figures.source_dependence import load_groups, partition | |
| from tabbiomed.analysis.figures.win_rates import pairwise_matrix | |
| from tabbiomed.analysis.paths import CONFIG, PACKAGE, RESULTS, PROCESSED | |
| from tabbiomed.analysis.pretraining_overlap import pretraining_overlaps | |
| from tabbiomed.analysis.ranks import normalized_scores_with_failures | |
| CLASSIFICATION = {"binary", "classification"} | |
| ATLAS_SOURCE_LABELS = { | |
| "bioresponse": "Kaggle Predicting a Biological Response via OpenML 4134", | |
| "Cardiovascular-Disease-dataset": "Kaggle via TALENT / OpenML 45547", | |
| "CDC_Diabetes_Health_Indicators": "CDC BRFSS 2015, Alex Teboul’s Kaggle release via TALENT", | |
| "vep_pathogenic_coding": ( | |
| "InstaDeepAI Genomics Long-Range Benchmark, " | |
| "Coding Variant Pathogenicity Dataset (June 17, 2024 release)" | |
| ), | |
| } | |
| ATLAS_SOURCE_URLS = { | |
| "vep_pathogenic_coding": ( | |
| "https://huggingface.co/datasets/InstaDeepAI/genomics-long-range-benchmark" | |
| "/blob/84cfc69322404818d28fb80921b54d8edc39d39e" | |
| "/variant_effect_pathogenic/vep_pathogenic_coding.csv" | |
| ), | |
| } | |
| REPO = Path(__file__).resolve().parents[1] | |
| def finite(value): | |
| return float(value) if pd.notna(value) and np.isfinite(value) else None | |
| def ratings_table(frame, scoped=False): | |
| if scoped: | |
| frame = frame.rename( | |
| columns={ | |
| "model": "competitor", | |
| "elo_calibrated": "elo", | |
| "elo_cal_lo_95": "elo_lo_95", | |
| "elo_cal_hi_95": "elo_hi_95", | |
| } | |
| ) | |
| table = {} | |
| for row in frame.itertuples(): | |
| table[row.competitor] = ( | |
| float(row.elo), | |
| finite(row.elo_lo_95), | |
| finite(row.elo_hi_95), | |
| ) | |
| if len(table) != len(frame): | |
| raise ValueError("Duplicate competitor in Elo table") | |
| return table | |
| def match_table(frame, saved, reference, count, observation_groups): | |
| if saved is None: | |
| if observation_groups is not None: | |
| bootstrap = 200 if len(set(observation_groups.values())) >= MIN_BOOTSTRAP_GROUPS else 0 | |
| fitted = cluster_rate(frame, observation_groups, bootstrap=bootstrap, seed=42) | |
| return ratings_table(fitted, scoped=True) | |
| bootstrap = 200 if count >= 5 else 0 | |
| fitted, _, _ = rate(frame, bootstrap=bootstrap, seed=42, reference=reference) | |
| if not bootstrap: | |
| fitted[["elo_lo_95", "elo_hi_95"]] = np.nan | |
| return ratings_table(fitted) | |
| if set(frame.competitor) != set(saved): | |
| raise ValueError(f"Result roster differs from paper summary for {reference}") | |
| return saved | |
| def mode_result(raw, timings, models, selected, saved, observation_groups=None): | |
| reference = "RandomForest" if selected else "RandomForest::default" | |
| frame, _ = apply_missing_policy(raw, "rf_imputed", reference=reference) | |
| if frame.empty: | |
| return None | |
| if selected and frame.imputed.any(): | |
| # Selected Random Forest anchors Elo, but only its Default score fills gaps. | |
| defaults = raw[raw.model.eq("RandomForest")].set_index("dataset").default_metric | |
| imputed = frame.loc[frame.imputed, "dataset"].map(defaults) | |
| if not np.isfinite(imputed).all(): | |
| raise ValueError("Random Forest Default is missing for an imputed task") | |
| frame.loc[frame.imputed, "metric_value"] = imputed | |
| count = frame.dataset.nunique() | |
| elo = match_table(frame, saved, reference, count, observation_groups) | |
| scores = raw[raw.dataset.isin(frame.dataset) & raw.competitor.isin(elo)] | |
| tasks = scores.groupby("dataset").task.first() | |
| matrix = scores.pivot(index="dataset", columns="competitor", values="metric_value") | |
| normalized = normalized_scores_with_failures(matrix, tasks) | |
| ranks = pd.DataFrame( | |
| rankdata(-normalized, axis=1), index=matrix.index, columns=matrix.columns | |
| ) | |
| oriented = matrix.mul(np.where(tasks.eq("binary"), 1, -1), axis=0) | |
| values = oriented.to_numpy() | |
| # A completed result beats a failure. Two failures count as a tie. | |
| left = np.where(np.isnan(values), -np.inf, values)[:, :, None] | |
| right = left.transpose(0, 2, 1) | |
| wins = (left > right) + 0.5 * (left == right) | |
| mean_win = pd.Series( | |
| 100 * (wins.sum(axis=(0, 2)) - count / 2) / (count * (len(elo) - 1)), | |
| index=matrix.columns, | |
| ) | |
| timed = frame.merge( | |
| timings, | |
| on=["model", "dataset", "timing_variant"], | |
| how="left", | |
| validate="many_to_one", | |
| ) | |
| timed["timing_imputed"] = timed.imputed | |
| # In the paper's Pareto tables, imputed scores use the corresponding RF timing. | |
| rf_times = timings[ | |
| timings.model.eq("RandomForest") & timings.timing_variant.eq("default") | |
| ].set_index("dataset") | |
| fields = ["train_per_1k", "inference_per_1k", "final_fit_per_1k"] | |
| for field in fields: | |
| timed.loc[timed.imputed, field] = timed.loc[timed.imputed, "dataset"].map( | |
| rf_times[field] | |
| ) | |
| elo_frame = pd.DataFrame( | |
| [{"competitor": key, "elo": val[0]} for key, val in elo.items()] | |
| ) | |
| train = summarize_costs( | |
| timed, elo_frame, "train_per_1k", "x_median_train_per_1k" | |
| ).set_index("competitor") | |
| inference = summarize_costs( | |
| timed, elo_frame, "inference_per_1k", "x_median_inference_per_1k" | |
| ).set_index("competitor") | |
| entries = [] | |
| ordered = sorted(elo, key=lambda key: (-elo[key][0], key.casefold(), key)) | |
| for index, key in enumerate(ordered, 1): | |
| model, _, variant = key.partition("::") | |
| variant = variant or "selected" | |
| score, low, high = elo[key] | |
| stats = timed[timed.competitor.eq(key)] | |
| n_completed = int(matrix[key].notna().sum()) | |
| paired = not selected and all( | |
| f"{model}::{v}" in elo for v in ("default", "tuned") | |
| ) | |
| entries.append( | |
| dict( | |
| key=key, | |
| modelId=model, | |
| variant=variant, | |
| family=models[model]["family"], | |
| hpoEnabled=models[model]["hpoEnabled"], | |
| classificationOnly=models[model]["classificationOnly"], | |
| paretoColor=models[model]["paretoColor"], | |
| rank=index, | |
| elo=score, | |
| displayName=model if selected else f"{model} ({variant.title()})", | |
| eloLo=low, | |
| eloHi=high, | |
| tunedMinusDefaultElo=( | |
| elo[f"{model}::tuned"][0] - elo[f"{model}::default"][0] | |
| ) | |
| if paired | |
| else None, | |
| meanWinRate=float(mean_win[key]), | |
| normScoreMean=float(normalized[key].mean()), | |
| normScoreMedian=float(normalized[key].median()), | |
| meanRank=float(ranks[key].mean()), | |
| rankMin=float(ranks[key].min()), | |
| rankMax=float(ranks[key].max()), | |
| improvabilityPct=float(100 * (1 - normalized[key].mean())), | |
| nCompleted=n_completed, | |
| nImputed=int(stats.imputed.sum()), | |
| nDefaultFallback=int(stats.metric_source.eq("default_fallback").sum()), | |
| nDatasets=count, | |
| completionPct=100 * n_completed / count, | |
| medianTrainPer1k=finite(train.at[key, "x_median_train_per_1k"]) | |
| if key in train.index | |
| else None, | |
| medianInferPer1k=finite(inference.at[key, "x_median_inference_per_1k"]) | |
| if key in inference.index | |
| else None, | |
| medianFinalFitPer1k=finite(stats.final_fit_per_1k.median()), | |
| isParetoTrain=bool(train.at[key, "is_pareto"]) | |
| if key in train.index | |
| else False, | |
| isParetoInfer=bool(inference.at[key, "is_pareto"]) | |
| if key in inference.index | |
| else False, | |
| ) | |
| ) | |
| result = { | |
| "nDatasets": count, | |
| "datasetIds": sorted(frame.dataset.unique()), | |
| "entries": entries, | |
| } | |
| if selected: | |
| participants = sorted(elo, key=str.casefold) | |
| dataset_meta = scores[["dataset", "task"]].drop_duplicates() | |
| win_matrix, _, pairs, _ = pairwise_matrix(scores, dataset_meta, participants) | |
| result["winRate"] = { | |
| "models": participants, | |
| "matrix": [ | |
| [finite(value) for value in row] for row in win_matrix.to_numpy() | |
| ], | |
| "counts": { | |
| f"{r.model_a}__{r.model_b}": [ | |
| r.wins, | |
| r.losses, | |
| r.ties, | |
| r.wins_on_score, | |
| r.wins_by_completion, | |
| ] | |
| for r in pairs.itertuples() | |
| }, | |
| } | |
| return result | |
| def scope_result(job): | |
| key, label, datasets, raw, paired, timings, models, saved = job | |
| names = {d["id"] for d in datasets} | |
| classification_only = ( | |
| key == "global_classification" | |
| or key.startswith(("domain_cls:", "quadrant_cls:")) | |
| or key in {"task:binary", "task:classification"} | |
| ) | |
| roster = { | |
| name: model | |
| for name, model in models.items() | |
| if classification_only or not model["classificationOnly"] | |
| } | |
| raw = raw[raw.dataset.isin(names) & raw.model.isin(roster)] | |
| paired = paired[paired.dataset.isin(names) & paired.model.isin(roster)] | |
| if raw.shape[0] != len(datasets) * len(roster): | |
| raise ValueError(f"Incomplete selected-result grid for {key}") | |
| block = dict( | |
| key=key, | |
| label=label, | |
| nDatasets=len(datasets), | |
| nModels=len(roster), | |
| datasetIds=sorted(names), | |
| samplesMin=min(d["numSamples"] for d in datasets), | |
| samplesMax=max(d["numSamples"] for d in datasets), | |
| featuresMin=min(d["numFeatures"] for d in datasets), | |
| featuresMax=max(d["numFeatures"] for d in datasets), | |
| ) | |
| for view, results in (("selected", raw), ("paired", paired)): | |
| block[view] = mode_result( | |
| results, timings, roster, view == "selected", saved.get(view) | |
| ) | |
| retained = names - set(pretraining_overlaps()["all"]) | |
| block["nDatasetsNoOverlap"] = len(retained) | |
| if retained and retained != names: | |
| cluster_scope = ( | |
| key.startswith("panel:") | |
| or (key.startswith("domain:") and key.count(":") == 1) | |
| ) | |
| observation_groups = ( | |
| {d["id"]: d["observationGroup"] for d in datasets if d["id"] in retained} | |
| if cluster_scope else None | |
| ) | |
| for view, results in (("selected", raw), ("paired", paired)): | |
| block[f"{view}NoOverlap"] = mode_result( | |
| results[results.dataset.isin(retained)], | |
| timings, | |
| roster, | |
| view == "selected", | |
| None, | |
| observation_groups=observation_groups if view == "selected" else None, | |
| ) | |
| return key, block | |
| def scopes_for(datasets, domains, classification_quadrants): | |
| def entries(key, label, predicate): | |
| return key, label, [d for d in datasets if predicate(d)] | |
| is_classification = lambda d: d["task"] in CLASSIFICATION | |
| has_high_missingness = lambda d: d["missingCellFraction"] >= 0.01 | |
| yield entries( | |
| "global_all_tasks", "All Biomedical Tasks (21 Models)", lambda d: True | |
| ) | |
| yield entries( | |
| "global_classification", | |
| "Classification Tasks (All 23 Models)", | |
| is_classification, | |
| ) | |
| for domain in domains: | |
| slug, label = domain["slug"], domain["fullName"] | |
| yield entries(f"domain:{slug}", label, lambda d: d["domainSlug"] == slug) | |
| yield entries( | |
| f"domain_cls:{slug}", | |
| f"{label} (Classification)", | |
| lambda d: d["domainSlug"] == slug and is_classification(d), | |
| ) | |
| for flag, name in ((True, "with_missing"), (False, "no_missing")): | |
| yield entries( | |
| f"domain:{slug}:{name}", | |
| f"{label} · {name.replace('_', ' ')} values", | |
| lambda d: d["domainSlug"] == slug and d["hasMissing"] == flag, | |
| ) | |
| yield entries( | |
| f"domain:{slug}:high_missing", | |
| f"{label} · At Least 1% Missing Cells", | |
| lambda d: d["domainSlug"] == slug and has_high_missingness(d), | |
| ) | |
| for suffix, label in (("up_to_2000", "≤ 2,000"), ("over_2000", "> 2,000")): | |
| key = f"ehr_{suffix}" | |
| yield entries( | |
| f"panel:{key}", | |
| f"Clinical & Healthcare ({label} Samples)", | |
| lambda d: d["clinicalSubpanel"] == key, | |
| ) | |
| for key, label, predicate in ( | |
| ( | |
| "missing:with_missing", | |
| "Datasets with Missing Values", | |
| lambda d: d["hasMissing"], | |
| ), | |
| ( | |
| "missing:no_missing", | |
| "Complete Datasets (0% Missing)", | |
| lambda d: not d["hasMissing"], | |
| ), | |
| ( | |
| "missing:high_missing", | |
| "At Least 1% Missing Cells", | |
| has_high_missingness, | |
| ), | |
| ( | |
| "wide:over_2000", | |
| "Wide Datasets (> 2,000 Features)", | |
| lambda d: d["numFeatures"] > 2000, | |
| ), | |
| ( | |
| "cat:has_categorical", | |
| "Datasets with Categorical Features", | |
| lambda d: d["numCategorical"] > 0, | |
| ), | |
| ( | |
| "cat:numeric_only", | |
| "Purely Numerical Features", | |
| lambda d: d["numCategorical"] == 0, | |
| ), | |
| ): | |
| yield entries(key, label, predicate) | |
| for task, label in ( | |
| ("binary", "Binary Classification"), | |
| ("classification", "Multiclass Classification"), | |
| ("regression", "Single-Target Regression"), | |
| ("multiregression", "Multi-Output Regression"), | |
| ): | |
| yield entries(f"task:{task}", label, lambda d: d["task"] == task) | |
| for quad in ("Q1", "Q2", "Q3", "Q4"): | |
| label = next(d["quadrantLabel"] for d in datasets if d["quadrant"] == quad) | |
| yield entries(f"quadrant:{quad}", label, lambda d: d["quadrant"] == quad) | |
| yield entries( | |
| f"quadrant_cls:{quad}", | |
| f"{label} (Classification)", | |
| lambda d: classification_quadrants.get(d["id"]) == quad, | |
| ) | |
| def dataset_metadata(read): | |
| domains = read("elo_domains/datasets.csv")[ | |
| ["dataset", "domain", "domain_slug", "observation_group"] | |
| ] | |
| regimes = read("elo_regimes/all_tasks_datasets.csv")[ | |
| ["dataset", "task", "num_samples", "num_features", "quadrant"] | |
| ] | |
| return domains.merge(regimes, on="dataset", validate="one_to_one") | |
| def missingness(x): | |
| seen = np.zeros(x.shape[1], dtype=bool) | |
| missing_cells = 0 | |
| step = max(1, 8_000_000 // x.shape[1]) | |
| for start in range(0, len(x), step): | |
| chunk = x[start : start + step] | |
| mask = ( | |
| np.isnan(chunk) | |
| if np.issubdtype(x.dtype, np.number) | |
| else (pd.isna(chunk) | (chunk == "")) | |
| ) | |
| seen |= mask.any(axis=0) | |
| missing_cells += int(mask.sum()) | |
| return missing_cells / x.size, int(seen.sum()) | |
| def dataset_records(metadata, catalog, known_missingness): | |
| _, source_groups, _ = load_groups(set(metadata.dataset)) | |
| families = partition(source_groups, set(metadata.dataset)) | |
| records = [] | |
| for row in metadata.sort_values("dataset").itertuples(): | |
| md = catalog[row.dataset]["metadata"] | |
| source_url = ATLAS_SOURCE_URLS.get(row.dataset) | |
| if (row.num_samples, row.num_features, row.task) != ( | |
| md["num_samples"], | |
| md["num_features"], | |
| md["task"], | |
| ): | |
| raise ValueError( | |
| f"Dataset metadata does not match the release: {row.dataset}" | |
| ) | |
| missing_fraction, missing_features = 0.0, 0 | |
| expected_missing = known_missingness.get(row.dataset) | |
| if expected_missing is None or expected_missing: | |
| x, _ = load_arrays(row.dataset, PROCESSED) | |
| missing_fraction, missing_features = missingness(x) | |
| if expected_missing is not None and not np.isclose( | |
| expected_missing, missing_fraction | |
| ): | |
| raise ValueError( | |
| f"Missingness does not match released data: {row.dataset}" | |
| ) | |
| num_categorical = len(md.get("cat_idx", [])) | |
| split_strategy = "cv" if md.get("split_strategy") == "cv" else "fixed" | |
| records.append( | |
| dict( | |
| id=row.dataset, | |
| domainSlug=row.domain_slug, | |
| domain=row.domain, | |
| clinicalSubpanel=( | |
| "ehr_up_to_2000" if row.num_samples <= 2000 else "ehr_over_2000" | |
| ) | |
| if row.domain_slug == "ehr" | |
| else None, | |
| observationGroup=row.observation_group, | |
| sourceFamily=families[row.dataset], | |
| task=row.task, | |
| primaryMetric=PRIMARY_METRIC[row.task], | |
| higherIsBetter=row.task == "binary", | |
| numSamples=row.num_samples, | |
| numFeatures=row.num_features, | |
| numCategorical=num_categorical, | |
| catFraction=num_categorical / row.num_features, | |
| numClasses=md.get("num_classes"), | |
| imbalanceRatio=md.get("imbalance_ratio"), | |
| nFolds=5 if split_strategy == "cv" else 1, | |
| splitStrategy=split_strategy, | |
| quadrant=row.quadrant.split()[0], | |
| quadrantLabel=row.quadrant, | |
| missingCellFraction=missing_fraction, | |
| missingFeatureCount=missing_features, | |
| hasMissing=bool(missing_features), | |
| hasZeroCodedMissingNote=row.dataset == "pima_diabetes", | |
| description=md.get("description", ""), | |
| sourceShort=ATLAS_SOURCE_LABELS.get( | |
| row.dataset, md.get("source", row.domain) | |
| ), | |
| license=md.get("license"), | |
| licenseUrl=md.get("license_url"), | |
| **({"sourceUrl": source_url} if source_url else {}), | |
| ) | |
| ) | |
| return records | |
| def saved_ratings(read): | |
| by_scope = { | |
| "global_all_tasks": { | |
| "paired": ratings_table(read("elo/rf_imputed/elo_all_entries.csv")), | |
| }, | |
| } | |
| paper_domains = read("elo_domains/summary.csv") | |
| try: | |
| clinical = read("elo_domains/clinical_all_summary.csv") | |
| except FileNotFoundError: | |
| metadata = read("elo_domains/datasets.csv") | |
| clinical_datasets = metadata[metadata.domain_slug.eq("ehr")].assign(panel="ehr") | |
| inputs = read("elo_domains/inputs.csv") | |
| clinical, _ = summarize_domains( | |
| inputs, | |
| clinical_datasets, | |
| sorted(inputs.model.unique()), | |
| bootstrap=200, | |
| seed=0, | |
| ) | |
| domain = pd.concat([paper_domains, clinical]) | |
| for name, group in domain.groupby("panel"): | |
| prefix = "panel" if name.startswith("ehr_") else "domain" | |
| by_scope[f"{prefix}:{name}"] = {"selected": ratings_table(group, scoped=True)} | |
| for case, prefix in (("all_tasks", "quadrant"), ("classification", "quadrant_cls")): | |
| for name, group in read(f"elo_regimes/{case}_summary.csv").groupby("quadrant"): | |
| by_scope[f"{prefix}:{name.split()[0]}"] = { | |
| "selected": ratings_table(group, scoped=True) | |
| } | |
| for name in ("with_missing", "no_missing"): | |
| for mode in ("paired", "selected"): | |
| try: | |
| frame = read(f"elo_missingness/{name}_{mode}.csv") | |
| except FileNotFoundError: | |
| continue | |
| by_scope.setdefault(f"missing:{name}", {})[mode] = ratings_table(frame) | |
| return by_scope | |
| def metric_inputs(raw): | |
| paired = pd.concat( | |
| [ | |
| raw.assign(variant="default", metric_value=raw.default_metric), | |
| raw[raw.hpo_enabled].assign( | |
| variant="tuned", metric_value=raw.loc[raw.hpo_enabled, "tuned_metric"] | |
| ), | |
| ], | |
| ignore_index=True, | |
| ) | |
| paired["competitor"] = paired.model + "::" + paired.variant | |
| paired["timing_variant"] = paired.variant | |
| paired["metric_source"] = "variant" | |
| paired["fold"] = 0 | |
| return paired | |
| def timing_inputs(paired, raw, plan): | |
| imputed, _ = apply_missing_policy(paired, "rf_imputed") | |
| selections = raw[["model", "dataset", "report_sha256"]].to_dict("records") | |
| frame, folds = collect_timings(imputed, selections, plan) | |
| fitted = folds.assign( | |
| final_fit_per_1k=1000 * folds.final_fit_seconds / folds.n_fitted | |
| ) | |
| final_fit = ( | |
| fitted.groupby(["model", "dataset", "variant"]) | |
| .final_fit_per_1k.agg( | |
| lambda values: values.mean() if np.isfinite(values).all() else np.nan | |
| ) | |
| .reset_index() | |
| ) | |
| fields = ["model", "dataset", "variant", "train_per_1k", "inference_per_1k"] | |
| return ( | |
| frame.loc[~frame.imputed, fields] | |
| .merge(final_fit, on=fields[:3]) | |
| .rename(columns={"variant": "timing_variant"}) | |
| ) | |
| def model_records(config, plan, raw, failures): | |
| colors = json.loads((CONFIG / "pareto_colors.json").read_text()) | |
| models = {} | |
| for name in sorted(plan["models"], key=str.casefold): | |
| rows = raw[raw.model.eq(name)] | |
| failed = failures[failures.model.eq(name) & ~failures.completed] | |
| pairs = rows.dropna(subset=["default_metric", "tuned_metric"]) | |
| better = np.where(pairs.task.eq("binary"), 1, -1) * ( | |
| pairs.tuned_metric - pairs.default_metric | |
| ) | |
| models[name] = dict( | |
| id=name, | |
| family=config.model_family[name], | |
| hpoEnabled=plan["models"][name]["hpo_enabled"], | |
| classificationOnly=name in config.regression_optional_missing_models, | |
| paretoColor=colors.get(name, "#64748b"), | |
| failureSummary=dict( | |
| defaultFailed=int(failed.variant.eq("default").sum()), | |
| tunedFailed=int(failed.variant.eq("tuned").sum()), | |
| usable=int(rows.metric_value.notna().sum()), | |
| totalApplicable=len(rows), | |
| causes=dict(Counter(failed.cause.fillna("unknown"))), | |
| ), | |
| hpoComparison=dict( | |
| pairedCount=len(pairs), | |
| tunedBetter=int((better > 0).sum()), | |
| tie=int((better == 0).sum()), | |
| defaultBetter=int((better < 0).sum()), | |
| ), | |
| ) | |
| return models | |
| def evaluation_records(raw, timings, failures, models, datasets): | |
| clocks = timings.set_index(["model", "dataset", "timing_variant"]) | |
| causes = ( | |
| failures[~failures.completed].set_index(["model", "dataset", "variant"]).cause | |
| ) | |
| model_indices = {name: i for i, name in enumerate(models)} | |
| dataset_indices = {d["id"]: i for i, d in enumerate(datasets)} | |
| records = [] | |
| for row in raw.itertuples(): | |
| def clock(variant, field): | |
| key = (row.model, row.dataset, variant) | |
| return finite(clocks.at[key, field]) if key in clocks.index else None | |
| fail = [causes.get((row.model, row.dataset, v)) for v in ("default", "tuned")] | |
| fail = {c for c in fail if pd.notna(c)} | |
| records.append( | |
| dict( | |
| m=model_indices[row.model], | |
| d=dataset_indices[row.dataset], | |
| defScore=finite(row.default_metric), | |
| tunScore=finite(row.tuned_metric), | |
| selScore=finite(row.metric_value), | |
| selVar=row.selected_variant, | |
| src=row.metric_source, | |
| defTrain1k=clock("default", "train_per_1k"), | |
| defInfer1k=clock("default", "inference_per_1k"), | |
| tunTrain1k=clock("tuned", "train_per_1k"), | |
| tunInfer1k=clock("tuned", "inference_per_1k"), | |
| selFit1k=clock(row.selected_variant, "final_fit_per_1k"), | |
| failCause=("mixed" if len(fail) > 1 else next(iter(fail), None)), | |
| ) | |
| ) | |
| return records | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument( | |
| "--summaries", | |
| type=Path, | |
| required=True, | |
| help="Generated paper CSV output directory", | |
| ) | |
| parser.add_argument( | |
| "--output", type=Path, default=REPO / "public/data/benchmarkData.json" | |
| ) | |
| parser.add_argument("--workers", type=int, default=4) | |
| args = parser.parse_args() | |
| def read(name): | |
| return pd.read_csv(args.summaries / name, float_precision="round_trip") | |
| plan = load_benchmark() | |
| config = load_benchmark_models_config() | |
| catalog = json.loads((PACKAGE / "configs/datasets.json").read_text())["datasets"] | |
| metadata = dataset_metadata(read) | |
| try: | |
| known_missingness = ( | |
| read("datasets_metadata.csv") | |
| .set_index("dataset") | |
| .missing_cell_fraction.to_dict() | |
| ) | |
| except FileNotFoundError: | |
| known_missingness = {} | |
| if metadata.dataset.duplicated().any() or set(metadata.dataset) != set( | |
| plan["datasets"] | |
| ): | |
| raise ValueError("Dataset summary scope does not match the paper plan") | |
| raw_main = read("model_summary/selected_results.csv").drop( | |
| columns=["train_per_1k", "inference_per_1k"] | |
| ) | |
| raw_extra = read("elo_regimes/classification_inputs.csv") | |
| raw_extra = raw_extra[ | |
| raw_extra.model.isin(config.regression_optional_missing_models) | |
| ].copy() | |
| raw_extra["metric_value"] = raw_extra.raw_metric | |
| raw_extra.loc[raw_extra.imputed, "metric_source"] = "failed" | |
| raw = pd.concat( | |
| [raw_main, raw_extra.reindex(columns=raw_main.columns)], ignore_index=True | |
| ) | |
| raw["variant"] = "selected" | |
| raw["competitor"] = raw.model | |
| raw["fold"] = 0 | |
| raw["timing_variant"] = raw.selected_variant | |
| paired = metric_inputs(raw) | |
| failures = read("failures/variant_completion.csv") | |
| models = model_records(config, plan, raw, failures) | |
| print("Validating consolidated reports and timings", flush=True) | |
| timings = timing_inputs(paired, raw, plan) | |
| print("Reading release dataset metadata", flush=True) | |
| datasets = dataset_records(metadata, catalog, known_missingness) | |
| metadata["has_missing"] = metadata.dataset.map( | |
| {d["id"]: d["hasMissing"] for d in datasets} | |
| ) | |
| domains = [] | |
| domain_failures = [] | |
| for slug, subset in metadata.groupby("domain_slug", sort=True): | |
| names = set(subset.dataset) | |
| name = str(subset.domain.iloc[0]) | |
| domains.append( | |
| dict( | |
| slug=slug, | |
| name="Drug Discovery" if slug == "drug" else name, | |
| fullName=name, | |
| nDatasets=len(subset), | |
| nWithMissing=int(subset.has_missing.sum()), | |
| samplesMin=int(subset.num_samples.min()), | |
| samplesMax=int(subset.num_samples.max()), | |
| featuresMin=int(subset.num_features.min()), | |
| featuresMax=int(subset.num_features.max()), | |
| taskCounts=dict(Counter(subset.task)), | |
| ) | |
| ) | |
| applicable = failures[failures.dataset.isin(names)] | |
| failed = applicable[~applicable.completed] | |
| causes = Counter(failed.cause) | |
| domain_failures.append( | |
| dict( | |
| domain=name, | |
| domainSlug=slug, | |
| nExpected=len(applicable), | |
| nFailed=len(failed), | |
| **{ | |
| k: causes[k] | |
| for k in ( | |
| "gpu_oom", | |
| "host_ram_oom", | |
| "time_limit", | |
| "segmentation_fault", | |
| "mixed", | |
| "unknown", | |
| ) | |
| }, | |
| ) | |
| ) | |
| classification_quadrants = ( | |
| read("elo_regimes/classification_datasets.csv") | |
| .set_index("dataset") | |
| .quadrant.str.split() | |
| .str[0] | |
| .to_dict() | |
| ) | |
| saved = saved_ratings(read) | |
| jobs = [ | |
| (key, label, subset, raw, paired, timings, models, saved.get(key, {})) | |
| for key, label, subset in scopes_for( | |
| datasets, domains, classification_quadrants | |
| ) | |
| if subset | |
| ] | |
| scopes = {} | |
| print("Building static benchmark scopes", flush=True) | |
| with ProcessPoolExecutor(max_workers=args.workers) as pool: | |
| for key, block in pool.map(scope_result, jobs): | |
| scopes[key] = block | |
| print(key, block["nDatasets"], flush=True) | |
| paper_elo = json.loads((args.summaries / "elo/summary.json").read_text()) | |
| if ( | |
| scopes["global_all_tasks"]["paired"]["nDatasets"] | |
| != paper_elo["rf_imputed"]["n_datasets"] | |
| ): | |
| raise ValueError("RF-imputed scope disagrees with the paper") | |
| report_identity = hashlib.sha256( | |
| "\n".join(sorted(raw.report_sha256)).encode() | |
| ).hexdigest() | |
| revision = json.loads((PACKAGE / "configs/data_source.json").read_text())[ | |
| "revision" | |
| ] | |
| output = dict( | |
| meta=dict( | |
| benchmarkName="TabBioMed", | |
| subtitle="Benchmark for Biomedical Tabular Learning", | |
| generatedAt=datetime.now(timezone.utc).isoformat(), | |
| resultsRevision=f"sha256:{report_identity}", | |
| dataRevision=revision, | |
| nDatasets=len(datasets), | |
| nDomains=len(domains), | |
| nModelsAllTasks=len(config.expected_models_regression), | |
| nModelsClassification=len(models), | |
| medLog10Samples=float(np.median(np.log10(metadata.num_samples))), | |
| medLog10Features=float(np.median(np.log10(metadata.num_features))), | |
| ), | |
| domains=domains, | |
| models=list(models.values()), | |
| datasets=datasets, | |
| evaluations=evaluation_records(raw, timings, failures, models, datasets), | |
| scopes=scopes, | |
| domainFailures=domain_failures, | |
| ) | |
| text = json.dumps( | |
| output, ensure_ascii=False, allow_nan=False, separators=(",", ":") | |
| ) | |
| target = args.output | |
| target.parent.mkdir(parents=True, exist_ok=True) | |
| staged = target.with_suffix(".tmp") | |
| staged.write_text(text + "\n", encoding="utf-8") | |
| staged.replace(target) | |
| print( | |
| f"Wrote {target}: {len(scopes)} scopes, {len(output['evaluations'])} evaluations" | |
| ) | |
| if __name__ == "__main__": | |
| main() | |