diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..7d42744ac28630f12f1103031d0c200647872b59 100644 --- a/.gitattributes +++ b/.gitattributes @@ -19,7 +19,6 @@ *.pb filter=lfs diff=lfs merge=lfs -text *.pickle filter=lfs diff=lfs merge=lfs -text *.pkl filter=lfs diff=lfs merge=lfs -text -*.pt filter=lfs diff=lfs merge=lfs -text *.pth filter=lfs diff=lfs merge=lfs -text *.rar filter=lfs diff=lfs merge=lfs -text *.safetensors filter=lfs diff=lfs merge=lfs -text @@ -33,3 +32,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +models/norman/model.ptw filter=lfs diff=lfs merge=lfs -text +models/replogle_k562/model.ptw filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..069e69b84f7542311b1b39f80859df6233cccc36 --- /dev/null +++ b/.gitignore @@ -0,0 +1,31 @@ +# large data & downloads (regenerated from public sources via src/data/preprocess.py) +/data/ +/gears_data/ +/logs/ + +# generated artifacts kept out of the repo +/paper/ +/figures/ +*.h5ad +*.npz +*.npy + +# model checkpoints / large binaries +*.pt +*.ckpt + +# python +__pycache__/ +*.pyc +*.pyo +.ipynb_checkpoints/ + +# latex build artifacts +paper/*.aux +paper/*.bbl +paper/*.blg +paper/*.out +paper/*.log + +# os +.DS_Store diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..230d745ccffce48280b9d23c985a763058fd484a --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Bryan Cheng + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 154df8298fab5ecf322016157858e08cd1bccbe1..a242d95d49a61f17341a0c2fdf2d5e83e832dcf4 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,90 @@ ---- -license: apache-2.0 ---- +# PIVOT + +Perturbation-Informed Vector-field Optimization for Transcriptomic state control. + +PIVOT learns a perturbation-conditioned flow map over single-cell state embeddings and +uses its Jacobian for differentiable inverse design: given a control cell state and a +desired target state, it nominates gene-level interventions that move cells toward the +target. The same model also does ordinary forward response prediction. + +## Layout + +``` +src/ + data/ loading + preprocessing of perturb-seq data, splits + models/ perturbation encoder, flow map, the PIVOT module + training/ training loop and losses + evaluation/ inference, rewards, metrics, baselines + experiments/ drivers for the result tables, ablations, figures + utils/ +scripts/ figure generation, extra ablations, GEARS comparison +experiments/ saved result json +``` + +## Setup + +```bash +pip install -r requirements.txt +``` + +Data is not committed. Download and preprocess from the public sources first (Norman 2019 +and Replogle 2022 are pulled from scPerturb): + +```bash +python -m src.data.preprocess norman +python -m src.data.preprocess replogle_k562 +``` + +This writes a PCA(50) embedding over 2000 highly variable genes plus the held-out splits to +`data/processed//`. + +## Running things + +```bash +# train one model +python -m src.training.train --dataset norman --split perturbation + +# forward + nomination tables +python -m src.experiments.run_tables --dataset norman --tables forward_cell forward_perturbation + +# ablations +python -m src.experiments.run_ablations --dataset norman + +# figures +python scripts/figures.py +``` + +The GEARS head-to-head runs in its own conda env (older torch + pyg), since the package is +finicky about versions: + +```bash +bash scripts/setup_gears_env.sh +conda run -n pivot_gears python scripts/gears_ranking.py +``` + +## Models + +Trained PIVOT checkpoints live under `models/`: + +- `models/norman/` - trained on Norman 2019 (CRISPRa K562) +- `models/replogle_k562/` - trained on Replogle 2022 (CRISPRi K562) + +each folder has `model.ptw` (a plain torch state dict), `config.json` (the training config), +and `train_info.json` (history + run info). loading needs the matching preprocessed dataset, +since the perturbation encoder vocabulary comes from the data: + +```python +import json, torch +from src.data.perturb_data import load_dataset +from src.training.train import TrainConfig, make_model + +cfg = TrainConfig(**json.load(open("models/norman/config.json"))) +data = load_dataset(cfg.dataset) +model = make_model(data, cfg, device="cpu") +model.load_state_dict(torch.load("models/norman/model.ptw", map_location="cpu")) +model.eval() +``` + +## License + +MIT, Bryan Cheng 2026. See `LICENSE`. diff --git a/experiments/results/gears_norman.json b/experiments/results/gears_norman.json new file mode 100644 index 0000000000000000000000000000000000000000..c9494d41a71bcc2d79be5916750b174d332c50fb --- /dev/null +++ b/experiments/results/gears_norman.json @@ -0,0 +1,56 @@ +{ + "n_test_perts": 45, + "test_perts": [ + "ARID1A+ctrl", + "ARRDC3+ctrl", + "ATL1+ctrl", + "BAK1+ctrl", + "BCL2L11+BAK1", + "BCL2L11+TGFBR2", + "BCL2L11+ctrl", + "BPGM+ZBTB1", + "C3orf72+FOXL2", + "CBFA2T3+ctrl", + "CBL+TGFBR2", + "CBL+ctrl", + "CEBPB+OSR2", + "CEBPE+PTPN12", + "CKS1B+ctrl", + "CNNM4+ctrl", + "DUSP9+IGDCC3", + "DUSP9+ctrl", + "EGR1+ctrl", + "ETS2+CEBPE", + "FOXA1+FOXF1", + "FOXF1+HOXB9", + "FOXF1+ctrl", + "GLB1L2+ctrl", + "HOXC13+ctrl", + "IGDCC3+PRTG", + "IRF1+SET", + "KLF1+CEBPA", + "KLF1+FOXA1", + "KMT2A+ctrl", + "MAP2K3+IKZF3", + "MAP2K3+MAP2K6", + "MAPK1+ctrl", + "NCL+ctrl", + "PLK4+STIL", + "PTPN12+UBASH3A", + "RREB1+ctrl", + "SNAI1+DLX2", + "TBX3+TBX2", + "TGFBR2+ETS2", + "TGFBR2+ctrl", + "TSC22D1+ctrl", + "UBASH3B+PTPN12", + "ZBTB10+DLX2", + "ZNF318+ctrl" + ], + "metrics": { + "mse": 0.003074304576471655, + "mse_de": 0.10127744511701167, + "pearson": 0.9916384220123291, + "pearson_de": 0.8981071008576287 + } +} \ No newline at end of file diff --git a/experiments/results/gears_ranking.json b/experiments/results/gears_ranking.json new file mode 100644 index 0000000000000000000000000000000000000000..908c9847c978483ac79b77999644558c1c3b0b44 --- /dev/null +++ b/experiments/results/gears_ranking.json @@ -0,0 +1,19 @@ +{ + "single": { + "top1": 0.047619047619047616, + "top5": 0.19047619047619047, + "ndcg": 0.14115129481729458, + "rank": 21.666666666666668, + "n": 21, + "med_rank": 19.0 + }, + "combo": { + "top1": 0.038461538461538464, + "top5": 0.34615384615384615, + "overlap": 0.15384615384615385, + "ndcg": 0.26756681334907767, + "n": 26 + }, + "n_candidates_single": 101, + "n_candidates_combo": 128 +} \ No newline at end of file diff --git a/experiments/results/norman_ablation_components.json b/experiments/results/norman_ablation_components.json new file mode 100644 index 0000000000000000000000000000000000000000..9e7d28d3b1caf9a309b2f8d5625c3e0516c48444 --- /dev/null +++ b/experiments/results/norman_ablation_components.json @@ -0,0 +1,75 @@ +{ + "split": "perturbation", + "rows": { + "flow-map-only": { + "forward": { + "mse": 0.0017506419577291532, + "de_corr": 0.8443752184034662, + "mmd": 0.40636296707676617, + "pearson": 0.7724152516098098 + }, + "inverse": { + "top1": 0.09090909090909091, + "top5": 0.18181818181818182, + "ndcg": 0.1762888354358521, + "endpoint_dist": 0.9921409541910345 + } + }, + "no-tangent": { + "forward": { + "mse": 0.001731510258941079, + "de_corr": 0.8408393241722691, + "mmd": 0.37349285466271814, + "pearson": 0.7714462100349843 + }, + "inverse": { + "top1": 0.045454545454545456, + "top5": 0.18181818181818182, + "ndcg": 0.1446297312285945, + "endpoint_dist": 0.9842348342592065 + } + }, + "no-semigroup": { + "forward": { + "mse": 0.00195918270780586, + "de_corr": 0.8041895085826833, + "mmd": 0.35951861984482014, + "pearson": 0.721258614925628 + }, + "inverse": { + "top1": 0.0, + "top5": 0.09090909090909091, + "ndcg": 0.07627732654967648, + "endpoint_dist": 1.1208079132166775 + } + }, + "PIVOT-full": { + "forward": { + "mse": 0.0018839154465550042, + "de_corr": 0.8260913965232829, + "mmd": 0.2779389477818921, + "pearson": 0.7429934785641888 + }, + "inverse": { + "top1": 0.0, + "top5": 0.13636363636363635, + "ndcg": 0.0741331706168103, + "endpoint_dist": 1.10553578896956 + } + }, + "EndpointMLP": { + "forward": { + "mse": 0.0011545048934962362, + "de_corr": 0.6582605464661375, + "mmd": 0.048875581526963176, + "pearson": 0.6028436505572593 + }, + "inverse": { + "top1": 0.13636363636363635, + "top5": 0.22727272727272727, + "ndcg": 0.21410929644360008, + "endpoint_dist": 1.0358923240141436 + } + } + } +} \ No newline at end of file diff --git a/experiments/results/norman_ablation_cost.json b/experiments/results/norman_ablation_cost.json new file mode 100644 index 0000000000000000000000000000000000000000..358fc6b5fa8caaf9f66806b0a7f548c9c82175ce --- /dev/null +++ b/experiments/results/norman_ablation_cost.json @@ -0,0 +1,9 @@ +{ + "split": "perturbation", + "PIVOT": { + "params": 892658, + "train_time_s": 102.39015054702759, + "rank_time_per_query_s": 0.14307117462158203, + "candidate_evals": 105 + } +} \ No newline at end of file diff --git a/experiments/results/norman_ablation_datascale.json b/experiments/results/norman_ablation_datascale.json new file mode 100644 index 0000000000000000000000000000000000000000..5e87ade4950906d852e7fb296f0e43e120e66079 --- /dev/null +++ b/experiments/results/norman_ablation_datascale.json @@ -0,0 +1,75 @@ +{ + "split": "perturbation", + "rows": { + "0.1": { + "forward": { + "mse": 0.0026412838559097393, + "de_corr": 0.7419204415476068, + "mmd": 0.0816293043824765, + "pearson": 0.6466187456761427 + }, + "inverse": { + "top1": 0.0, + "top5": 0.0, + "ndcg": 0.04355759901977392, + "endpoint_dist": 1.2526228725910187 + } + }, + "0.25": { + "forward": { + "mse": 0.0025015621919165106, + "de_corr": 0.7626386221418989, + "mmd": 0.10973129114722474, + "pearson": 0.6617828040046895 + }, + "inverse": { + "top1": 0.0, + "top5": 0.0, + "ndcg": 0.015151515151499999, + "endpoint_dist": 1.2300338365814902 + } + }, + "0.5": { + "forward": { + "mse": 0.0020840686227026534, + "de_corr": 0.806516168916479, + "mmd": 0.167632126198037, + "pearson": 0.7073832176467205 + }, + "inverse": { + "top1": 0.0, + "top5": 0.045454545454545456, + "ndcg": 0.056701119364541815, + "endpoint_dist": 1.1461743387308987 + } + }, + "0.75": { + "forward": { + "mse": 0.002003367400128751, + "de_corr": 0.8197314882616294, + "mmd": 0.23520435415096233, + "pearson": 0.721792117633084 + }, + "inverse": { + "top1": 0.0, + "top5": 0.13636363636363635, + "ndcg": 0.07098210507469495, + "endpoint_dist": 1.1467380632053723 + } + }, + "1.0": { + "forward": { + "mse": 0.0018839154465550042, + "de_corr": 0.8260913965232829, + "mmd": 0.2779389477818921, + "pearson": 0.7429934785641888 + }, + "inverse": { + "top1": 0.0, + "top5": 0.13636363636363635, + "ndcg": 0.0741331706168103, + "endpoint_dist": 1.10553578896956 + } + } + } +} \ No newline at end of file diff --git a/experiments/results/norman_ablation_guidance_steps.json b/experiments/results/norman_ablation_guidance_steps.json new file mode 100644 index 0000000000000000000000000000000000000000..7747b7be3e9bec1ee25bfe16888b66f274d5f286 --- /dev/null +++ b/experiments/results/norman_ablation_guidance_steps.json @@ -0,0 +1,41 @@ +{ + "split": "perturbation", + "rows": { + "0": { + "top1": 0.0, + "top5": 0.13636363636363635, + "ndcg": 0.0741331706168103, + "endpoint_dist": 1.10553578896956 + }, + "5": { + "top1": 0.0, + "top5": 0.13636363636363635, + "ndcg": 0.08466528669577132, + "endpoint_dist": 1.1852125579660588 + }, + "10": { + "top1": 0.0, + "top5": 0.13636363636363635, + "ndcg": 0.0932238333390342, + "endpoint_dist": 1.1176448301835493 + }, + "25": { + "top1": 0.0, + "top5": 0.13636363636363635, + "ndcg": 0.09061663913083162, + "endpoint_dist": 1.2133627290075475 + }, + "50": { + "top1": 0.045454545454545456, + "top5": 0.13636363636363635, + "ndcg": 0.11054362496513667, + "endpoint_dist": 1.146642354401675 + }, + "100": { + "top1": 0.045454545454545456, + "top5": 0.13636363636363635, + "ndcg": 0.0968604433440603, + "endpoint_dist": 1.168973827903921 + } + } +} \ No newline at end of file diff --git a/experiments/results/norman_ablation_inverse_search.json b/experiments/results/norman_ablation_inverse_search.json new file mode 100644 index 0000000000000000000000000000000000000000..0356835d67c6f98d554f36dfc54f3d03f94cfcc4 --- /dev/null +++ b/experiments/results/norman_ablation_inverse_search.json @@ -0,0 +1,41 @@ +{ + "split": "perturbation", + "rows": { + "ranking-only": { + "top1": 0.0, + "top5": 0.13636363636363635, + "ndcg": 0.0741331706168103, + "endpoint_dist": 1.10553578896956 + }, + "random-init-opt": { + "top1": 0.09090909090909091, + "top5": 0.18181818181818182, + "ndcg": 0.13006150527926927, + "endpoint_dist": 1.916646886955608 + }, + "mean-top-init": { + "top1": 0.09090909090909091, + "top5": 0.13636363636363635, + "ndcg": 0.11048529809413464, + "endpoint_dist": 1.633959797295657 + }, + "guidance-no-norm": { + "top1": 0.0, + "top5": 0.18181818181818182, + "ndcg": 0.10475755936295951, + "endpoint_dist": 2.176489320668307 + }, + "guidance-norm": { + "top1": 0.045454545454545456, + "top5": 0.13636363636363635, + "ndcg": 0.10540057074853007, + "endpoint_dist": 1.6980433003468947 + }, + "guidance+rerank": { + "top1": 0.0, + "top5": 0.13636363636363635, + "ndcg": 0.09061663913083162, + "endpoint_dist": 1.2133627290075475 + } + } +} \ No newline at end of file diff --git a/experiments/results/norman_ablation_matching.json b/experiments/results/norman_ablation_matching.json new file mode 100644 index 0000000000000000000000000000000000000000..48fd03222b3e34b2021c537094ebfb3aebcc724b --- /dev/null +++ b/experiments/results/norman_ablation_matching.json @@ -0,0 +1,75 @@ +{ + "split": "perturbation", + "rows": { + "random": { + "forward": { + "mse": 0.0020935234107186105, + "de_corr": 0.8198527663629106, + "mmd": 0.286064224443859, + "pearson": 0.7269562229197076 + }, + "inverse": { + "top1": 0.0, + "top5": 0.13636363636363635, + "ndcg": 0.0689901164002037, + "endpoint_dist": 1.1582550325176932 + } + }, + "batch": { + "forward": { + "mse": 0.0018839154465550042, + "de_corr": 0.8260913965232829, + "mmd": 0.2779389477818921, + "pearson": 0.7429934785641888 + }, + "inverse": { + "top1": 0.0, + "top5": 0.13636363636363635, + "ndcg": 0.0741331706168103, + "endpoint_dist": 1.10553578896956 + } + }, + "celltype": { + "forward": { + "mse": 0.0020935234107186105, + "de_corr": 0.8198527663629106, + "mmd": 0.286064224443859, + "pearson": 0.7269562229197076 + }, + "inverse": { + "top1": 0.0, + "top5": 0.13636363636363635, + "ndcg": 0.0689901164002037, + "endpoint_dist": 1.1582550325176932 + } + }, + "batch_celltype": { + "forward": { + "mse": 0.0018839154465550042, + "de_corr": 0.8260913965232829, + "mmd": 0.2779389477818921, + "pearson": 0.7429934785641888 + }, + "inverse": { + "top1": 0.0, + "top5": 0.13636363636363635, + "ndcg": 0.0741331706168103, + "endpoint_dist": 1.10553578896956 + } + }, + "nearest": { + "forward": { + "mse": 0.0034554351654963846, + "de_corr": 0.6926855278459001, + "mmd": 0.08932766919879936, + "pearson": 0.5472117423060092 + }, + "inverse": { + "top1": 0.13636363636363635, + "top5": 0.13636363636363635, + "ndcg": 0.1363636363635, + "endpoint_dist": 1.462783862243999 + } + } + } +} \ No newline at end of file diff --git a/experiments/results/norman_ablation_representation.json b/experiments/results/norman_ablation_representation.json new file mode 100644 index 0000000000000000000000000000000000000000..b288996775d3739c2eb13ff373ab4beb7f101169 --- /dev/null +++ b/experiments/results/norman_ablation_representation.json @@ -0,0 +1,75 @@ +{ + "split": "perturbation", + "rows": { + "gene_op": { + "forward": { + "mse": 0.0018839154465550042, + "de_corr": 0.8260913965232829, + "mmd": 0.2779389477818921, + "pearson": 0.7429934785641888 + }, + "inverse": { + "top1": 0.0, + "top5": 0.13636363636363635, + "ndcg": 0.0741331706168103, + "endpoint_dist": 1.10553578896956 + } + }, + "op_only": { + "forward": { + "mse": 0.0036388238541474765, + "de_corr": 0.4861906795425618, + "mmd": 0.2997179725547505, + "pearson": 0.4692922436850185 + }, + "inverse": { + "top1": 0.0, + "top5": 0.18181818181818182, + "ndcg": 0.10475755936295951, + "endpoint_dist": 1.5618621165102178 + } + }, + "gene_only": { + "forward": { + "mse": 0.0019814494465963243, + "de_corr": 0.8269104006442618, + "mmd": 0.2864381163653205, + "pearson": 0.7344451238341788 + }, + "inverse": { + "top1": 0.0, + "top5": 0.045454545454545456, + "ndcg": 0.059209173521086636, + "endpoint_dist": 1.128328177061948 + } + }, + "random_id": { + "forward": { + "mse": 0.003925378121029427, + "de_corr": 0.4371013550999317, + "mmd": 0.3452091584109617, + "pearson": 0.4009652477887241 + }, + "inverse": { + "top1": 0.0, + "top5": 0.0, + "ndcg": 0.0, + "endpoint_dist": 1.181209458546205 + } + }, + "gene_pathway_op": { + "forward": { + "mse": 0.002060087917228923, + "de_corr": 0.8255276011818267, + "mmd": 0.2896023761246202, + "pearson": 0.7541268989443779 + }, + "inverse": { + "top1": 0.0, + "top5": 0.13636363636363635, + "ndcg": 0.09442383563302575, + "endpoint_dist": 1.1720866127447649 + } + } + } +} \ No newline at end of file diff --git a/experiments/results/norman_ablation_reward.json b/experiments/results/norman_ablation_reward.json new file mode 100644 index 0000000000000000000000000000000000000000..c30b94fe34d353f027f13c87ec3ca0e48753a6fd --- /dev/null +++ b/experiments/results/norman_ablation_reward.json @@ -0,0 +1,35 @@ +{ + "split": "perturbation", + "rows": { + "centroid": { + "top1": 0.0, + "top5": 0.13636363636363635, + "ndcg": 0.0741331706168103, + "endpoint_dist": 1.10553578896956 + }, + "cosine": { + "top1": 0.22727272727272727, + "top5": 0.2727272727272727, + "ndcg": 0.24485694578314335, + "endpoint_dist": 3.251927175305106 + }, + "nn_target": { + "top1": 0.0, + "top5": 0.045454545454545456, + "ndcg": 0.03273573366214339, + "endpoint_dist": 1.7553648054599762 + }, + "mmd": { + "top1": 0.0, + "top5": 0.18181818181818182, + "ndcg": 0.1380222478334554, + "endpoint_dist": 1.3466052819382062 + }, + "wasserstein": { + "top1": 0.045454545454545456, + "top5": 0.13636363636363635, + "ndcg": 0.1533429541697297, + "endpoint_dist": 1.222495680505579 + } + } +} \ No newline at end of file diff --git a/experiments/results/norman_benchmark.json b/experiments/results/norman_benchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..6a14ac3f748f0ccbcbf3755725944aad29158be0 --- /dev/null +++ b/experiments/results/norman_benchmark.json @@ -0,0 +1,231 @@ +{ + "dataset": "norman", + "n_targets": 22, + "n_candidates": 105, + "random_top5": 0.047619047619047616, + "forward": { + "PIVOT": { + "mse": 0.0017491921988697326, + "r2": -0.8161821530220357, + "pearson": 0.7693309462133874, + "spearman": 0.5815323970363715, + "de_corr": 0.8388409530545803, + "mmd": 0.3034484875795543, + "wasserstein": 0.43272001688923617 + }, + "PIVOT-OT": { + "mse": 0.0025015840371278054, + "r2": -2.4559457745957882, + "pearson": 0.6559229388516, + "spearman": 0.46955744461871557, + "de_corr": 0.7519170356557724, + "mmd": 0.1500954119146127, + "wasserstein": 0.6666752559372957 + }, + "MeanControl": { + "mse": 0.004455934176339144, + "r2": -0.05741366300177067, + "pearson": 0.06247465056862603, + "spearman": 0.019984302966237155, + "de_corr": 0.10322069715797663, + "mmd": 0.10706937820897701, + "wasserstein": 0.323807051495741 + }, + "AvgPerturbationEffect": { + "mse": 0.003602388566310339, + "r2": -1.6519504455809897, + "pearson": 0.4787262177213709, + "spearman": 0.3930456408465337, + "de_corr": 0.5168158494015324, + "mmd": 0.09934840288274557, + "wasserstein": 0.30081602373086835 + }, + "Additive": { + "mse": 0.0021094072577939865, + "r2": -1.448790396781678, + "pearson": 0.6326670402542074, + "spearman": 0.5108886765427764, + "de_corr": 0.7134229213474913, + "mmd": 0.07343746422286138, + "wasserstein": 0.25771137196436716 + }, + "LinearResponse": { + "mse": 0.0011306649979273293, + "r2": -0.253476581674941, + "pearson": 0.7451306042519021, + "spearman": 0.5799033801846452, + "de_corr": 0.8105884356067535, + "mmd": 0.047508293556133514, + "wasserstein": 0.20167587385465413 + }, + "NearestPerturbationCentroid": { + "mse": 0.003305152804774054, + "r2": -4.189406145126261, + "pearson": 0.6359215529358133, + "spearman": 0.505686299719842, + "de_corr": 0.7316346312773988, + "mmd": 0.10450971258729616, + "wasserstein": 0.30843307594994734 + }, + "EndpointMLP": { + "mse": 0.0011545048891616232, + "r2": 0.04933880491459623, + "pearson": 0.6028436519839662, + "spearman": 0.5096878649785176, + "de_corr": 0.6582605471002295, + "mmd": 0.048875581526963176, + "wasserstein": 0.19931416156968693 + }, + "kNN-latent": { + "mse": 0.0021908569667968857, + "r2": -1.5670528272365003, + "pearson": 0.6744848155674148, + "spearman": 0.5199807748887847, + "de_corr": 0.7707986322806236, + "mmd": 0.07314315796243004, + "wasserstein": 0.2532922264836747 + }, + "ConditionalMLP": { + "mse": 0.0010686125573788848, + "r2": 0.30849315257782634, + "pearson": 0.619310863127813, + "spearman": 0.5438978223783673, + "de_corr": 0.6658019708033572, + "mmd": 0.42855737667365007, + "wasserstein": 0.5315667629719005 + }, + "Random": { + "mse": 0.00673537037057921, + "r2": -6.333347964794077, + "pearson": 0.27716910228767294, + "spearman": 0.2415044605055355, + "de_corr": 0.28529975959595216, + "mmd": 0.16910403534817559, + "wasserstein": 0.3895185511116175 + }, + "PIVOT+dist": { + "mse": null, + "r2": -1.557, + "pearson": 0.759, + "spearman": null, + "de_corr": 0.844, + "mmd": 0.0507, + "wasserstein": 0.235 + } + }, + "inverse": { + "PIVOT": { + "top1": 0.18181818181818182, + "top5": 0.22727272727272727, + "ndcg": 0.237863170222463, + "func_top5": 0.8636363636363636, + "top5_ci": [ + 0.22727272727272727, + 0.045454545454545456, + 0.4090909090909091 + ] + }, + "PIVOT-guidance": { + "top1": 0.13636363636363635, + "top5": 0.22727272727272727, + "ndcg": 0.20036677015736315, + "func_top5": 0.8181818181818182 + }, + "MeanControl": { + "top1": 0.0, + "top5": 0.18181818181818182, + "ndcg": 0.10475755936295951, + "func_top5": 0.8181818181818182 + }, + "AvgPerturbationEffect": { + "top1": 0.0, + "top5": 0.18181818181818182, + "ndcg": 0.10475755936295951, + "func_top5": 0.8181818181818182 + }, + "Additive": { + "top1": 0.0, + "top5": 0.0, + "ndcg": 0.0, + "func_top5": 0.9545454545454546 + }, + "LinearResponse": { + "top1": 0.045454545454545456, + "top5": 0.13636363636363635, + "ndcg": 0.1028117957791206, + "func_top5": 0.8181818181818182 + }, + "NearestPerturbationCentroid": { + "top1": 0.18181818181818182, + "top5": 0.18181818181818182, + "ndcg": 0.181818181818, + "func_top5": 0.9090909090909091 + }, + "EndpointMLP": { + "top1": 0.045454545454545456, + "top5": 0.13636363636363635, + "ndcg": 0.13489139848118523, + "func_top5": 0.9545454545454546 + }, + "kNN-latent": { + "top1": 0.045454545454545456, + "top5": 0.09090909090909091, + "ndcg": 0.09305324684186617, + "func_top5": 0.7727272727272727 + }, + "ConditionalMLP": { + "top1": 0.18181818181818182, + "top5": 0.2727272727272727, + "ndcg": 0.2535147447237758, + "func_top5": 0.9090909090909091 + }, + "Random": { + "top1": 0.0, + "top5": 0.045454545454545456, + "ndcg": 0.03472772233663465, + "func_top5": 0.7727272727272727 + } + }, + "significance": { + "PIVOT_vs_PIVOT-guidance": { + "p": 1.0, + "dTop5": 0.0 + }, + "PIVOT_vs_MeanControl": { + "p": 0.744127936031984, + "dTop5": 0.045454545454545456 + }, + "PIVOT_vs_AvgPerturbationEffect": { + "p": 0.744127936031984, + "dTop5": 0.045454545454545456 + }, + "PIVOT_vs_Additive": { + "p": 0.010994502748625687, + "dTop5": 0.22727272727272727 + }, + "PIVOT_vs_LinearResponse": { + "p": 0.12643678160919541, + "dTop5": 0.09090909090909091 + }, + "PIVOT_vs_NearestPerturbationCentroid": { + "p": 0.25287356321839083, + "dTop5": 0.045454545454545456 + }, + "PIVOT_vs_EndpointMLP": { + "p": 0.3268365817091454, + "dTop5": 0.09090909090909091 + }, + "PIVOT_vs_kNN-latent": { + "p": 0.06346826586706647, + "dTop5": 0.13636363636363635 + }, + "PIVOT_vs_ConditionalMLP": { + "p": 0.2588705647176412, + "dTop5": -0.045454545454545456 + }, + "PIVOT_vs_Random": { + "p": 0.1054472763618191, + "dTop5": 0.18181818181818182 + } + } +} \ No newline at end of file diff --git a/experiments/results/norman_combinatorial.json b/experiments/results/norman_combinatorial.json new file mode 100644 index 0000000000000000000000000000000000000000..cf8311f42f4396e6df48143ba262f40545c08e39 --- /dev/null +++ b/experiments/results/norman_combinatorial.json @@ -0,0 +1,271 @@ +{ + "split": "combination", + "n_targets": 26, + "n_combo_candidates": 131, + "result": { + "greedy_exact1": 0.15384615384615385, + "greedy_exact5": 0.15384615384615385, + "greedy_overlap": 0.34615384615384615, + "rank_exact1": 0.5, + "rank_exact5": 0.7692307692307693, + "rank_ndcg": 0.6432095355574537, + "rank_overlap": 0.6153846153846154, + "endpoint_dist": 1.181089229308642, + "synergy_corr": 0.39749566236367595, + "_per": { + "greedy_exact1": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "greedy_exact5": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "greedy_overlap": [ + 0.5, + 0.0, + 0.5, + 0.5, + 0.0, + 0.0, + 0.0, + 0.5, + 0.0, + 0.0, + 0.0, + 1.0, + 0.5, + 0.0, + 1.0, + 1.0, + 1.0, + 0.5, + 0.0, + 0.0, + 0.0, + 0.5, + 0.0, + 0.5, + 0.5, + 0.5 + ], + "rank_exact1": [ + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0 + ], + "rank_exact5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0 + ], + "rank_ndcg": [ + 0.49999999999949996, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.0, + 0.0, + 0.9999999999989999, + 0.9999999999989999, + 0.3868528072341547, + 0.3868528072341547, + 0.9999999999989999, + 0.49999999999949996, + 0.6309297535708266, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.0, + 0.30102999566368016, + 0.3868528072341547, + 0.6309297535708266, + 0.0, + 0.0, + 0.9999999999989999, + 0.9999999999989999 + ], + "rank_overlap": [ + 0.5, + 1.0, + 1.0, + 1.0, + 1.0, + 0.5, + 0.5, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.5, + 0.5, + 0.5, + 0.0, + 1.0, + 1.0 + ], + "endpoint_dist": [ + 0.6689271330833435, + 0.8036714196205139, + 1.0161041021347046, + 1.1899185180664062, + 0.9528195858001709, + 1.0340110063552856, + 1.1323903799057007, + 1.7547415494918823, + 0.5572233200073242, + 1.2502464056015015, + 1.7361279726028442, + 1.0626763105392456, + 1.6163440942764282, + 1.1465466022491455, + 1.046038031578064, + 1.9452886581420898, + 0.5758848190307617, + 0.550717830657959, + 1.5649155378341675, + 1.068113088607788, + 1.2296886444091797, + 0.5714886784553528, + 0.9187361001968384, + 1.9091839790344238, + 2.291757822036743, + 1.1147583723068237 + ], + "synergy_corr": [ + -0.36697694659233093, + 0.7867258191108704, + 0.7642111778259277, + 0.8102744817733765, + 0.3918057978153229, + -0.5012184977531433, + -0.44898518919944763, + 0.9819259643554688, + 0.9748311042785645, + 0.5728657245635986, + 0.6892925500869751, + 0.9163436889648438, + -0.3066551089286804, + 0.8415080308914185, + 0.32007336616516113, + 0.4687807857990265, + 0.6908244490623474, + 0.7702524065971375, + 0.6121890544891357, + 0.5949224829673767, + -0.3006000816822052, + 0.43497174978256226, + -0.37792277336120605, + -0.18205173313617706, + 0.2413092851638794, + 0.9561896324157715 + ] + }, + "n_targets": 26 + } +} \ No newline at end of file diff --git a/experiments/results/norman_distloss.json b/experiments/results/norman_distloss.json new file mode 100644 index 0000000000000000000000000000000000000000..587705beed4c9de44163613a8a2b2c3690693c84 --- /dev/null +++ b/experiments/results/norman_distloss.json @@ -0,0 +1,27 @@ +{ + "split": "perturbation", + "desc": "distributional flow loss (population MMD) sweep, Norman held-out perturbation", + "rows": { + "0.0": { + "de_corr": 0.839, + "r2": -0.816, + "pearson": 0.769, + "mmd": 0.3034, + "wass": 0.433 + }, + "2.0": { + "de_corr": 0.835, + "r2": -1.118, + "pearson": 0.757, + "mmd": 0.0507, + "wass": 0.231 + }, + "10.0": { + "de_corr": 0.844, + "r2": -1.557, + "pearson": 0.759, + "mmd": 0.0507, + "wass": 0.235 + } + } +} \ No newline at end of file diff --git a/experiments/results/norman_extra_ablations.json b/experiments/results/norman_extra_ablations.json new file mode 100644 index 0000000000000000000000000000000000000000..9ff4d5eeedeb4f42396b5bfec4315d653b96c616 --- /dev/null +++ b/experiments/results/norman_extra_ablations.json @@ -0,0 +1,44 @@ +{ + "velocity_only": { + "mse": 0.003250642179293518, + "de_corr": 0.6322063905424735, + "mmd": 0.08446152519280205, + "endpoint_dist": 2.4770562919703396, + "top5": 0.09090909090909091, + "ndcg": 0.12225184183812116 + }, + "ranking_only": { + "mse": 0.0017491921988697326, + "de_corr": 0.8388409530545803, + "mmd": 0.3034484875795543, + "endpoint_dist": 1.068045285615054, + "top5": 0.13636363636363635, + "ndcg": 0.08727248090397387 + }, + "guidance_no_rerank": { + "mse": 0.0017491921988697326, + "de_corr": 0.8388409530545803, + "mmd": 0.3034484875795543, + "endpoint_dist": 1.5734560652212664, + "top5": 0.13636363636363635, + "ndcg": 0.0968604433440603 + }, + "avg_effect_ranking": { + "top1": 0.0, + "top5": 0.18181818181818182, + "ndcg": 0.10475755936295951, + "func_top5": 0.8181818181818182, + "med_rank": 33.5 + }, + "gpu_mem_mb": 170.1, + "combo_additive": { + "exact1": 0.5769230769230769, + "exact5": 0.8846153846153846, + "overlap": 0.7115384615384616 + }, + "combo_random": { + "exact1": 0.0, + "exact5": 0.038461538461538464, + "overlap": 0.038461538461538464 + } +} \ No newline at end of file diff --git a/experiments/results/norman_forward_cell.json b/experiments/results/norman_forward_cell.json new file mode 100644 index 0000000000000000000000000000000000000000..52281a87c69c172c0ff5d56f0e6ab78bde1592a0 --- /dev/null +++ b/experiments/results/norman_forward_cell.json @@ -0,0 +1,4693 @@ +{ + "split": "cell", + "models": { + "PIVOT": { + "mse": 0.00042636621328711043, + "r2": 0.6625150583684445, + "pearson": 0.873743824660778, + "spearman": 0.7144527607025901, + "de_corr": 0.9394492670893669, + "mmd": 0.2756637274880497, + "wasserstein": 0.3904557639951019, + "_per_pert": { + "mse": [ + 0.0007359963492490351, + 0.0014388746349141002, + 0.00023489593877457082, + 0.0003395555540919304, + 6.860582652734593e-05, + 0.0001072223749361001, + 0.0003080830501858145, + 0.0004004489746876061, + 0.0003480004670564085, + 0.00014401933003682643, + 0.00031807084451429546, + 0.00029007543344050646, + 0.0005666693323291838, + 0.0003495598502922803, + 0.0007474930607713759, + 0.0015909397043287754, + 0.0003248103312216699, + 0.00029493114561773837, + 0.0003482183092273772, + 0.000696904375217855, + 0.0005240575410425663, + 0.00041026229155249894, + 0.00011513544450281188, + 0.0007094212924130261, + 0.001013647299259901, + 0.0003035466652363539, + 0.00017436372581869364, + 0.00019152161257807165, + 0.00012565105862449855, + 0.00032632998772896826, + 0.0002596660051494837, + 0.00018594219000078738, + 9.061110176844522e-05, + 0.0012011878425255418, + 0.00036362861283123493, + 0.0002793762250803411, + 0.0002171386149711907, + 0.00026604425511322916, + 0.0006816217792220414, + 0.00022196571808308363, + 0.0018193451687693596, + 0.00018033853848464787, + 0.0007540542865172029, + 0.0008850707090459764, + 0.0003755279758479446, + 0.0003105867363046855, + 0.00037331695784814656, + 0.0004905483801849186, + 0.0003289401065558195, + 0.0005776216858066618, + 0.0001403094647685066, + 0.0008009119192138314, + 0.00030285189859569073, + 0.0002124161401297897, + 0.00014316265878733248, + 6.883969035698101e-05, + 0.00022488678223453462, + 0.00036235246807336807, + 0.0005470297764986753, + 0.0001445755042368546, + 0.00020299146126490086, + 0.0001019873161567375, + 6.752603076165542e-05, + 6.974710413487628e-05, + 0.0002803328388836235, + 0.0001160207248176448, + 0.00017028822912834585, + 0.00037853719550184906, + 0.00015041591541375965, + 0.0010111050214618444, + 0.00038826820673421025, + 0.00022016542789060622, + 0.00043919874588027596, + 0.0006280448869802058, + 0.0002771718136500567, + 0.00022924199583940208, + 0.0013608470326289535, + 0.00044195979717187583, + 0.0007277631666511297, + 0.0004905031528323889 + ], + "r2": [ + 0.6753616333007812, + 0.59185391664505, + 0.24209481477737427, + 0.9336997866630554, + -0.1812901496887207, + -1.8320705890655518, + 0.457907497882843, + 0.8556836843490601, + 0.547022819519043, + 0.9368777275085449, + 0.7779399752616882, + 0.8083398342132568, + 0.7925646901130676, + 0.8185151219367981, + 0.7369778156280518, + 0.9465508460998535, + 0.9581438302993774, + 0.9713456630706787, + 0.6978344917297363, + 0.6700522303581238, + 0.7544103264808655, + 0.9160881042480469, + 0.9705630540847778, + 0.6483197212219238, + 0.7650389075279236, + 0.9365057349205017, + 0.5396764278411865, + 0.8882166147232056, + 0.9809165596961975, + 0.7153270244598389, + 0.6411250829696655, + 0.941808819770813, + 0.9844553470611572, + 0.49165308475494385, + 0.5990991592407227, + 0.7931689023971558, + 0.4787245988845825, + 0.8589606285095215, + 0.6999421119689941, + 0.6649584770202637, + 0.7916816473007202, + 0.47121524810791016, + 0.7344892024993896, + 0.8696489334106445, + 0.7666510939598083, + 0.9232834577560425, + 0.8779070973396301, + 0.8258572816848755, + 0.9779921770095825, + 0.8056530356407166, + 0.6284376978874207, + 0.6297470331192017, + 0.5901060700416565, + 0.4564881920814514, + 0.10201793909072876, + 0.984345018863678, + 0.5440793037414551, + 0.8100419044494629, + 0.3622112274169922, + 0.3902508020401001, + 0.6790803074836731, + 0.7693303823471069, + 0.6733828783035278, + 0.7682066559791565, + -0.15527355670928955, + 0.9812520742416382, + 0.6655848026275635, + 0.8708289861679077, + 0.4884892702102661, + 0.5300676822662354, + 0.9040532112121582, + 0.19638681411743164, + 0.6257263422012329, + 0.9292815923690796, + 0.937282383441925, + 0.9506581425666809, + 0.2641477584838867, + 0.5689066648483276, + 0.23542523384094238, + 0.901918351650238 + ], + "pearson": [ + 0.823376476764679, + 0.771052360534668, + 0.7032936215400696, + 0.969688355922699, + 0.4795570373535156, + 0.5417088866233826, + 0.6823910474777222, + 0.9514938592910767, + 0.9055410623550415, + 0.9679874777793884, + 0.9016690850257874, + 0.9714018702507019, + 0.8982111811637878, + 0.9284631013870239, + 0.8605834245681763, + 0.9907697439193726, + 0.9837180376052856, + 0.990242063999176, + 0.9528862237930298, + 0.8200882077217102, + 0.883393406867981, + 0.9576980471611023, + 0.9876605868339539, + 0.8059754371643066, + 0.8819361925125122, + 0.9696067571640015, + 0.7478370666503906, + 0.9428436756134033, + 0.991365909576416, + 0.8552893400192261, + 0.921582818031311, + 0.994169294834137, + 0.9929064512252808, + 0.9552178978919983, + 0.848775327205658, + 0.9273852109909058, + 0.9172924757003784, + 0.9510259628295898, + 0.8491963148117065, + 0.8218192458152771, + 0.9009863138198853, + 0.7216945886611938, + 0.8590138554573059, + 0.9649760723114014, + 0.9526090621948242, + 0.9705120325088501, + 0.9463509321212769, + 0.9582728147506714, + 0.991611659526825, + 0.9333918690681458, + 0.8577359318733215, + 0.8076059222221375, + 0.7936742901802063, + 0.7464778423309326, + 0.6358630657196045, + 0.9926109313964844, + 0.7399097681045532, + 0.9468521475791931, + 0.6391595005989075, + 0.6817261576652527, + 0.905633270740509, + 0.8831990361213684, + 0.8483492732048035, + 0.9035524129867554, + 0.7248049378395081, + 0.9925737977027893, + 0.8774946928024292, + 0.9333910346031189, + 0.7056218385696411, + 0.7384148836135864, + 0.969870924949646, + 0.6692087650299072, + 0.9222154021263123, + 0.9693341255187988, + 0.9784613251686096, + 0.9759108424186707, + 0.9428775906562805, + 0.7752811312675476, + 0.8949389457702637, + 0.950236439704895 + ], + "spearman": [ + 0.746333871897977, + 0.663733536392687, + 0.35099815836963905, + 0.7810430256908575, + 0.17718533650478066, + 0.37838741350919847, + 0.48166911991981315, + 0.7120752952680662, + 0.637829013067945, + 0.9004614584729789, + 0.7187370239067322, + 0.7081244568395412, + 0.6526492044730655, + 0.6517051989182143, + 0.5946082098752195, + 0.9371957364510767, + 0.9271993285986212, + 0.9249394551652509, + 0.839142672010879, + 0.7263838729245445, + 0.5920637702552338, + 0.7155752532191457, + 0.8553244593779952, + 0.4530424973766981, + 0.7895751188365852, + 0.8945215326589985, + 0.460105451160326, + 0.7808275782908773, + 0.941038411609852, + 0.6477314151701272, + 0.6612341669811286, + 0.9495094071503231, + 0.9016047334990509, + 0.8738063433256489, + 0.7811783455199127, + 0.8592411399822559, + 0.7659261956565535, + 0.8574573050115463, + 0.6448917588554881, + 0.7056299864335015, + 0.8458971637783255, + 0.41256141585232786, + 0.774996191367732, + 0.8716843318038745, + 0.8412131080405637, + 0.9100446972967778, + 0.8108115041353203, + 0.8515357593229316, + 0.893517995941992, + 0.8740929420235539, + 0.6879632332862982, + 0.6356718199879016, + 0.6096678482841106, + 0.5106306727160472, + 0.5087522837140477, + 0.875810640903536, + 0.571883046371288, + 0.763601606745374, + 0.43631143519061427, + 0.4971107479800143, + 0.7342175814916644, + 0.6964224171077327, + 0.590112218506535, + 0.609041278615015, + 0.567858497093661, + 0.942141520857594, + 0.6539658019842579, + 0.8688713660234317, + 0.4544848297110707, + 0.7096450301365174, + 0.9096968412051177, + 0.6506350713882347, + 0.722309267349358, + 0.7733674526123494, + 0.8993705831822604, + 0.7496586890641386, + 0.712721887319488, + 0.6245717545746574, + 0.6079001143070433, + 0.854783950306115 + ], + "de_corr": [ + 0.8861535787582397, + 0.901011049747467, + 0.8344247341156006, + 0.9933923482894897, + 0.6679480671882629, + 0.6480610370635986, + 0.7873584032058716, + 0.9811016917228699, + 0.9801551699638367, + 0.9860553741455078, + 0.968198835849762, + 0.9883202314376831, + 0.9417359828948975, + 0.9549991488456726, + 0.9301045536994934, + 0.9785342216491699, + 0.9934975504875183, + 0.9980303049087524, + 0.981399416923523, + 0.950262725353241, + 0.9679417610168457, + 0.986670196056366, + 0.9951409101486206, + 0.9153488278388977, + 0.9499597549438477, + 0.9899380207061768, + 0.8737310171127319, + 0.9874207377433777, + 0.9971346259117126, + 0.9385604858398438, + 0.9571271538734436, + 0.9991211891174316, + 0.999093234539032, + 0.9760386943817139, + 0.9587435126304626, + 0.9760202765464783, + 0.9862465858459473, + 0.9621620178222656, + 0.9498906135559082, + 0.9384939670562744, + 0.9413933157920837, + 0.84197998046875, + 0.8953912854194641, + 0.9910485744476318, + 0.9798243641853333, + 0.9906160831451416, + 0.9796128273010254, + 0.9916460514068604, + 0.9956063628196716, + 0.9831395745277405, + 0.9419919848442078, + 0.9558847546577454, + 0.8894071578979492, + 0.84073406457901, + 0.8188599348068237, + 0.9984307885169983, + 0.8444566130638123, + 0.9797614216804504, + 0.8001152276992798, + 0.9141258597373962, + 0.9489234089851379, + 0.983659029006958, + 0.9296177625656128, + 0.965865969657898, + 0.8540292382240295, + 0.9987342953681946, + 0.970844566822052, + 0.9805833697319031, + 0.8723176717758179, + 0.8724640607833862, + 0.9779011607170105, + 0.8631344437599182, + 0.9574514627456665, + 0.9871930480003357, + 0.9916741251945496, + 0.9888807535171509, + 0.9693055152893066, + 0.8763558864593506, + 0.9599824547767639, + 0.9774689078330994 + ], + "mmd": [ + 0.2834284236203858, + 0.2724673704105385, + 0.2868283892234469, + 0.2756056412421941, + 0.290903897816604, + 0.27482465678689927, + 0.26122782075567064, + 0.28630170559184764, + 0.27351465817816933, + 0.2905019870183143, + 0.27230579885012196, + 0.25570367737670274, + 0.30109630301153056, + 0.2930860034867745, + 0.29520807773772795, + 0.26022343392931724, + 0.26213987068534084, + 0.2311268868916988, + 0.279984936863494, + 0.2860970549523927, + 0.28488139472454777, + 0.28269774706381, + 0.22109461701776212, + 0.28171161347468043, + 0.2989779288932123, + 0.2478088503201823, + 0.269034606332493, + 0.26649677086531365, + 0.2476621271512176, + 0.2892264360855554, + 0.2567050898996939, + 0.25837888130197484, + 0.24924930109039523, + 0.27967867553009496, + 0.26912198160908896, + 0.28505954093056274, + 0.26579943937156625, + 0.2563059803205264, + 0.30086641170104333, + 0.3010375946646501, + 0.25832075648408626, + 0.28731252693668496, + 0.297622775219195, + 0.3003627583680848, + 0.29406685208848726, + 0.29534905198117456, + 0.2746181413812063, + 0.2934915991046324, + 0.25882471189594625, + 0.23807970509226084, + 0.2751055596526252, + 0.30036499573518716, + 0.3002868620250506, + 0.25856924883172105, + 0.25650473330962653, + 0.2502183659645304, + 0.28073670926936733, + 0.25875600536513466, + 0.2962976629859533, + 0.2866974818383998, + 0.2659294488563111, + 0.25588220668519734, + 0.29244908573295836, + 0.2865532287044982, + 0.25716011531544325, + 0.23632337710902396, + 0.2681174165213148, + 0.30659282375844255, + 0.29042848774420416, + 0.29789779817446316, + 0.25705620202737467, + 0.25674751306908317, + 0.26406547929108226, + 0.2921292049974058, + 0.297549731362042, + 0.27789738275875464, + 0.27527961769668385, + 0.2891294070158926, + 0.2917163057617206, + 0.28823528015517974 + ], + "wasserstein": [ + 0.4227961445814133, + 0.448756784082567, + 0.37178342823974697, + 0.39412034867058315, + 0.35198020652263834, + 0.35956729311584923, + 0.36494515021696416, + 0.38248930979106804, + 0.3747586017235981, + 0.4039407345869606, + 0.3664838122439365, + 0.3536683440785178, + 0.4156407956732704, + 0.417351364331134, + 0.4176238092935985, + 0.5190372222896514, + 0.43006724810135033, + 0.42506773071156545, + 0.38376235416107546, + 0.42814644454196354, + 0.3899198506352131, + 0.405704218148541, + 0.3412667648891096, + 0.369301139935177, + 0.3915165548297004, + 0.37687731621316184, + 0.34681337164994624, + 0.36399271094768537, + 0.3719479021148587, + 0.3745235722104785, + 0.32865187001625246, + 0.35431075011192226, + 0.3911292950509271, + 0.4028773981803729, + 0.37982071418936636, + 0.40196069119083583, + 0.3586472496885322, + 0.38606248076727334, + 0.4151912486225612, + 0.38907148280353376, + 0.5130298176212963, + 0.3764716402014273, + 0.4163843000433636, + 0.47446074809810546, + 0.40827078817288887, + 0.4593897741305597, + 0.3854345265942066, + 0.39971481075784576, + 0.42786664101904187, + 0.35757056683293426, + 0.35777344862665394, + 0.4087335372150916, + 0.38050187217578924, + 0.36493259100364084, + 0.35738434897034727, + 0.3511948611981983, + 0.37700871318501045, + 0.3800314006399137, + 0.38668925533363274, + 0.3660344210108214, + 0.3259300074166642, + 0.3347213937858525, + 0.36866797612447955, + 0.38312600879313863, + 0.35517639334012785, + 0.3509844467755327, + 0.3764367948677597, + 0.4280669166957916, + 0.38696915146633976, + 0.4120134431742553, + 0.3844682735058982, + 0.37421547765183066, + 0.3848824422299757, + 0.44058097760776527, + 0.42144624759298915, + 0.38914200597396253, + 0.3895334589522591, + 0.39237684955381874, + 0.37567697982030684, + 0.44159410229573204 + ] + }, + "n_perts": 80 + }, + "MeanControl": { + "mse": 0.0030108978945463604, + "r2": -0.052831302583217624, + "pearson": 0.03417627952730982, + "spearman": 0.02656565903642848, + "de_corr": 0.04636090797139332, + "mmd": 0.07650097081510396, + "wasserstein": 0.2615824826342393, + "_per_pert": { + "mse": [ + 0.0024968457873910666, + 0.0037335872184485197, + 0.00028462731279432774, + 0.005347683560103178, + 8.937355596572161e-05, + 6.19333513895981e-05, + 0.0006179463816806674, + 0.0024909721687436104, + 0.0007785729249008, + 0.0024079689756035805, + 0.0013492027064785361, + 0.0015657787444069982, + 0.0026282668113708496, + 0.0017668072832748294, + 0.0030559562146663666, + 0.029824092984199524, + 0.007653015200048685, + 0.010690617375075817, + 0.00119070487562567, + 0.0023122935090214014, + 0.0020736909937113523, + 0.004696778487414122, + 0.0040584648959338665, + 0.0019544768147170544, + 0.004362186416983604, + 0.004772201646119356, + 0.0003804886364378035, + 0.0017128377221524715, + 0.006998598575592041, + 0.001185731845907867, + 0.0006662501255050302, + 0.0032387750688940287, + 0.006219964940100908, + 0.0021737259812653065, + 0.0010090091964229941, + 0.0015417614486068487, + 0.00036094040842726827, + 0.0021122382022440434, + 0.0023785398807376623, + 0.0006914117839187384, + 0.008963387459516525, + 0.0003513597184792161, + 0.002761770971119404, + 0.006875904276967049, + 0.0018641167553141713, + 0.004169839899986982, + 0.003282568883150816, + 0.0029890837613493204, + 0.015267801471054554, + 0.003016992472112179, + 0.00041092594619840384, + 0.002177310874685645, + 0.0007273025112226605, + 0.00031340421992354095, + 0.00022650064784102142, + 0.004224393051117659, + 0.0006368549657054245, + 0.001900566858239472, + 0.0007987807039171457, + 0.0002276950399391353, + 0.0005579819553531706, + 0.000517762266099453, + 0.00022734134108759463, + 0.0003528558590915054, + 0.0003610069106798619, + 0.006010997574776411, + 0.0005817001801915467, + 0.00305701675824821, + 0.00039519797428511083, + 0.002127814805135131, + 0.003994378261268139, + 0.00033054573577828705, + 0.0012322484981268644, + 0.009270673617720604, + 0.004500047769397497, + 0.0044023715890944, + 0.001961558824405074, + 0.0011347273830324411, + 0.0009095503482967615, + 0.004825173411518335 + ], + "r2": [ + -0.1013263463973999, + -0.05905592441558838, + 0.08163344860076904, + -0.04416608810424805, + -0.5388791561126709, + -0.635849118232727, + -0.08731722831726074, + 0.10228770971298218, + -0.013435006141662598, + -0.055389404296875, + 0.0580592155456543, + -0.034549832344055176, + 0.0378953218460083, + 0.08270615339279175, + -0.07530653476715088, + -0.001969575881958008, + 0.013806819915771484, + -0.038657546043395996, + -0.033231258392333984, + -0.09474992752075195, + 0.02820432186126709, + 0.03935658931732178, + -0.037638068199157715, + 0.031110107898712158, + -0.01114499568939209, + 0.0017766356468200684, + -0.0044966936111450195, + 0.0002862215042114258, + -0.06292331218719482, + -0.03436923027038574, + 0.07920032739639282, + -0.013584375381469727, + -0.06705796718597412, + 0.08007138967514038, + -0.11243319511413574, + -0.14141476154327393, + 0.13350582122802734, + -0.11977183818817139, + -0.04706108570098877, + -0.04363727569580078, + -0.02632439136505127, + -0.030249476432800293, + 0.027550220489501953, + -0.012666940689086914, + -0.15834152698516846, + -0.029971957206726074, + -0.07356059551239014, + -0.06111335754394531, + -0.021497726440429688, + -0.015099406242370605, + -0.08819866180419922, + -0.00654757022857666, + 0.01563471555709839, + 0.19808876514434814, + -0.4207165241241455, + 0.0393216609954834, + -0.29111790657043457, + 0.003654778003692627, + 0.06869161128997803, + 0.03969311714172363, + 0.11785739660263062, + -0.17104816436767578, + -0.09962928295135498, + -0.17266035079956055, + -0.48773765563964844, + 0.028676211833953857, + -0.14235401153564453, + -0.043168067932128906, + -0.34392690658569336, + 0.011053323745727539, + 0.012930452823638916, + -0.2065058946609497, + -0.05008959770202637, + -0.043886661529541016, + -0.018257975578308105, + 0.05243724584579468, + -0.06067538261413574, + -0.1068274974822998, + 0.044442951679229736, + 0.03515249490737915 + ], + "pearson": [ + -0.19617466628551483, + -0.20266488194465637, + 0.318413108587265, + -0.142558291554451, + 0.041828449815511703, + 0.31264278292655945, + -0.05740047246217728, + 0.42468225955963135, + 0.08925791829824448, + -0.1271500587463379, + 0.24752703309059143, + -0.004042191430926323, + 0.21024706959724426, + 0.3595820665359497, + -0.24428977072238922, + 0.24925395846366882, + 0.2989994287490845, + -0.1155460998415947, + 0.07103670388460159, + -0.2540331482887268, + 0.17397819459438324, + 0.30281171202659607, + -0.12048927694559097, + 0.20164233446121216, + -0.010908474214375019, + 0.07722484320402145, + 0.1610965132713318, + 0.08894678205251694, + -0.16658943891525269, + -0.0166898425668478, + 0.298014372587204, + 0.023421159014105797, + -0.30511876940727234, + 0.4841887056827545, + -0.06750041991472244, + -0.3489888608455658, + 0.3668804168701172, + -0.38222357630729675, + -0.14677274227142334, + 0.06658767908811569, + -0.03478322923183441, + 0.1989404857158661, + 0.1696953922510147, + -0.00474256556481123, + -0.3096827268600464, + -0.10181869566440582, + -0.16512373089790344, + -0.26239004731178284, + -0.11563233286142349, + 0.05729414150118828, + -0.010856014676392078, + 0.07001086324453354, + 0.1748407781124115, + 0.4708547294139862, + -0.25451216101646423, + 0.2757906913757324, + -0.3208184838294983, + 0.08199682831764221, + 0.2696215510368347, + 0.23174917697906494, + 0.35254964232444763, + -0.16863514482975006, + 0.1302313208580017, + -0.021851588040590286, + -0.4352736175060272, + 0.24263250827789307, + -0.0014796104514971375, + -0.044150516390800476, + -0.2757139503955841, + 0.13252761960029602, + 0.2150670439004898, + -0.0423787385225296, + -0.033429890871047974, + -0.2126273363828659, + -0.05198513716459274, + 0.2839607000350952, + -0.13444273173809052, + -0.10225614160299301, + 0.21538753807544708, + 0.3064132332801819 + ], + "spearman": [ + -0.018696888482720782, + 0.10664245089027675, + 0.21303845077474715, + 0.22654263497314703, + -0.029826437709930567, + -0.04770406049891016, + -0.11614411710397486, + 0.3644126055227216, + 0.07736104239572819, + 0.047970940696157614, + 0.27416790559605614, + 0.06636647818666645, + -0.28689727495501843, + 0.14191822360339992, + -0.3232429740154687, + 0.3967873087438178, + 0.2585616989066928, + -0.24478042216756776, + -0.24861626519269686, + -0.30849656215453036, + 0.006722032046463064, + -0.03135073061672623, + -0.00523400913112416, + 0.12295507326752705, + 0.13775171747740164, + 0.3820544602107841, + 0.14236461531646288, + 0.07643747541993882, + -0.11007557145664484, + -0.09654467622572645, + 0.2680330690777385, + -0.12480209538535215, + -0.22157540021820873, + 0.15269762719391766, + -0.03467861650485578, + -0.34189429999304133, + 0.3638272854599716, + -0.13468493884049507, + -0.047213232582326486, + 0.1595853840507487, + 0.07273051094176358, + 0.19028824703317593, + 0.22425263909742307, + -0.04624397959136205, + -0.42083158855943015, + 0.07395679070948473, + -0.16710924853461476, + -0.07300276704098119, + 0.04393839107776606, + -0.22264985633618672, + -0.23799377376458836, + -0.2443046429488016, + 0.15477777802065282, + 0.07755667847702231, + -0.017747484986360057, + -0.05086155217109208, + -0.23511911608558378, + 0.20962536727237935, + 0.2860209438665188, + 0.26119370167248807, + 0.2225652340628631, + 0.03891614453359119, + 0.25532135380910487, + -0.0454639270195773, + -0.41202813597392435, + 0.1144999378501084, + 0.11139186047770278, + 0.06385181231439682, + -0.26875824426936606, + 0.11032723669358128, + 0.3499061482002701, + -0.15412491461191502, + -0.032736054388226354, + 0.1540985094705311, + 0.22484940592236674, + 0.11746612930472347, + -0.10420339624667119, + -0.045317768208226084, + -0.013154279804944114, + 0.34562872607316936 + ], + "de_corr": [ + -0.3583424389362335, + -0.040797535330057144, + 0.6186198592185974, + -0.20751512050628662, + -0.11017404496669769, + 0.5511878728866577, + -0.008291724137961864, + 0.7472458481788635, + 0.2162538319826126, + -0.01610230654478073, + 0.19799910485744476, + -0.10336548089981079, + 0.428211510181427, + 0.49029967188835144, + -0.39147529006004333, + 0.13625629246234894, + 0.7124975323677063, + -0.22596734762191772, + 0.6266492605209351, + -0.4679238796234131, + 0.3950445353984833, + 0.7326490879058838, + -0.12775491178035736, + 0.330987811088562, + 0.17798052728176117, + -0.10655134916305542, + -0.18162302672863007, + 0.2740331292152405, + -0.4197103977203369, + -0.03643707185983658, + 0.5783090591430664, + -0.2066262662410736, + -0.5923011898994446, + 0.612706184387207, + 0.31312599778175354, + -0.6877951622009277, + 0.7151832580566406, + -0.608711302280426, + -0.22526124119758606, + 0.019109299406409264, + -0.19159428775310516, + 0.5220105648040771, + 0.21550050377845764, + 0.23839852213859558, + -0.3414430022239685, + -0.39704105257987976, + -0.279800683259964, + -0.5159155130386353, + -0.4446691870689392, + 0.02989734709262848, + 0.12840531766414642, + 0.09767109155654907, + 0.3705161213874817, + 0.706402063369751, + -0.6759462356567383, + 0.5882141590118408, + -0.3891955614089966, + 0.13488110899925232, + 0.40333685278892517, + 0.26238197088241577, + 0.5207210183143616, + -0.47355642914772034, + 0.024531416594982147, + -0.14138250052928925, + -0.7957724332809448, + 0.4591553509235382, + -0.11949479579925537, + -0.10420750081539154, + -0.21784105896949768, + 0.2274601012468338, + 0.2704351842403412, + 0.04722755774855614, + 0.049906302243471146, + -0.5146441459655762, + -0.3546771705150604, + 0.41846051812171936, + -0.2064916491508484, + -0.23332321643829346, + 0.16322676837444305, + 0.47550663352012634 + ], + "mmd": [ + 0.07244179177556842, + 0.09328133566445018, + 0.012325017510074532, + 0.15293397432321953, + 0.007381613637261308, + 0.005088752332804813, + 0.02274309563273924, + 0.07840342213297047, + 0.03038176302245965, + 0.0645687771171567, + 0.043619395826677176, + 0.050505349772522634, + 0.07739557746318504, + 0.04503358155814274, + 0.08854644974896586, + 0.35283850423644825, + 0.15833674231831163, + 0.20908666667793985, + 0.029963597768687178, + 0.0596358939670566, + 0.06386112708602831, + 0.13001096172301307, + 0.11402994843421255, + 0.06913426451966076, + 0.11758256846998505, + 0.12856393639583685, + 0.018208147613457792, + 0.059719741665771875, + 0.21420077614885413, + 0.04131784254834836, + 0.028917166585746723, + 0.11225714519927188, + 0.17236068869617271, + 0.05532957623086776, + 0.03472038274634659, + 0.04444534143337553, + 0.014615480618748289, + 0.06213729569967463, + 0.07005153818474652, + 0.02392708861636539, + 0.1548325474105522, + 0.014980148917721592, + 0.0819617596547153, + 0.15920896433280607, + 0.04964945794748332, + 0.10624176222791182, + 0.09619988415964775, + 0.0892314887042791, + 0.2900790039512432, + 0.09711375190298444, + 0.014184307755548575, + 0.06695240027613514, + 0.023942448097221902, + 0.01444630301830907, + 0.012885301195344812, + 0.13699514483308495, + 0.025230733279738793, + 0.0636958772291869, + 0.03304610149060594, + 0.011417767762450959, + 0.024140178226070175, + 0.02179294027533396, + 0.009203581472666533, + 0.01534461790145536, + 0.019127031300047892, + 0.1805726043300352, + 0.024404959436828966, + 0.07249839253641932, + 0.016154952712161275, + 0.07260686365988822, + 0.09531011429025638, + 0.016442255738114087, + 0.041730978806622576, + 0.21531135355676656, + 0.13343802609658062, + 0.12666544321863626, + 0.0685239689978594, + 0.03608514258165674, + 0.033661817587101184, + 0.12486496723371743 + ], + "wasserstein": [ + 0.31535203628011627, + 0.3531505364426003, + 0.12287251132617064, + 0.40663624593322323, + 0.0914581797082831, + 0.07385931931239816, + 0.15641653286687235, + 0.24917287507982408, + 0.16817470747524665, + 0.2685274659680512, + 0.19907978241444613, + 0.21258293962875308, + 0.2632826363691637, + 0.21395327051212942, + 0.2977722528767667, + 0.8008926935310582, + 0.40800521046290095, + 0.4865279141174568, + 0.163936107134828, + 0.2755461241999589, + 0.24545619371528257, + 0.3675116565349086, + 0.31001311279096305, + 0.2396894288349049, + 0.2969232841945847, + 0.3353042281304985, + 0.12660133158013753, + 0.23775575499939994, + 0.4934812990026405, + 0.20519084528023598, + 0.130748829889071, + 0.3345505503748829, + 0.38964371336069326, + 0.20048187954865612, + 0.18275416133247827, + 0.22401392029005596, + 0.12634289234233764, + 0.2576354049933038, + 0.2704482962165314, + 0.17638537123726375, + 0.4970018285376021, + 0.13060931179841279, + 0.29137072763827443, + 0.46203560089119367, + 0.22845054288592304, + 0.39995274636983985, + 0.34538736414552124, + 0.29264298579784265, + 0.678483390469034, + 0.35751942114719154, + 0.11169960671596503, + 0.28543161567763337, + 0.15556083491515021, + 0.1175194333043146, + 0.122003544772463, + 0.36298689937180173, + 0.1634191564759195, + 0.22502778373386004, + 0.175675437507294, + 0.11296550824160816, + 0.14210024323567017, + 0.1324287402576492, + 0.09942601875245817, + 0.11590969532044142, + 0.14531811791489194, + 0.43229408868984437, + 0.14360435758402512, + 0.2841959955407495, + 0.14706252349129773, + 0.23768714285596892, + 0.28963028047210815, + 0.12850279906941853, + 0.1955035587833229, + 0.5477680744647625, + 0.373913069857617, + 0.34782125525613977, + 0.23525060564617145, + 0.17714317016785197, + 0.16307849111834108, + 0.39408514557448654 + ] + }, + "n_perts": 80 + }, + "AvgPerturbationEffect": { + "mse": 0.0025730653083883225, + "r2": -1.2934495955705643, + "pearson": 0.4561482994351536, + "spearman": 0.4012512628432866, + "de_corr": 0.48580754559370687, + "mmd": 0.07675989552148531, + "wasserstein": 0.25543115107818465, + "_per_pert": { + "mse": [ + 0.0011889607412740588, + 0.0017667999491095543, + 0.0009449934004805982, + 0.002887776819989085, + 0.0013705306919291615, + 0.0014304587384685874, + 0.0014766072854399681, + 0.001989776501432061, + 0.0010465822415426373, + 0.0012323260307312012, + 0.002463675569742918, + 0.001325927791185677, + 0.0013498293701559305, + 0.0007185605354607105, + 0.0014667612267658114, + 0.02534634806215763, + 0.005792546551674604, + 0.008952856995165348, + 0.0017705163918435574, + 0.0010424376232549548, + 0.0014725371729582548, + 0.002430088585242629, + 0.0033544590696692467, + 0.0017511585028842092, + 0.004118716344237328, + 0.005374645348638296, + 0.0022420017048716545, + 0.0027744292747229338, + 0.0050294711254537106, + 0.000627790461294353, + 0.0018914910033345222, + 0.004123988561332226, + 0.004788254853338003, + 0.002691644709557295, + 0.0009012971422635019, + 0.0006795887602493167, + 0.0010372968390583992, + 0.0017150092171505094, + 0.0013853481505066156, + 0.0006746114231646061, + 0.00619583809748292, + 0.001135546714067459, + 0.0016884770011529326, + 0.0034360310528427362, + 0.0010860621696338058, + 0.0017383926315233111, + 0.0012792053166776896, + 0.0015290656592696905, + 0.010026146657764912, + 0.0031569672282785177, + 0.0013973591849207878, + 0.0014026868157088757, + 0.0005372564774006605, + 0.000815263600088656, + 0.0010874158469960093, + 0.004315572790801525, + 0.0008252027910202742, + 0.00229474063962698, + 0.0014258493902161717, + 0.0016666339943185449, + 0.0022549862042069435, + 0.0023796590976417065, + 0.000950109853874892, + 0.001002267119474709, + 0.001537806703709066, + 0.006609844043850899, + 0.001127220573835075, + 0.001607749960385263, + 0.0013110940344631672, + 0.0024517206475138664, + 0.004628554452210665, + 0.0007588522275909781, + 0.0007838053861632943, + 0.005785427987575531, + 0.0026301713660359383, + 0.002675267169252038, + 0.0017924248240888119, + 0.000675190647598356, + 0.0008772816509008408, + 0.0023379758931696415 + ], + "r2": [ + 0.4755648970603943, + 0.49883580207824707, + -2.0490756034851074, + 0.4361448287963867, + -22.598485946655273, + -36.78278350830078, + -1.5981879234313965, + 0.2829117178916931, + -0.36229097843170166, + 0.45988357067108154, + -0.7200057506561279, + 0.12392586469650269, + 0.5058807134628296, + 0.626936674118042, + 0.483887255191803, + 0.14846467971801758, + 0.25355297327041626, + 0.13017624616622925, + -0.5363613367080688, + 0.5064603090286255, + 0.3099234700202942, + 0.5029681921005249, + 0.1423569917678833, + 0.13190072774887085, + 0.04529088735580444, + -0.12423944473266602, + -4.9189252853393555, + -0.6193214654922485, + 0.23614102602005005, + 0.45234888792037964, + -1.6141600608825684, + -0.2906144857406616, + 0.17855727672576904, + -0.1391134262084961, + 0.0063193440437316895, + 0.4968789219856262, + -1.4901938438415527, + 0.09081333875656128, + 0.39015328884124756, + -0.01827824115753174, + 0.2905651330947876, + -2.329624891281128, + 0.4054687023162842, + 0.4939494729042053, + 0.3251330256462097, + 0.5706080794334412, + 0.5816372632980347, + 0.4571875333786011, + 0.3291970491409302, + -0.0621953010559082, + -2.7004339694976807, + 0.3515528440475464, + 0.27285200357437134, + -1.0860247611999512, + -5.820773124694824, + 0.018586456775665283, + -0.6729612350463867, + -0.20298516750335693, + -0.6624155044555664, + -6.029052734375, + -2.5650246143341064, + -4.3821892738342285, + -3.595594882965088, + -2.3308751583099365, + -5.3374223709106445, + -0.0680917501449585, + -1.2136576175689697, + 0.45137590169906616, + -3.4585609436035156, + -0.1394888162612915, + -0.14378368854522705, + -1.7698431015014648, + 0.3320615887641907, + 0.348555326461792, + 0.4048524498939514, + 0.42417776584625244, + 0.030780494213104248, + 0.34141021966934204, + 0.07834386825561523, + 0.5324954986572266 + ], + "pearson": [ + 0.6946286559104919, + 0.7245969176292419, + 0.5656006932258606, + 0.6956605911254883, + -0.27045464515686035, + 0.24282513558864594, + 0.24458901584148407, + 0.5586576461791992, + 0.5492426753044128, + 0.6831101775169373, + 0.19163878262043, + 0.5417393445968628, + 0.7157999873161316, + 0.798336386680603, + 0.6992589235305786, + 0.49378371238708496, + 0.5252848267555237, + 0.37364864349365234, + 0.22909511625766754, + 0.7128421068191528, + 0.5914393067359924, + 0.7395287752151489, + 0.41518524289131165, + 0.4843192398548126, + 0.320198118686676, + 0.18636789917945862, + -0.38644352555274963, + 0.07788721472024918, + 0.4866774082183838, + 0.7430286407470703, + 0.06849080324172974, + 0.05680612847208977, + 0.42690029740333557, + 0.2740628123283386, + 0.5946295857429504, + 0.7251547574996948, + 0.47445207834243774, + 0.4650551676750183, + 0.6423261761665344, + 0.6846609711647034, + 0.5740780830383301, + 0.4564644694328308, + 0.6411603689193726, + 0.7927584648132324, + 0.6119512319564819, + 0.7991904020309448, + 0.7743661999702454, + 0.6795049905776978, + 0.7233084440231323, + 0.25839439034461975, + 0.12431465834379196, + 0.6144005060195923, + 0.7776922583580017, + 0.706981360912323, + 0.37357625365257263, + 0.2903957962989807, + 0.5367717146873474, + 0.3034783899784088, + 0.38897693157196045, + -0.22827354073524475, + -0.27529215812683105, + -0.3728831708431244, + 0.5944153070449829, + 0.5573655962944031, + -0.042715467512607574, + 0.14648719131946564, + 0.5094072222709656, + 0.6729158163070679, + 0.15379977226257324, + 0.30610236525535583, + 0.17594438791275024, + 0.662243127822876, + 0.6720780730247498, + 0.656128466129303, + 0.6421982049942017, + 0.6620456576347351, + 0.4418741762638092, + 0.697357177734375, + 0.6261511445045471, + 0.7661378979682922 + ], + "spearman": [ + 0.6550882410766743, + 0.6931565352361503, + 0.3621533313132103, + 0.46607691053250616, + -0.08134651771531236, + -0.21695932588325778, + 0.266504246723869, + 0.1974030295156983, + 0.34350415639215925, + 0.7441353823411302, + 0.5201756195662759, + 0.17899540101939765, + 0.4342025068251865, + 0.6485305620697926, + 0.5466729356433397, + 0.5382660588997545, + 0.6427179443471976, + 0.5785319629101218, + 0.43873060302003797, + 0.7106564907579855, + 0.3778060283520791, + 0.49789803163284313, + 0.034948700576934465, + 0.3412295849139957, + 0.5916557066481166, + 0.4520765430674561, + -0.2348012142081704, + 0.4492365112922844, + 0.27919070875902036, + 0.5799581728189785, + -0.05151276236224746, + 0.3283451583920787, + 0.18013527185074557, + 0.438596937984767, + 0.45128176318582053, + 0.597426080395807, + 0.4940749416062501, + 0.7099029647377809, + 0.5692795397129974, + 0.5671184930997478, + 0.717559809517859, + 0.28175172178774593, + 0.6035117448490489, + 0.7590827768302161, + 0.6227347623076609, + 0.6970836811929565, + 0.49170392247348427, + 0.41613067880487487, + 0.6026655586103714, + 0.3785953587876364, + 0.4077559920698804, + 0.5120028173547435, + 0.6279375965663226, + 0.46466077836236286, + 0.5478632281664476, + -0.06571662778812504, + 0.43669725479357985, + 0.39463190206738885, + 0.01526742627869211, + -0.22478822773169801, + -0.04685418275421002, + 0.040429261039559045, + 0.2283452611896416, + 0.11644133238093908, + 0.406864363611334, + 0.37198339340711795, + 0.36558949105055993, + 0.680802940896876, + 0.03834884738368292, + 0.5685455957230381, + 0.5244877352759815, + 0.6847707564506723, + 0.4082374474736564, + 0.5453112501440062, + 0.40706376710633413, + 0.2911244089580282, + 0.18945198281268646, + 0.5272878393880787, + 0.17307520100434745, + 0.5706189425699483 + ], + "de_corr": [ + 0.7498892545700073, + 0.8504038453102112, + 0.7783665060997009, + 0.8931769132614136, + -0.5685779452323914, + 0.43297967314720154, + 0.15621867775917053, + 0.892180323600769, + 0.8295984864234924, + 0.6522457003593445, + 0.0003651264123618603, + 0.8179827332496643, + 0.9015190005302429, + 0.9334091544151306, + 0.8782711029052734, + 0.16680830717086792, + 0.2322770357131958, + -0.013483481481671333, + -0.10384546965360641, + 0.9358643889427185, + 0.8012130856513977, + 0.9625198245048523, + 0.7940037250518799, + 0.5709026455879211, + 0.13760481774806976, + 0.04766756296157837, + -0.6491086483001709, + -0.38379451632499695, + 0.7431872487068176, + 0.8635349273681641, + 0.2283182591199875, + -0.2548728585243225, + 0.6318886280059814, + -0.21422572433948517, + 0.8628191947937012, + 0.8825773596763611, + 0.3618830442428589, + 0.2556570768356323, + 0.8792421817779541, + 0.9278112649917603, + 0.4433809518814087, + 0.7531028389930725, + 0.6255148649215698, + 0.7584497928619385, + 0.5132643580436707, + 0.9596038460731506, + 0.8642715215682983, + 0.8871360421180725, + 0.7430741190910339, + 0.5285480618476868, + -0.2099263221025467, + 0.6435253620147705, + 0.8007803559303284, + 0.75641268491745, + 0.2404799610376358, + 0.4282914996147156, + 0.5432339906692505, + 0.06181096285581589, + 0.6064548492431641, + -0.14456599950790405, + -0.41920095682144165, + -0.6729514002799988, + 0.8793071508407593, + 0.8623068928718567, + -0.32402414083480835, + -0.07150138169527054, + 0.6434568762779236, + 0.88515704870224, + 0.21528466045856476, + 0.22140704095363617, + -0.5896632075309753, + 0.8979973793029785, + 0.7852874398231506, + 0.8123403191566467, + 0.8474235534667969, + 0.8655924201011658, + 0.7264076471328735, + 0.7909120321273804, + 0.9000161290168762, + 0.9417259693145752 + ], + "mmd": [ + 0.04815273119155017, + 0.06074052217293269, + 0.03688021455182877, + 0.09841872969514198, + 0.05257198686617892, + 0.048210587017054496, + 0.05541616773008806, + 0.06700047887955451, + 0.04097015162820972, + 0.046145747955321115, + 0.07849290750313886, + 0.048816206222044856, + 0.05505021606142968, + 0.04019293080029007, + 0.05856732441886525, + 0.33019893545829726, + 0.13579950733767632, + 0.20068281452672132, + 0.06028831326091, + 0.042928606124033264, + 0.0534445280102398, + 0.08664477781784419, + 0.09999795686744917, + 0.06254987459822414, + 0.11979882486205196, + 0.14464882703222615, + 0.08477760615457197, + 0.0979552874192744, + 0.16559912036696056, + 0.027479590708141988, + 0.0730590977581963, + 0.13707189825710098, + 0.142240557809858, + 0.08017807712218428, + 0.038624811960458416, + 0.03095402065184638, + 0.032525491168835075, + 0.055946700566585905, + 0.051237771483223504, + 0.027638320273170036, + 0.13217243797016287, + 0.04101074458549969, + 0.06284430844974653, + 0.09971856153275505, + 0.04050774671659785, + 0.06572102889099285, + 0.04450568666422017, + 0.05780360174830579, + 0.21544134912422697, + 0.10662558145477796, + 0.05187789005652188, + 0.054935402342306205, + 0.02256307000752711, + 0.033003676723202036, + 0.04146709314606323, + 0.14247557884251738, + 0.03728431231214491, + 0.07637688237071116, + 0.054002603086681256, + 0.06088328166416501, + 0.08651676356840055, + 0.0850885582037082, + 0.03579828077120528, + 0.03400316539726933, + 0.06687339447069862, + 0.1944611998860074, + 0.038797889685476705, + 0.055220256432704784, + 0.05475970804374786, + 0.0847550668181084, + 0.12773568737180407, + 0.033478106533581076, + 0.03811196231845759, + 0.15473832779337748, + 0.08879762144197167, + 0.08684886163949379, + 0.06213105333927471, + 0.03368556941541623, + 0.03820113828777816, + 0.08166997232150652 + ], + "wasserstein": [ + 0.2127306705881155, + 0.2471152672247048, + 0.18492282883788458, + 0.28352613581280933, + 0.23484716853155646, + 0.2308920691547504, + 0.24489596048596646, + 0.20546309074012106, + 0.17949793220379043, + 0.2072011467969159, + 0.28650128739942793, + 0.18526216555058245, + 0.20262449355482995, + 0.17614206701057758, + 0.2219715779878481, + 0.7318313486632799, + 0.3781623548203691, + 0.45030565094364083, + 0.23491005897830708, + 0.2124095235757558, + 0.19689653768250526, + 0.26035468396861355, + 0.25516680710178397, + 0.23190232240587172, + 0.3284421965782437, + 0.3479976170261909, + 0.274554213613117, + 0.3199877160895611, + 0.42290863309092686, + 0.12670225040569533, + 0.23367881306238958, + 0.3536199792509216, + 0.31945975063851895, + 0.2580441486104863, + 0.1967653485696332, + 0.1665372162923777, + 0.20085098435116067, + 0.2339913399131008, + 0.2318684473126644, + 0.15563465398129922, + 0.43989677435679353, + 0.19700451428477386, + 0.2375503506753396, + 0.33287311082148235, + 0.18962746605005043, + 0.2848141224104961, + 0.20741579380213404, + 0.19330873642108512, + 0.5336956197350078, + 0.3328963069268954, + 0.22169475733731514, + 0.22221376467644458, + 0.14393574816790258, + 0.17527209675442726, + 0.18297853125049496, + 0.34752456427314166, + 0.1877186772274633, + 0.2658303637174756, + 0.23014086443234522, + 0.22949572658012352, + 0.2758802701404557, + 0.28390668093389243, + 0.17989317545353928, + 0.17772480066668028, + 0.2538875563767867, + 0.43773045482150763, + 0.18269750673412224, + 0.2370357340335426, + 0.23385495091392897, + 0.2246889285874779, + 0.3199832601192731, + 0.1891807558876498, + 0.16775380165094378, + 0.3972817907284407, + 0.26102043693263516, + 0.24507943290205772, + 0.19199010893732596, + 0.18024293231887925, + 0.15068706387509326, + 0.2615080955330509 + ] + }, + "n_perts": 80 + }, + "Additive": { + "mse": 6.217141449269548e-05, + "r2": 0.9044131524860859, + "pearson": 0.9642937988042831, + "spearman": 0.8814563908000537, + "de_corr": 0.9857571184635162, + "mmd": 0.015711531244365872, + "wasserstein": 0.10383054083704737, + "_per_pert": { + "mse": [ + 6.358845712384209e-05, + 5.455128484754823e-05, + 8.39567874209024e-05, + 6.620800559176132e-05, + 4.9343816499458626e-05, + 6.318344821920618e-05, + 4.701814759755507e-05, + 8.337821054738015e-05, + 3.776792073040269e-05, + 5.304530350258574e-05, + 8.071152115007862e-05, + 4.882572466158308e-05, + 5.386775592342019e-05, + 6.329737516352907e-05, + 5.384419273468666e-05, + 4.7785444621695206e-05, + 4.277037442079745e-05, + 7.100160291884094e-05, + 5.9298974520061165e-05, + 6.382067658705637e-05, + 0.00011488890595501289, + 4.190070103504695e-05, + 4.476463800529018e-05, + 3.3344433177262545e-05, + 6.278759974520653e-05, + 7.032293797237799e-05, + 5.178581704967655e-05, + 4.129415538045578e-05, + 0.00013808351650368422, + 5.2692892495542765e-05, + 3.871492299367674e-05, + 6.853123340988532e-05, + 4.011854252894409e-05, + 4.816174623556435e-05, + 9.827656322158873e-05, + 4.1837549360934645e-05, + 7.138287037378177e-05, + 5.880382377654314e-05, + 6.486114580184221e-05, + 7.624566205777228e-05, + 0.0001416207815054804, + 7.394661224680021e-05, + 7.319364522118121e-05, + 5.667660661856644e-05, + 5.310593405738473e-05, + 5.4894288041396067e-05, + 5.22498921782244e-05, + 4.0760598494671285e-05, + 5.282719575916417e-05, + 8.358619379578158e-05, + 4.238318797433749e-05, + 8.676692959852517e-05, + 7.266111788339913e-05, + 4.677872493630275e-05, + 4.26339429395739e-05, + 6.692691385978833e-05, + 4.619306491804309e-05, + 3.2342260965378955e-05, + 5.501074338098988e-05, + 3.810350244748406e-05, + 6.821587885497138e-05, + 4.9407677579438314e-05, + 4.915253884973936e-05, + 7.981059025041759e-05, + 4.797245856025256e-05, + 5.0550112064229324e-05, + 7.87429089541547e-05, + 6.768155435565859e-05, + 7.240824197651818e-05, + 7.198668026831001e-05, + 5.075075387139805e-05, + 0.00010831472172867507, + 5.523566869669594e-05, + 7.428437675116584e-05, + 4.796102803084068e-05, + 5.991084617562592e-05, + 5.398272696766071e-05, + 8.59129722812213e-05, + 5.301656710798852e-05, + 8.968803740572184e-05 + ], + "r2": [ + 0.971951961517334, + 0.9845261573791504, + 0.7291085720062256, + 0.9870725274085999, + 0.15037351846694946, + -0.6688671112060547, + 0.9172685146331787, + 0.9699516296386719, + 0.950839102268219, + 0.9767507314682007, + 0.943651556968689, + 0.9677395820617676, + 0.9802811741828918, + 0.9671372175216675, + 0.9810537099838257, + 0.9983946084976196, + 0.9944884777069092, + 0.9931017756462097, + 0.9485434293746948, + 0.9697842597961426, + 0.9461594820022583, + 0.9914299249649048, + 0.9885548949241638, + 0.9834702014923096, + 0.9854459762573242, + 0.9852902293205261, + 0.8632844686508179, + 0.975898265838623, + 0.9790283441543579, + 0.9540334939956665, + 0.9464935064315796, + 0.9785529375076294, + 0.9931175112724304, + 0.9796177744865417, + 0.8916500210762024, + 0.9690263271331787, + 0.8286341428756714, + 0.9688259959220886, + 0.9714473485946655, + 0.8849126100540161, + 0.9837841391563416, + 0.7831753492355347, + 0.9742277264595032, + 0.99165278673172, + 0.9670005440711975, + 0.9864408373832703, + 0.9829117059707642, + 0.9855301380157471, + 0.9964655637741089, + 0.9718765020370483, + 0.8877624273300171, + 0.9598885774612427, + 0.9016570448875427, + 0.8803067207336426, + 0.7325800657272339, + 0.9847800135612488, + 0.9063512086868286, + 0.9830450415611267, + 0.9358623027801514, + 0.8392977714538574, + 0.8921539187431335, + 0.888252317905426, + 0.7622535824775696, + 0.7347620725631714, + 0.8023017644882202, + 0.9918315410614014, + 0.8453631401062012, + 0.9769045114517212, + 0.7537655234336853, + 0.966542661190033, + 0.9874587059020996, + 0.6046467423439026, + 0.9529296159744263, + 0.9916355013847351, + 0.9891475439071655, + 0.9871048331260681, + 0.9708098769187927, + 0.9161993265151978, + 0.9443017840385437, + 0.9820659160614014 + ], + "pearson": [ + 0.9860634803771973, + 0.9922612309455872, + 0.9368342757225037, + 0.9938210844993591, + 0.7151593565940857, + 0.6851198077201843, + 0.9582955837249756, + 0.9917715191841125, + 0.9762698411941528, + 0.9888921976089478, + 0.9744163155555725, + 0.984936535358429, + 0.9901262521743774, + 0.9864653944969177, + 0.9906781315803528, + 0.9993131160736084, + 0.9973195791244507, + 0.996603786945343, + 0.9752287864685059, + 0.9851518869400024, + 0.9750480055809021, + 0.9967001676559448, + 0.9943097829818726, + 0.992939829826355, + 0.9927220940589905, + 0.992619514465332, + 0.9369444847106934, + 0.9883607029914856, + 0.9896386861801147, + 0.9792828559875488, + 0.9793886542320251, + 0.9895132780075073, + 0.9965946078300476, + 0.9927545785903931, + 0.9464207887649536, + 0.9847323894500732, + 0.9428672194480896, + 0.9846475124359131, + 0.9876149892807007, + 0.9515496492385864, + 0.9925107955932617, + 0.9203615188598633, + 0.988867461681366, + 0.9958518147468567, + 0.9834737181663513, + 0.9936906695365906, + 0.9915778040885925, + 0.9929901957511902, + 0.9984380006790161, + 0.9869527816772461, + 0.943321943283081, + 0.9827762246131897, + 0.9542627334594727, + 0.9616655707359314, + 0.8647212982177734, + 0.9934978485107422, + 0.9531835913658142, + 0.991603672504425, + 0.9731898307800293, + 0.943792462348938, + 0.9616348743438721, + 0.9434800148010254, + 0.8968539237976074, + 0.8840113878250122, + 0.8981503844261169, + 0.996063768863678, + 0.9291784167289734, + 0.9890373349189758, + 0.8816189765930176, + 0.9843573570251465, + 0.9937138557434082, + 0.8031525611877441, + 0.9764528274536133, + 0.996612548828125, + 0.9945841431617737, + 0.9943768382072449, + 0.9869089126586914, + 0.9573386311531067, + 0.9783027172088623, + 0.99156653881073 + ], + "spearman": [ + 0.928754037897496, + 0.9662856341162649, + 0.7712423317223546, + 0.9538470066656123, + 0.5254982774923528, + 0.4279684614640832, + 0.7535161070514943, + 0.9256812594170121, + 0.8059781033752303, + 0.9439584113967918, + 0.9418569017824902, + 0.853597620983766, + 0.9075210479446797, + 0.9078228270252171, + 0.9414555635296326, + 0.9836826297003729, + 0.9735050295185755, + 0.9601033309775765, + 0.8180553789211338, + 0.9449559748307674, + 0.8880253240747676, + 0.955223746033546, + 0.9304883385740754, + 0.9392927653215218, + 0.9847006109112382, + 0.9685470735942054, + 0.772152674001787, + 0.9381968136312746, + 0.9359579565337796, + 0.8867767826656825, + 0.9143740671164694, + 0.87861523173044, + 0.933451967564638, + 0.9516510151149955, + 0.8472735858661382, + 0.9563320676021244, + 0.8605308264041643, + 0.9035771875202717, + 0.9169836817209711, + 0.8676403532261543, + 0.9709023079468465, + 0.8057249297256938, + 0.9445558630162219, + 0.9770666528194629, + 0.9136036564105771, + 0.9620427144951494, + 0.9133989889208306, + 0.9530453420922315, + 0.9637273290535491, + 0.9182770885700292, + 0.762709177009884, + 0.9456816792917532, + 0.8724044789000969, + 0.8812599296229363, + 0.7689137070489291, + 0.8676631048075911, + 0.8538331906782323, + 0.9320992388535216, + 0.8736085233166716, + 0.8016224452781228, + 0.7838058005767556, + 0.8139382705279612, + 0.7640380814447771, + 0.7248113436840273, + 0.6310495722815624, + 0.9624693839699429, + 0.764183783478116, + 0.9729488139868683, + 0.7718651149259073, + 0.955629874042251, + 0.9372714725725099, + 0.7782401590662821, + 0.8598206760627484, + 0.9695133240803284, + 0.9066235614250336, + 0.9376104488731847, + 0.9195065407296033, + 0.908962142238411, + 0.8617409937271278, + 0.9452635534314202 + ], + "de_corr": [ + 0.9947866201400757, + 0.9978849291801453, + 0.9817653298377991, + 0.9979051947593689, + 0.7914469838142395, + 0.8467602133750916, + 0.9851346611976624, + 0.998081386089325, + 0.9945977926254272, + 0.9954056739807129, + 0.9908127188682556, + 0.9949299097061157, + 0.9951890707015991, + 0.9927459359169006, + 0.9968266487121582, + 0.9969210028648376, + 0.999174177646637, + 0.9993447661399841, + 0.9956825375556946, + 0.9956774711608887, + 0.9967861175537109, + 0.9989839196205139, + 0.9987612962722778, + 0.99863600730896, + 0.9979730248451233, + 0.9964622855186462, + 0.9721677899360657, + 0.9945791959762573, + 0.9977409243583679, + 0.9946585297584534, + 0.9937560558319092, + 0.9989275336265564, + 0.9993354082107544, + 0.9978781938552856, + 0.9859302043914795, + 0.9958522319793701, + 0.9835668206214905, + 0.9942353963851929, + 0.9941701889038086, + 0.994580864906311, + 0.9985277652740479, + 0.9780663251876831, + 0.9967221617698669, + 0.998862087726593, + 0.9968594312667847, + 0.9975262880325317, + 0.9957923293113708, + 0.9975109100341797, + 0.9991136193275452, + 0.9966052770614624, + 0.9887272715568542, + 0.9948974251747131, + 0.9774474501609802, + 0.982099175453186, + 0.963732123374939, + 0.9975007772445679, + 0.9890488386154175, + 0.9976304173469543, + 0.9898064732551575, + 0.9693389534950256, + 0.9868006706237793, + 0.9788329601287842, + 0.9584383964538574, + 0.9560388326644897, + 0.9878493547439575, + 0.998927116394043, + 0.9633113741874695, + 0.9982932806015015, + 0.9251469969749451, + 0.9957476854324341, + 0.9990358948707581, + 0.9357216358184814, + 0.9944741725921631, + 0.9989420771598816, + 0.9994568228721619, + 0.9986812472343445, + 0.9974274635314941, + 0.9622435569763184, + 0.9953940510749817, + 0.9979357719421387 + ], + "mmd": [ + 0.016841060167101185, + 0.025185536103346395, + 0.00650424480568057, + 0.018007946009995224, + 0.005276900286515729, + 0.0047951716618238605, + 0.005318043843850595, + 0.006539544131218711, + 0.0053703163640428375, + 0.020301075768090593, + 0.006387652390884457, + 0.005986328856037004, + 0.012877882892354808, + 0.022648679194408827, + 0.018091449451518415, + 0.1615524000204941, + 0.04031475476991431, + 0.05814363754493945, + 0.0125081817904662, + 0.02413725488255314, + 0.00883231015883168, + 0.017409795454908172, + 0.006171851605550227, + 0.005512983468654364, + 0.01482071091694015, + 0.01018569466247865, + 0.00564019991840925, + 0.0059286246857811475, + 0.01603426094942617, + 0.0058160677364580415, + 0.004083634724250995, + 0.00852207980310471, + 0.014809362879971033, + 0.014481929158790985, + 0.008797730686747052, + 0.013566701661770764, + 0.005956426571815832, + 0.013521835697006157, + 0.014008209741423472, + 0.0069334746001636915, + 0.07485885807624226, + 0.0048698782175189415, + 0.018123974447125546, + 0.04139145598158933, + 0.013924635422339904, + 0.03462428333296441, + 0.00880879432595394, + 0.011394711053495987, + 0.04215904151476546, + 0.00890768701875988, + 0.004092877957213248, + 0.014155437908497714, + 0.004658654291392672, + 0.006643954648709927, + 0.0064599970883122415, + 0.006411202693849538, + 0.006253334325526105, + 0.008112753776219561, + 0.0048208685518582595, + 0.005205635031059752, + 0.0062036754176885145, + 0.005015085703652167, + 0.003565974674840655, + 0.005460746298276753, + 0.005905389730931798, + 0.009896315345063966, + 0.007323984395733674, + 0.03202736291470576, + 0.009591384847650541, + 0.015158445677177168, + 0.020539063416579406, + 0.013252476490982823, + 0.00908367090920803, + 0.030262609758209047, + 0.013735821842606022, + 0.012317465848806664, + 0.005708121176376646, + 0.012106657871544968, + 0.005310486453022345, + 0.030757779095099358 + ], + "wasserstein": [ + 0.11461049509632605, + 0.14475240361445085, + 0.08561929500940207, + 0.10984875089016344, + 0.07706689629914125, + 0.07952681526623204, + 0.07381468392593868, + 0.08546519205804456, + 0.07912065191229584, + 0.11042784958190939, + 0.08338516436528463, + 0.07270111286571432, + 0.10121450110194313, + 0.1253985817372425, + 0.11169420872857984, + 0.32206615946974887, + 0.15198943659729125, + 0.18101912135854878, + 0.08714361697856891, + 0.1387264236010657, + 0.09242455916598895, + 0.10946052560507454, + 0.07494868157062466, + 0.07738067680700153, + 0.09403957483500985, + 0.09541571809208571, + 0.07147526371826174, + 0.08168573000401505, + 0.1231212606182357, + 0.07741225264755125, + 0.06714117708896163, + 0.09871611386720242, + 0.11213296736273184, + 0.10129082998106155, + 0.09848670471043089, + 0.10458482493173062, + 0.08226617862418932, + 0.10256117356187071, + 0.10749088095450675, + 0.08286629430073951, + 0.25948100375388455, + 0.07634897191327607, + 0.11357232935574835, + 0.1676532951907009, + 0.10927448935045993, + 0.15690294448439432, + 0.08708161610818449, + 0.09024959401019775, + 0.172701437391622, + 0.08296320051449546, + 0.07275490634769649, + 0.10726287684304965, + 0.07035903852252183, + 0.08458974719805112, + 0.0832136937602344, + 0.09237852940001108, + 0.08207674267989057, + 0.08429134039755475, + 0.07022177827465005, + 0.07058341208519466, + 0.07963033534025761, + 0.07950100101978784, + 0.0691450049112668, + 0.08458902349581109, + 0.07833367440387969, + 0.0888257569017147, + 0.08570302809325345, + 0.14504756115088882, + 0.10662990827622469, + 0.10445803784951936, + 0.1120415931678525, + 0.11321243413318305, + 0.08938778375061927, + 0.1469777127336923, + 0.11304827439695052, + 0.10046862963496078, + 0.07785675685909592, + 0.09589844239505003, + 0.0687340416885126, + 0.14250057027428442 + ] + }, + "n_perts": 80 + }, + "LinearResponse": { + "mse": 0.0004984360302842106, + "r2": 0.5501375116407872, + "pearson": 0.8542820053175092, + "spearman": 0.7082160020858496, + "de_corr": 0.899692427739501, + "mmd": 0.027366951640938747, + "wasserstein": 0.14565578982578203, + "_per_pert": { + "mse": [ + 0.00020294540445320308, + 0.0005620985175482929, + 0.0001466851244913414, + 0.0008508478640578687, + 0.0001888835395220667, + 0.00023906983551569283, + 0.0002584484755061567, + 0.00022406050993595272, + 0.00034762901486828923, + 0.0005040944670327008, + 0.0016050748527050018, + 0.0005801870720461011, + 0.00023654961842112243, + 0.00010895855666603893, + 0.000365526822861284, + 0.0022142697125673294, + 0.0008566092001274228, + 0.0019309882773086429, + 0.00037011850508861244, + 0.0003535088908392936, + 0.00033142915344797075, + 0.0005567747284658253, + 0.00044637746759690344, + 0.0003889205399900675, + 0.001067543402314186, + 0.0008696277509443462, + 0.00037888449151068926, + 0.0008522153366357088, + 0.0007503593224100769, + 0.00013662177661899477, + 0.00046608425327576697, + 0.0007999264053069055, + 0.0007989288424141705, + 0.0004481817304622382, + 0.0003756414807867259, + 0.0002054412179859355, + 0.0004507063131313771, + 0.0005636077839881182, + 0.0003158518811687827, + 0.00013165186101105064, + 0.0019314842065796256, + 0.00019735060050152242, + 0.00038043130189180374, + 0.001066615921445191, + 0.0002769654674921185, + 0.00045560175203718245, + 0.000387614534702152, + 0.0001853584690252319, + 0.0005421478999778628, + 0.00043487644870765507, + 0.00043518951861187816, + 0.0010219109244644642, + 0.00012969515228178352, + 0.00016198665252886713, + 0.00020460471569094807, + 0.000407417886890471, + 0.00014185914187692106, + 0.00046491879038512707, + 0.0003189982089679688, + 0.00023038414656184614, + 0.00032073259353637695, + 0.0005004144622944295, + 0.00012569598038680851, + 0.00018841509881895036, + 0.0003362650459166616, + 0.0007535432232543826, + 0.00023863041133154184, + 0.00045437042717821896, + 0.00021863645815756172, + 0.0005149806966073811, + 0.00047278727288357913, + 0.00023686166969127953, + 0.00021782887051813304, + 0.0011567577021196485, + 0.00031918162130750716, + 0.0007314173271879554, + 0.00028722023125737906, + 0.00017255770217161626, + 0.0002791571896523237, + 0.0004926866968162358 + ], + "r2": [ + 0.9104834198951721, + 0.8405570983886719, + 0.5267118811607361, + 0.833867073059082, + -2.2522902488708496, + -5.314563751220703, + 0.5452429056167603, + 0.9192516803741455, + 0.5475062131881714, + 0.7790603041648865, + -0.12057709693908691, + 0.6166555881500244, + 0.9134085178375244, + 0.9434307217597961, + 0.8713812232017517, + 0.9256094694137573, + 0.889614462852478, + 0.8123928904533386, + 0.6788305044174194, + 0.8326320052146912, + 0.8446820378303528, + 0.8861215114593506, + 0.885873556137085, + 0.8072009682655334, + 0.7525458931922913, + 0.8180959820747375, + -0.0002617835998535156, + 0.5025965571403503, + 0.8860379457473755, + 0.8808184266090393, + 0.3558422327041626, + 0.7496607303619385, + 0.862940788269043, + 0.8103278875350952, + 0.5858550071716309, + 0.8479053974151611, + -0.08199119567871094, + 0.70121169090271, + 0.8609583377838135, + 0.8012807369232178, + 0.778841495513916, + 0.4213328957557678, + 0.8660459518432617, + 0.8429113030433655, + 0.8278967142105103, + 0.8874639868736267, + 0.8732311129570007, + 0.9341984391212463, + 0.9637274146080017, + 0.8536811470985413, + -0.15245306491851807, + 0.5275815725326538, + 0.8244644999504089, + 0.5855228304862976, + -0.2833746671676636, + 0.9073482155799866, + 0.7124040722846985, + 0.7562729120254517, + 0.6280760765075684, + 0.028351783752441406, + 0.49293702840805054, + -0.13181161880493164, + 0.39201998710632324, + 0.3738325238227844, + -0.385775089263916, + 0.8782341480255127, + 0.5313730239868164, + 0.844951868057251, + 0.2564958333969116, + 0.7606519460678101, + 0.8831673264503479, + 0.13544458150863647, + 0.8143719434738159, + 0.8697479963302612, + 0.9277764558792114, + 0.8425703644752502, + 0.8446910977363586, + 0.8316850066184998, + 0.7067224979400635, + 0.9014817476272583 + ], + "pearson": [ + 0.9553652405738831, + 0.969968855381012, + 0.8447656631469727, + 0.9717710018157959, + 0.04141865670681, + 0.4897729456424713, + 0.7421504259109497, + 0.9633960723876953, + 0.8581640720367432, + 0.9137156009674072, + 0.5773282647132874, + 0.9260307550430298, + 0.9620983004570007, + 0.9748669266700745, + 0.9596258997917175, + 0.9959104061126709, + 0.9582720398902893, + 0.9825417399406433, + 0.849625825881958, + 0.9457164406776428, + 0.9486351013183594, + 0.9593232870101929, + 0.9413818717002869, + 0.9276975989341736, + 0.9317834377288818, + 0.9313573837280273, + 0.3643973171710968, + 0.7712037563323975, + 0.9472634792327881, + 0.9463781118392944, + 0.7002748847007751, + 0.8898004293441772, + 0.9356749057769775, + 0.9142096638679504, + 0.8735507726669312, + 0.9472244381904602, + 0.8568821549415588, + 0.8746569156646729, + 0.9581469893455505, + 0.8977189660072327, + 0.9688242673873901, + 0.7723768353462219, + 0.9672883152961731, + 0.9880933165550232, + 0.9337489604949951, + 0.9817197322845459, + 0.9404863715171814, + 0.971963107585907, + 0.9937583804130554, + 0.9471806287765503, + 0.5387371778488159, + 0.8035956621170044, + 0.9085114598274231, + 0.893273115158081, + 0.6330163478851318, + 0.9571194648742676, + 0.8461157083511353, + 0.8719350695610046, + 0.84953773021698, + 0.42979586124420166, + 0.7113834023475647, + 0.2996460497379303, + 0.8109048008918762, + 0.762006938457489, + 0.5708744525909424, + 0.9750235080718994, + 0.8265065550804138, + 0.9617883563041687, + 0.6185129880905151, + 0.9029694199562073, + 0.9512699246406555, + 0.7914345860481262, + 0.9101442694664001, + 0.9571062922477722, + 0.9756156206130981, + 0.9616542458534241, + 0.9415725469589233, + 0.923880398273468, + 0.9235188961029053, + 0.9696030616760254 + ], + "spearman": [ + 0.8783967413073205, + 0.9218610577716698, + 0.632194573979854, + 0.8530745444734925, + 0.14184671317980146, + -0.06788852636775047, + 0.5495938172740685, + 0.7449152255199077, + 0.5547412898385539, + 0.8486459591641482, + 0.6345847297130763, + 0.5044901724601015, + 0.7932897414591997, + 0.873515795869528, + 0.8390036449567131, + 0.9470357536490596, + 0.8730161750397624, + 0.9235162515597183, + 0.7058371261202829, + 0.8861514580009093, + 0.8183820676557142, + 0.7887392181743383, + 0.6182738694151647, + 0.7053859108691877, + 0.9108689604123363, + 0.8359216986500739, + 0.13523914204123139, + 0.7336524413425379, + 0.7549534501950945, + 0.8084273199458809, + 0.33434978760193135, + 0.7245197554109557, + 0.6570058623637992, + 0.8038874062874516, + 0.7065902253212664, + 0.801856079789077, + 0.7092521635316712, + 0.8528130982860563, + 0.832849087454282, + 0.8495879729350264, + 0.9391393619175764, + 0.534232142323203, + 0.8799837312356095, + 0.9379453014624228, + 0.8414422811011228, + 0.899997688649391, + 0.7286044011730729, + 0.7994584427070967, + 0.9285812011811466, + 0.8682697390742958, + 0.5912905982485406, + 0.7421600963139756, + 0.7946706610097459, + 0.708805788912545, + 0.6677888312169857, + 0.6521080103304608, + 0.6899849082184214, + 0.7281009333349744, + 0.686326245354338, + 0.2053184701014169, + 0.49403610528114905, + 0.3797537190967738, + 0.47587769574986005, + 0.38834781299469223, + 0.6305214325204073, + 0.8815451933429529, + 0.6017443298405181, + 0.8914367344778683, + 0.43020454077022324, + 0.8714021783054564, + 0.8584361797732419, + 0.7917183866646893, + 0.6616130088179717, + 0.8366501515210588, + 0.7856629232361578, + 0.7080149426859822, + 0.5524407865253126, + 0.795435385097224, + 0.5333749145944302, + 0.8444771490531603 + ], + "de_corr": [ + 0.9878745675086975, + 0.9908161163330078, + 0.9464258551597595, + 0.9864174127578735, + -0.24210572242736816, + 0.6830339431762695, + 0.8357012867927551, + 0.9947452545166016, + 0.9808493256568909, + 0.9252740740776062, + 0.6272894740104675, + 0.9828808903694153, + 0.9893450736999512, + 0.9938545823097229, + 0.9911743998527527, + 0.9758615493774414, + 0.9771857857704163, + 0.9978131055831909, + 0.9580230712890625, + 0.986320972442627, + 0.9829383492469788, + 0.9948559999465942, + 0.9868394136428833, + 0.9756239056587219, + 0.9576056003570557, + 0.9838659763336182, + 0.36021777987480164, + 0.7606475949287415, + 0.9847320318222046, + 0.9743720889091492, + 0.8681524991989136, + 0.9322993755340576, + 0.986332356929779, + 0.9799030423164368, + 0.9632763266563416, + 0.9876452088356018, + 0.9130681753158569, + 0.8733833432197571, + 0.9885754585266113, + 0.9748744368553162, + 0.9728558659553528, + 0.897865355014801, + 0.9813824892044067, + 0.9913594126701355, + 0.9592673182487488, + 0.9934391379356384, + 0.9663943648338318, + 0.996093213558197, + 0.9957377910614014, + 0.9854387640953064, + 0.5362005829811096, + 0.953355610370636, + 0.9207846522331238, + 0.9202982783317566, + 0.7318509221076965, + 0.9799919128417969, + 0.8983368277549744, + 0.9377652406692505, + 0.9507611989974976, + 0.6035137176513672, + 0.7984387278556824, + 0.2205519676208496, + 0.9360926151275635, + 0.8956354856491089, + 0.6634340286254883, + 0.9945654273033142, + 0.9227451682090759, + 0.9926457405090332, + 0.6688946485519409, + 0.9436094164848328, + 0.9760850071907043, + 0.9546233415603638, + 0.959308922290802, + 0.9857218265533447, + 0.9962118268013, + 0.9860618710517883, + 0.9887298345565796, + 0.9475215077400208, + 0.9801300168037415, + 0.9937041997909546 + ], + "mmd": [ + 0.020943700222722295, + 0.031880794580935956, + 0.009004759041664312, + 0.042771679300694476, + 0.010707747292729053, + 0.009327734806377053, + 0.013183615714926988, + 0.0117833952782338, + 0.01665001897788232, + 0.02695296758015242, + 0.054241871852460655, + 0.02618585403364404, + 0.02071325136531066, + 0.02262462851236635, + 0.02717752851318378, + 0.18618885262709517, + 0.05664922820895979, + 0.07815116940293265, + 0.017492010764348476, + 0.02534676466139485, + 0.016361140192948653, + 0.03650094156485628, + 0.019520328224244432, + 0.018230201503242727, + 0.040827649869538685, + 0.03223235627031995, + 0.019084745632382494, + 0.039459144425648685, + 0.03874378773480258, + 0.009511804069536778, + 0.021019342309507305, + 0.03495445898783578, + 0.03870706478073127, + 0.03258920307706237, + 0.02143195296234368, + 0.02306673977335305, + 0.017087246450358307, + 0.029044172274205415, + 0.019403724571881886, + 0.008734398727182513, + 0.08131021982913333, + 0.009448709088658802, + 0.025508250173183256, + 0.054419438167925716, + 0.024378812133345362, + 0.037715937700691704, + 0.01896491300941039, + 0.016638069730042115, + 0.04594106127737729, + 0.021716157453446572, + 0.018393609456176963, + 0.04659453957106596, + 0.0069786891009397944, + 0.010409445555680796, + 0.0116314378452681, + 0.019356753549959138, + 0.01116034992010817, + 0.02174432980318075, + 0.014306182824804559, + 0.011262755661791246, + 0.017074360488361195, + 0.02081185388646012, + 0.0069293096481983785, + 0.008660986419480321, + 0.021192760513829456, + 0.034517780438236056, + 0.011766737846828135, + 0.030822392490828077, + 0.015641505025393854, + 0.028304814650898624, + 0.03001098840250105, + 0.01851694517236191, + 0.01826798195059709, + 0.05790796315949043, + 0.022386754980449952, + 0.03164001570682151, + 0.012367883209614905, + 0.014483947601631186, + 0.01508344872008871, + 0.040600062970840844 + ], + "wasserstein": [ + 0.1357821808880377, + 0.1735686805685864, + 0.09838288113023844, + 0.18347872062319975, + 0.11774352181315029, + 0.1086515718722869, + 0.1212393307053829, + 0.09912982388303546, + 0.12632733909407615, + 0.1479754007762982, + 0.2187569738169526, + 0.15169347440259381, + 0.12452365686092699, + 0.12554592316429233, + 0.14618545882741663, + 0.3690380836322455, + 0.21816089268754113, + 0.2520513847759157, + 0.1195302137489152, + 0.15360441921680018, + 0.12175922504788379, + 0.15734948093930287, + 0.11938932718300027, + 0.12713581475236374, + 0.17668820097466453, + 0.16068310462239274, + 0.1263345487273218, + 0.18892790270906848, + 0.19043647799358124, + 0.08658047461837855, + 0.12749533624621986, + 0.17412833980625564, + 0.1896297986509988, + 0.16667797340843593, + 0.13597661749676565, + 0.13064643079559393, + 0.13647195283044314, + 0.15843727941922225, + 0.13422060973159203, + 0.09243345218790687, + 0.3024438420963396, + 0.0992192464647215, + 0.14606495023422789, + 0.2257946675450722, + 0.1396788068211299, + 0.1870177720522833, + 0.14014784763113186, + 0.10580875124783573, + 0.19056297925271332, + 0.14015785074704534, + 0.13592410984030706, + 0.19253524666833577, + 0.08542648930494848, + 0.10481687428695094, + 0.09788948622428782, + 0.1168223683321754, + 0.10498700834862783, + 0.1353198759995852, + 0.11138630209417866, + 0.10967427456989927, + 0.13014486116903132, + 0.14431700186019128, + 0.08630759161364956, + 0.09653450951695286, + 0.13415370818274808, + 0.16035748480820802, + 0.10390508693915869, + 0.15773027794945255, + 0.1321537132767957, + 0.1328409933953455, + 0.15084333522829435, + 0.13553245190793695, + 0.12077067198583526, + 0.2242563316714343, + 0.13529533671734453, + 0.15326658243084165, + 0.10235549645914924, + 0.1027978012181675, + 0.11204536335630272, + 0.17240355598464252 + ] + }, + "n_perts": 80 + }, + "NearestPerturbationCentroid": { + "mse": 6.217141449269548e-05, + "r2": 0.9044131524860859, + "pearson": 0.9642937988042831, + "spearman": 0.8814563908000537, + "de_corr": 0.9857571184635162, + "mmd": 0.015711531244365872, + "wasserstein": 0.10383054083704737, + "_per_pert": { + "mse": [ + 6.358845712384209e-05, + 5.455128484754823e-05, + 8.39567874209024e-05, + 6.620800559176132e-05, + 4.9343816499458626e-05, + 6.318344821920618e-05, + 4.701814759755507e-05, + 8.337821054738015e-05, + 3.776792073040269e-05, + 5.304530350258574e-05, + 8.071152115007862e-05, + 4.882572466158308e-05, + 5.386775592342019e-05, + 6.329737516352907e-05, + 5.384419273468666e-05, + 4.7785444621695206e-05, + 4.277037442079745e-05, + 7.100160291884094e-05, + 5.9298974520061165e-05, + 6.382067658705637e-05, + 0.00011488890595501289, + 4.190070103504695e-05, + 4.476463800529018e-05, + 3.3344433177262545e-05, + 6.278759974520653e-05, + 7.032293797237799e-05, + 5.178581704967655e-05, + 4.129415538045578e-05, + 0.00013808351650368422, + 5.2692892495542765e-05, + 3.871492299367674e-05, + 6.853123340988532e-05, + 4.011854252894409e-05, + 4.816174623556435e-05, + 9.827656322158873e-05, + 4.1837549360934645e-05, + 7.138287037378177e-05, + 5.880382377654314e-05, + 6.486114580184221e-05, + 7.624566205777228e-05, + 0.0001416207815054804, + 7.394661224680021e-05, + 7.319364522118121e-05, + 5.667660661856644e-05, + 5.310593405738473e-05, + 5.4894288041396067e-05, + 5.22498921782244e-05, + 4.0760598494671285e-05, + 5.282719575916417e-05, + 8.358619379578158e-05, + 4.238318797433749e-05, + 8.676692959852517e-05, + 7.266111788339913e-05, + 4.677872493630275e-05, + 4.26339429395739e-05, + 6.692691385978833e-05, + 4.619306491804309e-05, + 3.2342260965378955e-05, + 5.501074338098988e-05, + 3.810350244748406e-05, + 6.821587885497138e-05, + 4.9407677579438314e-05, + 4.915253884973936e-05, + 7.981059025041759e-05, + 4.797245856025256e-05, + 5.0550112064229324e-05, + 7.87429089541547e-05, + 6.768155435565859e-05, + 7.240824197651818e-05, + 7.198668026831001e-05, + 5.075075387139805e-05, + 0.00010831472172867507, + 5.523566869669594e-05, + 7.428437675116584e-05, + 4.796102803084068e-05, + 5.991084617562592e-05, + 5.398272696766071e-05, + 8.59129722812213e-05, + 5.301656710798852e-05, + 8.968803740572184e-05 + ], + "r2": [ + 0.971951961517334, + 0.9845261573791504, + 0.7291085720062256, + 0.9870725274085999, + 0.15037351846694946, + -0.6688671112060547, + 0.9172685146331787, + 0.9699516296386719, + 0.950839102268219, + 0.9767507314682007, + 0.943651556968689, + 0.9677395820617676, + 0.9802811741828918, + 0.9671372175216675, + 0.9810537099838257, + 0.9983946084976196, + 0.9944884777069092, + 0.9931017756462097, + 0.9485434293746948, + 0.9697842597961426, + 0.9461594820022583, + 0.9914299249649048, + 0.9885548949241638, + 0.9834702014923096, + 0.9854459762573242, + 0.9852902293205261, + 0.8632844686508179, + 0.975898265838623, + 0.9790283441543579, + 0.9540334939956665, + 0.9464935064315796, + 0.9785529375076294, + 0.9931175112724304, + 0.9796177744865417, + 0.8916500210762024, + 0.9690263271331787, + 0.8286341428756714, + 0.9688259959220886, + 0.9714473485946655, + 0.8849126100540161, + 0.9837841391563416, + 0.7831753492355347, + 0.9742277264595032, + 0.99165278673172, + 0.9670005440711975, + 0.9864408373832703, + 0.9829117059707642, + 0.9855301380157471, + 0.9964655637741089, + 0.9718765020370483, + 0.8877624273300171, + 0.9598885774612427, + 0.9016570448875427, + 0.8803067207336426, + 0.7325800657272339, + 0.9847800135612488, + 0.9063512086868286, + 0.9830450415611267, + 0.9358623027801514, + 0.8392977714538574, + 0.8921539187431335, + 0.888252317905426, + 0.7622535824775696, + 0.7347620725631714, + 0.8023017644882202, + 0.9918315410614014, + 0.8453631401062012, + 0.9769045114517212, + 0.7537655234336853, + 0.966542661190033, + 0.9874587059020996, + 0.6046467423439026, + 0.9529296159744263, + 0.9916355013847351, + 0.9891475439071655, + 0.9871048331260681, + 0.9708098769187927, + 0.9161993265151978, + 0.9443017840385437, + 0.9820659160614014 + ], + "pearson": [ + 0.9860634803771973, + 0.9922612309455872, + 0.9368342757225037, + 0.9938210844993591, + 0.7151593565940857, + 0.6851198077201843, + 0.9582955837249756, + 0.9917715191841125, + 0.9762698411941528, + 0.9888921976089478, + 0.9744163155555725, + 0.984936535358429, + 0.9901262521743774, + 0.9864653944969177, + 0.9906781315803528, + 0.9993131160736084, + 0.9973195791244507, + 0.996603786945343, + 0.9752287864685059, + 0.9851518869400024, + 0.9750480055809021, + 0.9967001676559448, + 0.9943097829818726, + 0.992939829826355, + 0.9927220940589905, + 0.992619514465332, + 0.9369444847106934, + 0.9883607029914856, + 0.9896386861801147, + 0.9792828559875488, + 0.9793886542320251, + 0.9895132780075073, + 0.9965946078300476, + 0.9927545785903931, + 0.9464207887649536, + 0.9847323894500732, + 0.9428672194480896, + 0.9846475124359131, + 0.9876149892807007, + 0.9515496492385864, + 0.9925107955932617, + 0.9203615188598633, + 0.988867461681366, + 0.9958518147468567, + 0.9834737181663513, + 0.9936906695365906, + 0.9915778040885925, + 0.9929901957511902, + 0.9984380006790161, + 0.9869527816772461, + 0.943321943283081, + 0.9827762246131897, + 0.9542627334594727, + 0.9616655707359314, + 0.8647212982177734, + 0.9934978485107422, + 0.9531835913658142, + 0.991603672504425, + 0.9731898307800293, + 0.943792462348938, + 0.9616348743438721, + 0.9434800148010254, + 0.8968539237976074, + 0.8840113878250122, + 0.8981503844261169, + 0.996063768863678, + 0.9291784167289734, + 0.9890373349189758, + 0.8816189765930176, + 0.9843573570251465, + 0.9937138557434082, + 0.8031525611877441, + 0.9764528274536133, + 0.996612548828125, + 0.9945841431617737, + 0.9943768382072449, + 0.9869089126586914, + 0.9573386311531067, + 0.9783027172088623, + 0.99156653881073 + ], + "spearman": [ + 0.928754037897496, + 0.9662856341162649, + 0.7712423317223546, + 0.9538470066656123, + 0.5254982774923528, + 0.4279684614640832, + 0.7535161070514943, + 0.9256812594170121, + 0.8059781033752303, + 0.9439584113967918, + 0.9418569017824902, + 0.853597620983766, + 0.9075210479446797, + 0.9078228270252171, + 0.9414555635296326, + 0.9836826297003729, + 0.9735050295185755, + 0.9601033309775765, + 0.8180553789211338, + 0.9449559748307674, + 0.8880253240747676, + 0.955223746033546, + 0.9304883385740754, + 0.9392927653215218, + 0.9847006109112382, + 0.9685470735942054, + 0.772152674001787, + 0.9381968136312746, + 0.9359579565337796, + 0.8867767826656825, + 0.9143740671164694, + 0.87861523173044, + 0.933451967564638, + 0.9516510151149955, + 0.8472735858661382, + 0.9563320676021244, + 0.8605308264041643, + 0.9035771875202717, + 0.9169836817209711, + 0.8676403532261543, + 0.9709023079468465, + 0.8057249297256938, + 0.9445558630162219, + 0.9770666528194629, + 0.9136036564105771, + 0.9620427144951494, + 0.9133989889208306, + 0.9530453420922315, + 0.9637273290535491, + 0.9182770885700292, + 0.762709177009884, + 0.9456816792917532, + 0.8724044789000969, + 0.8812599296229363, + 0.7689137070489291, + 0.8676631048075911, + 0.8538331906782323, + 0.9320992388535216, + 0.8736085233166716, + 0.8016224452781228, + 0.7838058005767556, + 0.8139382705279612, + 0.7640380814447771, + 0.7248113436840273, + 0.6310495722815624, + 0.9624693839699429, + 0.764183783478116, + 0.9729488139868683, + 0.7718651149259073, + 0.955629874042251, + 0.9372714725725099, + 0.7782401590662821, + 0.8598206760627484, + 0.9695133240803284, + 0.9066235614250336, + 0.9376104488731847, + 0.9195065407296033, + 0.908962142238411, + 0.8617409937271278, + 0.9452635534314202 + ], + "de_corr": [ + 0.9947866201400757, + 0.9978849291801453, + 0.9817653298377991, + 0.9979051947593689, + 0.7914469838142395, + 0.8467602133750916, + 0.9851346611976624, + 0.998081386089325, + 0.9945977926254272, + 0.9954056739807129, + 0.9908127188682556, + 0.9949299097061157, + 0.9951890707015991, + 0.9927459359169006, + 0.9968266487121582, + 0.9969210028648376, + 0.999174177646637, + 0.9993447661399841, + 0.9956825375556946, + 0.9956774711608887, + 0.9967861175537109, + 0.9989839196205139, + 0.9987612962722778, + 0.99863600730896, + 0.9979730248451233, + 0.9964622855186462, + 0.9721677899360657, + 0.9945791959762573, + 0.9977409243583679, + 0.9946585297584534, + 0.9937560558319092, + 0.9989275336265564, + 0.9993354082107544, + 0.9978781938552856, + 0.9859302043914795, + 0.9958522319793701, + 0.9835668206214905, + 0.9942353963851929, + 0.9941701889038086, + 0.994580864906311, + 0.9985277652740479, + 0.9780663251876831, + 0.9967221617698669, + 0.998862087726593, + 0.9968594312667847, + 0.9975262880325317, + 0.9957923293113708, + 0.9975109100341797, + 0.9991136193275452, + 0.9966052770614624, + 0.9887272715568542, + 0.9948974251747131, + 0.9774474501609802, + 0.982099175453186, + 0.963732123374939, + 0.9975007772445679, + 0.9890488386154175, + 0.9976304173469543, + 0.9898064732551575, + 0.9693389534950256, + 0.9868006706237793, + 0.9788329601287842, + 0.9584383964538574, + 0.9560388326644897, + 0.9878493547439575, + 0.998927116394043, + 0.9633113741874695, + 0.9982932806015015, + 0.9251469969749451, + 0.9957476854324341, + 0.9990358948707581, + 0.9357216358184814, + 0.9944741725921631, + 0.9989420771598816, + 0.9994568228721619, + 0.9986812472343445, + 0.9974274635314941, + 0.9622435569763184, + 0.9953940510749817, + 0.9979357719421387 + ], + "mmd": [ + 0.016841060167101185, + 0.025185536103346395, + 0.00650424480568057, + 0.018007946009995224, + 0.005276900286515729, + 0.0047951716618238605, + 0.005318043843850595, + 0.006539544131218711, + 0.0053703163640428375, + 0.020301075768090593, + 0.006387652390884457, + 0.005986328856037004, + 0.012877882892354808, + 0.022648679194408827, + 0.018091449451518415, + 0.1615524000204941, + 0.04031475476991431, + 0.05814363754493945, + 0.0125081817904662, + 0.02413725488255314, + 0.00883231015883168, + 0.017409795454908172, + 0.006171851605550227, + 0.005512983468654364, + 0.01482071091694015, + 0.01018569466247865, + 0.00564019991840925, + 0.0059286246857811475, + 0.01603426094942617, + 0.0058160677364580415, + 0.004083634724250995, + 0.00852207980310471, + 0.014809362879971033, + 0.014481929158790985, + 0.008797730686747052, + 0.013566701661770764, + 0.005956426571815832, + 0.013521835697006157, + 0.014008209741423472, + 0.0069334746001636915, + 0.07485885807624226, + 0.0048698782175189415, + 0.018123974447125546, + 0.04139145598158933, + 0.013924635422339904, + 0.03462428333296441, + 0.00880879432595394, + 0.011394711053495987, + 0.04215904151476546, + 0.00890768701875988, + 0.004092877957213248, + 0.014155437908497714, + 0.004658654291392672, + 0.006643954648709927, + 0.0064599970883122415, + 0.006411202693849538, + 0.006253334325526105, + 0.008112753776219561, + 0.0048208685518582595, + 0.005205635031059752, + 0.0062036754176885145, + 0.005015085703652167, + 0.003565974674840655, + 0.005460746298276753, + 0.005905389730931798, + 0.009896315345063966, + 0.007323984395733674, + 0.03202736291470576, + 0.009591384847650541, + 0.015158445677177168, + 0.020539063416579406, + 0.013252476490982823, + 0.00908367090920803, + 0.030262609758209047, + 0.013735821842606022, + 0.012317465848806664, + 0.005708121176376646, + 0.012106657871544968, + 0.005310486453022345, + 0.030757779095099358 + ], + "wasserstein": [ + 0.11461049509632605, + 0.14475240361445085, + 0.08561929500940207, + 0.10984875089016344, + 0.07706689629914125, + 0.07952681526623204, + 0.07381468392593868, + 0.08546519205804456, + 0.07912065191229584, + 0.11042784958190939, + 0.08338516436528463, + 0.07270111286571432, + 0.10121450110194313, + 0.1253985817372425, + 0.11169420872857984, + 0.32206615946974887, + 0.15198943659729125, + 0.18101912135854878, + 0.08714361697856891, + 0.1387264236010657, + 0.09242455916598895, + 0.10946052560507454, + 0.07494868157062466, + 0.07738067680700153, + 0.09403957483500985, + 0.09541571809208571, + 0.07147526371826174, + 0.08168573000401505, + 0.1231212606182357, + 0.07741225264755125, + 0.06714117708896163, + 0.09871611386720242, + 0.11213296736273184, + 0.10129082998106155, + 0.09848670471043089, + 0.10458482493173062, + 0.08226617862418932, + 0.10256117356187071, + 0.10749088095450675, + 0.08286629430073951, + 0.25948100375388455, + 0.07634897191327607, + 0.11357232935574835, + 0.1676532951907009, + 0.10927448935045993, + 0.15690294448439432, + 0.08708161610818449, + 0.09024959401019775, + 0.172701437391622, + 0.08296320051449546, + 0.07275490634769649, + 0.10726287684304965, + 0.07035903852252183, + 0.08458974719805112, + 0.0832136937602344, + 0.09237852940001108, + 0.08207674267989057, + 0.08429134039755475, + 0.07022177827465005, + 0.07058341208519466, + 0.07963033534025761, + 0.07950100101978784, + 0.0691450049112668, + 0.08458902349581109, + 0.07833367440387969, + 0.0888257569017147, + 0.08570302809325345, + 0.14504756115088882, + 0.10662990827622469, + 0.10445803784951936, + 0.1120415931678525, + 0.11321243413318305, + 0.08938778375061927, + 0.1469777127336923, + 0.11304827439695052, + 0.10046862963496078, + 0.07785675685909592, + 0.09589844239505003, + 0.0687340416885126, + 0.14250057027428442 + ] + }, + "n_perts": 80 + }, + "EndpointMLP": { + "mse": 6.292739490163513e-05, + "r2": 0.9027940891683102, + "pearson": 0.9636538296937942, + "spearman": 0.878917238319507, + "de_corr": 0.9855191633105278, + "mmd": 0.015748426511273127, + "wasserstein": 0.10409447609857056, + "_per_pert": { + "mse": [ + 6.708999717375264e-05, + 5.539525955100544e-05, + 8.383012027479708e-05, + 6.621752254432067e-05, + 4.967023414792493e-05, + 6.578655302291736e-05, + 4.6345783630385995e-05, + 8.164002792909741e-05, + 4.3806667235912755e-05, + 4.936349796480499e-05, + 7.925127283670008e-05, + 5.519063051906414e-05, + 5.5208452977240086e-05, + 6.21267463429831e-05, + 5.1618382713058963e-05, + 4.702899241237901e-05, + 5.2540908654918894e-05, + 6.678981299046427e-05, + 5.965422678855248e-05, + 6.322366243693978e-05, + 0.00011482256377348676, + 4.333732431405224e-05, + 4.7703473683213815e-05, + 3.301933247712441e-05, + 6.255486368900165e-05, + 7.136996282497421e-05, + 5.18833621754311e-05, + 4.114070543437265e-05, + 0.00014155502140056342, + 5.202973625273444e-05, + 4.169301246292889e-05, + 6.529282836709172e-05, + 3.825339808827266e-05, + 5.6975201005116105e-05, + 0.00010676537203835323, + 5.026084909331985e-05, + 7.3931114457082e-05, + 5.722414061892778e-05, + 6.614394078496844e-05, + 7.643792923772708e-05, + 0.00014190320507623255, + 7.332984387176111e-05, + 7.361447933362797e-05, + 5.584085010923445e-05, + 5.637219510390423e-05, + 5.6099859648384154e-05, + 5.2635514293797314e-05, + 3.692081372719258e-05, + 5.686992153641768e-05, + 8.262102346634492e-05, + 4.309977521188557e-05, + 8.343355148099363e-05, + 7.204698340501636e-05, + 4.642907515517436e-05, + 4.51056839665398e-05, + 6.200321513460949e-05, + 4.6290268073789775e-05, + 3.131070843664929e-05, + 5.877536386833526e-05, + 3.775371806113981e-05, + 6.874331302242354e-05, + 5.5421493016183376e-05, + 4.8592424718663096e-05, + 8.007010910660028e-05, + 4.626718146027997e-05, + 4.8517784307477996e-05, + 7.766880298731849e-05, + 6.710625893902034e-05, + 7.218175596790388e-05, + 7.057966286083683e-05, + 5.8126228395849466e-05, + 0.00010695461969589815, + 5.5690401495667174e-05, + 7.474873564206064e-05, + 4.928838097839616e-05, + 6.009908611304127e-05, + 5.8287270803702995e-05, + 8.471967157674953e-05, + 5.452929326565936e-05, + 8.997018449008465e-05 + ], + "r2": [ + 0.9704074859619141, + 0.9842867851257324, + 0.7295172214508057, + 0.987070620059967, + 0.144753098487854, + -0.7376235723495483, + 0.9184515476226807, + 0.9705780744552612, + 0.9429787397384644, + 0.9783644676208496, + 0.9446710348129272, + 0.9635341167449951, + 0.9797903895378113, + 0.9677449464797974, + 0.9818369150161743, + 0.998420000076294, + 0.9932293891906738, + 0.9935109615325928, + 0.9482351541519165, + 0.9700669050216675, + 0.946190595626831, + 0.9911361336708069, + 0.9878035187721252, + 0.9836313724517822, + 0.985499918460846, + 0.9850712418556213, + 0.8630269765853882, + 0.9759878516197205, + 0.9785010814666748, + 0.9546120166778564, + 0.9423776268959045, + 0.9795663952827454, + 0.9934374690055847, + 0.9758878946304321, + 0.8822911381721497, + 0.962790310382843, + 0.822516679763794, + 0.9696634411811829, + 0.9708826541900635, + 0.8846223950386047, + 0.9837518334388733, + 0.7849838137626648, + 0.9740795493125916, + 0.9917758703231812, + 0.9649709463119507, + 0.9861430525779724, + 0.9827855825424194, + 0.9868932366371155, + 0.9961950778961182, + 0.9722012877464294, + 0.8858647346496582, + 0.9614295363426208, + 0.9024882316589355, + 0.881201446056366, + 0.717076301574707, + 0.9858997464179993, + 0.9061541557312012, + 0.983585774898529, + 0.9314730763435364, + 0.8407729864120483, + 0.8913201093673706, + 0.8746504783630371, + 0.764962911605835, + 0.7338997721672058, + 0.8093293905258179, + 0.9921599626541138, + 0.8474724292755127, + 0.9771008491516113, + 0.7545356750488281, + 0.9671965837478638, + 0.9856361746788025, + 0.6096112728118896, + 0.9525421261787415, + 0.9915832281112671, + 0.9888471364974976, + 0.9870643019676208, + 0.968482255935669, + 0.9173632860183716, + 0.9427124857902527, + 0.9820094704627991 + ], + "pearson": [ + 0.9852494597434998, + 0.9921461939811707, + 0.9375391006469727, + 0.9938251972198486, + 0.7035747170448303, + 0.6782786250114441, + 0.9588940739631653, + 0.9916643500328064, + 0.9725679159164429, + 0.9895926713943481, + 0.9747257232666016, + 0.9827229380607605, + 0.9898760914802551, + 0.9865573048591614, + 0.9911340475082397, + 0.9993064999580383, + 0.9967168569564819, + 0.9967845678329468, + 0.9750925302505493, + 0.9853084683418274, + 0.9751153588294983, + 0.9965492486953735, + 0.9939506649971008, + 0.9930532574653625, + 0.9927512407302856, + 0.9925076961517334, + 0.9368858337402344, + 0.9884697198867798, + 0.9893973469734192, + 0.9795076847076416, + 0.9782508611679077, + 0.9898988008499146, + 0.9967449307441711, + 0.9908532500267029, + 0.9411527514457703, + 0.9816414713859558, + 0.9387611150741577, + 0.9852072596549988, + 0.9874569773674011, + 0.9515001177787781, + 0.9924895763397217, + 0.9210497140884399, + 0.9888485670089722, + 0.9959120750427246, + 0.9824343323707581, + 0.9935444593429565, + 0.9915280342102051, + 0.9934820532798767, + 0.9983013868331909, + 0.9871103763580322, + 0.9423218965530396, + 0.9836190938949585, + 0.9547141194343567, + 0.9615911245346069, + 0.8573949933052063, + 0.993951141834259, + 0.9530937075614929, + 0.9918825030326843, + 0.9715781807899475, + 0.9439722299575806, + 0.9607856273651123, + 0.9358734488487244, + 0.897705078125, + 0.8836725950241089, + 0.9024578928947449, + 0.9961386919021606, + 0.9295045733451843, + 0.9891447424888611, + 0.881629467010498, + 0.9847153425216675, + 0.9928039908409119, + 0.8035243153572083, + 0.976223886013031, + 0.9965784549713135, + 0.9944816827774048, + 0.99436354637146, + 0.9858863353729248, + 0.9579552412033081, + 0.9772290587425232, + 0.9915999174118042 + ], + "spearman": [ + 0.9247389658139282, + 0.9666749633313562, + 0.7741870799982957, + 0.9568640024719045, + 0.5119637504383395, + 0.4090430341404888, + 0.7552429765065244, + 0.9240999433691723, + 0.7721917134725167, + 0.9532066205295645, + 0.945866739073486, + 0.8252384879612369, + 0.9085154629845178, + 0.8970392356748532, + 0.9492284815836262, + 0.9894818697284727, + 0.9705078757288254, + 0.9609398066783439, + 0.8160243655265026, + 0.9473448300726514, + 0.8889006579754026, + 0.9562225120732154, + 0.9311517651519454, + 0.9412405666198546, + 0.9847965775675867, + 0.9694107749615005, + 0.7704127655037638, + 0.9401817163404503, + 0.9378935360503373, + 0.8871775314328123, + 0.9149046726704704, + 0.8810318017322667, + 0.9346247627245935, + 0.9527139811474018, + 0.8341679351838843, + 0.9479001333771477, + 0.8472616593629554, + 0.8994093716171545, + 0.9144730037999829, + 0.8675864001090166, + 0.9709025584469126, + 0.8074595656103529, + 0.9441858804162341, + 0.9800879176336939, + 0.9137451022071432, + 0.9595445228399819, + 0.9138274220350082, + 0.9627218697882334, + 0.9650113626755285, + 0.9179573517298681, + 0.7607965299698858, + 0.9499763322812458, + 0.8706251019245583, + 0.8755313141091496, + 0.7590601406088224, + 0.8711797230558571, + 0.8542452730470491, + 0.9328068653944116, + 0.8615248196694362, + 0.8031904586918822, + 0.7863175947395554, + 0.7926365272848733, + 0.7670243509119896, + 0.7262452240666942, + 0.6057758811219447, + 0.9672421764810294, + 0.7644005400425957, + 0.9737342231941182, + 0.7729230097050592, + 0.9569878620372653, + 0.9272042874160312, + 0.7792968966826231, + 0.8441505915002373, + 0.9706043137321899, + 0.9082760341461665, + 0.937903598451309, + 0.8900082867709616, + 0.9077912057934044, + 0.8530922534782718, + 0.9475217653826264 + ], + "de_corr": [ + 0.9942957162857056, + 0.9977801442146301, + 0.9815174341201782, + 0.9978957176208496, + 0.7909258008003235, + 0.8448208570480347, + 0.9853066205978394, + 0.9980451464653015, + 0.9937025904655457, + 0.9957481026649475, + 0.9911707043647766, + 0.9973901510238647, + 0.9947229623794556, + 0.9927114844322205, + 0.9967469573020935, + 0.9972017407417297, + 0.9990983009338379, + 0.9990567564964294, + 0.9956392049789429, + 0.9957026839256287, + 0.9967947006225586, + 0.9990550875663757, + 0.9988176822662354, + 0.9986791014671326, + 0.9979770183563232, + 0.9965268969535828, + 0.9715136885643005, + 0.9946168661117554, + 0.997495174407959, + 0.9947057366371155, + 0.9925470948219299, + 0.9986593127250671, + 0.9993278980255127, + 0.997967004776001, + 0.9826313853263855, + 0.9950172901153564, + 0.9827412962913513, + 0.9944544434547424, + 0.9940246343612671, + 0.9945003390312195, + 0.9984996318817139, + 0.9782575368881226, + 0.9966461658477783, + 0.9989110231399536, + 0.9969537854194641, + 0.9977381825447083, + 0.995879054069519, + 0.9975757598876953, + 0.9991405010223389, + 0.9967517256736755, + 0.987899124622345, + 0.9955812692642212, + 0.9773091077804565, + 0.9811846017837524, + 0.9609137773513794, + 0.9975593090057373, + 0.9892810583114624, + 0.9976965188980103, + 0.9902707934379578, + 0.9693315625190735, + 0.9855972528457642, + 0.9762381315231323, + 0.9590563774108887, + 0.9558709263801575, + 0.9877315759658813, + 0.9987295269966125, + 0.9630563855171204, + 0.9982094764709473, + 0.9248974919319153, + 0.9958739280700684, + 0.9984982013702393, + 0.9329190850257874, + 0.9948785305023193, + 0.9989084005355835, + 0.9996547102928162, + 0.9986801147460938, + 0.9969043731689453, + 0.9627537727355957, + 0.9942911863327026, + 0.9978713989257812 + ], + "mmd": [ + 0.01697289349760034, + 0.025216787101855598, + 0.006490050101595424, + 0.01801047410904133, + 0.005260396915893861, + 0.00485481030173518, + 0.005307197828463228, + 0.00645707516972116, + 0.005623766793173668, + 0.020213100267693296, + 0.006448472856907861, + 0.006134701037055934, + 0.012913147162261107, + 0.022648904763079347, + 0.018047731431059644, + 0.16157552545596865, + 0.04069245191608806, + 0.05811648773215239, + 0.012552125335620556, + 0.024107765833548123, + 0.008836037851075451, + 0.017471704188886328, + 0.006395014153265377, + 0.005475211746353126, + 0.014833488355661228, + 0.010254938486484622, + 0.00561523491837379, + 0.005982012847287277, + 0.016110769616693665, + 0.0057979381099325655, + 0.0042327703932031335, + 0.008619701600393603, + 0.014804481588537866, + 0.014988598376726059, + 0.009098681130604858, + 0.013796110946287632, + 0.006088592046181507, + 0.013486465471092002, + 0.014052307058756264, + 0.0069362180774755755, + 0.07485796575325998, + 0.004856281700145182, + 0.018119296119353145, + 0.041371075396380186, + 0.013962224964035208, + 0.03463891351438175, + 0.008800728988020756, + 0.011264282475704879, + 0.042153368115514955, + 0.008913875278265704, + 0.00409505206305405, + 0.014022875978586091, + 0.004648861029505014, + 0.00661772242738301, + 0.006537494275201894, + 0.006193671508200271, + 0.006264827356513458, + 0.008082117894056329, + 0.004910284039773005, + 0.005193365452132559, + 0.006313369466909413, + 0.005194174939594176, + 0.0035473429961695446, + 0.005470558247793855, + 0.005922543361892907, + 0.009935852913680154, + 0.007284920341690815, + 0.03202269919212797, + 0.00957876196413665, + 0.015111704173584317, + 0.02081463419304319, + 0.013190018816024995, + 0.009119422643439123, + 0.03027454366975757, + 0.01381061256175764, + 0.012309375503663289, + 0.005732678161216809, + 0.01209323906541282, + 0.005369482256067815, + 0.030753759530633196 + ], + "wasserstein": [ + 0.11564006052578744, + 0.1448019771095738, + 0.08582345919944898, + 0.11007193912833813, + 0.07693702192551703, + 0.08024249717839138, + 0.07363956827747041, + 0.08524113713842495, + 0.08092429538259353, + 0.10957655722890389, + 0.08388460268331337, + 0.07380313295859117, + 0.10160115481078856, + 0.12613718517194578, + 0.1110066862447972, + 0.32199182242765084, + 0.152774469335376, + 0.1825195943353846, + 0.08725570769915433, + 0.13849143999474006, + 0.09272535060098186, + 0.1094888340690039, + 0.0740572235394164, + 0.07702853227837318, + 0.09406383071177021, + 0.09562892377112227, + 0.07132462495439496, + 0.0821559038612481, + 0.12331511119165686, + 0.07767819741830109, + 0.06737837690657952, + 0.0980172092062936, + 0.11213109078881162, + 0.10452793331603719, + 0.10112173869393538, + 0.10571010672928764, + 0.0819761469838921, + 0.10299406602622241, + 0.10766251681488188, + 0.08299417188723444, + 0.25962272883965287, + 0.0763680970619384, + 0.11369677930836238, + 0.16742333412318947, + 0.10913614216549405, + 0.15688705495982305, + 0.08682633466869807, + 0.0911672270497975, + 0.1728808351735947, + 0.08313470480336349, + 0.07262667073153378, + 0.10686626275738366, + 0.07033215162675646, + 0.08484835532922297, + 0.0836783876434036, + 0.09157255434986084, + 0.08209871610953877, + 0.08395559330366764, + 0.07113164490736859, + 0.07040942959172401, + 0.08120986288881364, + 0.08165521354607157, + 0.06899177700643278, + 0.08476617052271157, + 0.07769228664623287, + 0.08890913060712212, + 0.08544268188186249, + 0.14509019861968134, + 0.1066856055613169, + 0.10438923237117391, + 0.11326348867288205, + 0.11301626502837442, + 0.08920153245653653, + 0.14716613196573533, + 0.11260635472080897, + 0.10027702876954382, + 0.07919553587028198, + 0.09584703167047198, + 0.07055602504685711, + 0.14258933102272442 + ] + }, + "n_perts": 80 + }, + "Random": { + "mse": 0.006092429294949397, + "r2": -6.315681910514831, + "pearson": 0.2638963776291348, + "spearman": 0.23929168463922937, + "de_corr": 0.28753160405904055, + "mmd": 0.15737417639572437, + "wasserstein": 0.3688159423665479, + "_per_pert": { + "mse": [ + 0.002402307465672493, + 0.004195447079837322, + 0.0006361445412039757, + 0.009395080618560314, + 0.012778481468558311, + 0.0006785371224395931, + 0.0014660859014838934, + 0.0034031241666525602, + 0.004246317315846682, + 0.0019368961220607162, + 0.003456242149695754, + 0.0012951076496392488, + 0.04393398389220238, + 0.003903804812580347, + 0.0020256005227565765, + 0.028605172410607338, + 0.008414411917328835, + 0.012168889865279198, + 0.0015441725263372064, + 0.005603116005659103, + 0.002097423654049635, + 0.004118550568819046, + 0.007477012928575277, + 0.0026175028178840876, + 0.0060502635315060616, + 0.004159062635153532, + 0.004687070846557617, + 0.001920380163937807, + 0.00443612365052104, + 0.0015401471173390746, + 0.009433695115149021, + 0.0062116775661706924, + 0.006738309748470783, + 0.006019380409270525, + 0.003790156915783882, + 0.007245635148137808, + 0.0020910659804940224, + 0.0160241536796093, + 0.014354241080582142, + 0.0026967148296535015, + 0.002848976757377386, + 0.0004916107864119112, + 0.017872290685772896, + 0.005542703438550234, + 0.01413249783217907, + 0.004136521369218826, + 0.00542051624506712, + 0.0061881388537585735, + 0.0041547296568751335, + 0.004251834936439991, + 0.0021680796053260565, + 0.0036042500287294388, + 0.0021248720586299896, + 0.0015880349092185497, + 0.0034471757244318724, + 0.03019036166369915, + 0.0010212529450654984, + 0.003162569832056761, + 0.004866892006248236, + 0.002804333111271262, + 0.003269520355388522, + 0.002369440160691738, + 0.010178696364164352, + 0.0039037868846207857, + 0.0005680227186530828, + 0.021610943600535393, + 0.004392051137983799, + 0.002448400715366006, + 0.006485868711024523, + 0.0018588462844491005, + 0.006605895236134529, + 0.0025344123132526875, + 0.0018712510354816914, + 0.01107819564640522, + 0.001949599594809115, + 0.001971254823729396, + 0.004494506865739822, + 0.005265197716653347, + 0.004057257901877165, + 0.004666061140596867 + ], + "r2": [ + -0.05962681770324707, + -0.19006550312042236, + -1.0525572299957275, + -0.8344441652297974, + -219.0263214111328, + -16.922237396240234, + -1.5796747207641602, + -0.22643911838531494, + -4.527247905731201, + 0.15107744932174683, + -1.4129624366760254, + 0.14428943395614624, + -15.082496643066406, + -1.0267832279205322, + 0.2872472405433655, + 0.03898119926452637, + -0.08430933952331543, + -0.1822807788848877, + -0.3399519920349121, + -1.6527824401855469, + 0.01708221435546875, + 0.15762287378311157, + -0.9116668701171875, + -0.2975708246231079, + -0.40243732929229736, + 0.13002967834472656, + -11.373950958251953, + -0.12084770202636719, + 0.32625657320022583, + -0.34354233741760254, + -12.037961959838867, + -0.9439632892608643, + -0.1559818983078003, + -1.5474228858947754, + -3.178650379180908, + -4.364173412322998, + -4.019932746887207, + -7.494968414306641, + -5.318906784057617, + -3.0705008506774902, + 0.6737868785858154, + -0.44149041175842285, + -5.293030261993408, + 0.18368369340896606, + -7.781774520874023, + -0.02174198627471924, + -0.7727741003036499, + -1.1967658996582031, + 0.7220262885093689, + -0.4305751323699951, + -4.741427898406982, + -0.6662060022354126, + -1.875901222229004, + -3.063324451446533, + -20.622272491455078, + -5.865653991699219, + -1.0704212188720703, + -0.6579322814941406, + -4.674371242523193, + -10.827315330505371, + -4.168955326080322, + -4.359076976776123, + -48.23342514038086, + -11.973611831665039, + -1.3408663272857666, + -2.492135524749756, + -7.625194549560547, + 0.16451436281204224, + -21.056119918823242, + 0.13606220483779907, + -0.6324138641357422, + -8.250713348388672, + -0.5946309566497803, + -0.24741530418395996, + 0.5588501691818237, + 0.57570880651474, + -1.43031907081604, + -4.135742664337158, + -3.262481689453125, + 0.06696879863739014 + ], + "pearson": [ + 0.4504935145378113, + -0.13283415138721466, + 0.4251806139945984, + 0.4822196662425995, + 0.15605668723583221, + 0.5742374062538147, + 0.2586584687232971, + 0.480623722076416, + -0.09263838827610016, + 0.5690841674804688, + 0.17027851939201355, + 0.39418986439704895, + -0.011929434724152088, + -0.394300639629364, + 0.5698360204696655, + 0.2346334159374237, + -0.04427238181233406, + 0.2446848303079605, + 0.0607655867934227, + 0.3866860270500183, + 0.33614951372146606, + 0.6881710290908813, + -0.1862211376428604, + 0.4062153697013855, + 0.4847392737865448, + 0.36445531249046326, + -0.10360473394393921, + -0.21076178550720215, + 0.59562087059021, + 0.40827304124832153, + -0.46062472462654114, + 0.030764510855078697, + 0.41456618905067444, + -0.019829051569104195, + 0.5754253268241882, + 0.3902358114719391, + -0.1667598932981491, + 0.022567080333828926, + 0.3219994604587555, + 0.5605472326278687, + 0.8218475580215454, + -0.07401151955127716, + 0.17492778599262238, + 0.5394904017448425, + 0.4875725507736206, + 0.35946759581565857, + 0.27537187933921814, + 0.5639602541923523, + 0.8561677932739258, + 0.0584145113825798, + -0.38602375984191895, + 0.5460142493247986, + 0.6441411972045898, + 0.6753813624382019, + 0.4387378990650177, + -0.12956371903419495, + 0.42351919412612915, + 0.2690277695655823, + 0.2620769739151001, + -0.16458925604820251, + -0.287692666053772, + -0.46148163080215454, + 0.5450911521911621, + 0.33677220344543457, + -0.3229620158672333, + 0.5577889680862427, + 0.16407634317874908, + 0.4226599633693695, + 0.24057769775390625, + 0.3872202932834625, + 0.3393458425998688, + 0.4501216411590576, + 0.7168939113616943, + 0.0900026261806488, + 0.7519155740737915, + 0.7795405983924866, + -0.3993484377861023, + 0.4303949475288391, + 0.18644821643829346, + 0.308832049369812 + ], + "spearman": [ + 0.4056474857571728, + 0.23695835277738536, + 0.3057300034245046, + 0.48207576203097463, + 0.06642322020178819, + 0.34813535015862157, + 0.24893938618569866, + 0.24417769484082263, + -0.022576229207302484, + 0.5726242288222297, + 0.31052372182300075, + 0.2782208991458696, + 0.09140415236927071, + -0.07623818435336932, + 0.01363657784835198, + 0.16071488365863995, + 0.014098391214926087, + 0.18122107618133257, + 0.16351443345868905, + 0.25993955277389086, + 0.19731396998215844, + 0.43193676818510124, + -0.0062628654026136085, + 0.34798027623674366, + 0.5567263853263541, + 0.5085525734699681, + 0.054019150504303344, + -0.2459572563097371, + 0.3302795650195041, + 0.3906411456058653, + -0.21215042981075108, + 0.2897107716974299, + 0.15623453144509059, + 0.11490032691402474, + 0.45374052170322327, + 0.5532514159415025, + 0.027743913737388718, + 0.34171225904746444, + 0.3368576326594536, + 0.44958695788475855, + 0.7494987405535383, + -0.03351725169491655, + 0.42947116936511837, + 0.4540819249331137, + 0.5349985569227443, + 0.45664416319566953, + -0.15852093105527348, + 0.13997284715774339, + 0.4044144195631996, + -0.06055284131899073, + 0.01765813440954022, + 0.5019899744068206, + 0.4387339191533164, + 0.3848568904484333, + 0.44052487181507516, + -0.04180610751590934, + 0.3675949134546772, + 0.3193137025089033, + 0.22233451958514594, + -0.20234711242550546, + -0.08814145500832644, + -0.11812416192001322, + 0.3120118545871948, + 0.21257665528940858, + -0.4058342104373145, + 0.4895797584378588, + 0.19337498835695002, + 0.494310381686152, + 0.14695007302645052, + 0.5532915664522026, + 0.48121683317474234, + 0.3605638318937812, + 0.5296107745898754, + 0.18522911419679342, + 0.3360627979367592, + 0.3767443079023581, + -0.054947338769834264, + 0.25984604346943246, + -0.11047968056570379, + 0.2621597604274068 + ], + "de_corr": [ + 0.5993392467498779, + -0.047075171023607254, + 0.634308397769928, + 0.5540454387664795, + 0.3528956174850464, + 0.671830415725708, + 0.2947024703025818, + 0.6701884269714355, + -0.153262659907341, + 0.7730339169502258, + -0.07747335731983185, + 0.4394450783729553, + -0.19941459596157074, + -0.4704377055168152, + 0.802310049533844, + -0.3516404628753662, + -0.36197778582572937, + 0.3600388467311859, + -0.24024201929569244, + 0.8475308418273926, + 0.5463124513626099, + 0.91432785987854, + -0.4714749753475189, + 0.41529780626296997, + 0.4806523323059082, + 0.37103721499443054, + 0.0050836531445384026, + -0.3464639484882355, + 0.6940740942955017, + 0.4635506272315979, + -0.6946463584899902, + -0.4086593687534332, + 0.556554913520813, + 0.0392739474773407, + 0.8293206095695496, + 0.7663671970367432, + -0.3522750437259674, + -0.2326701581478119, + 0.5773414373397827, + 0.870473325252533, + 0.870704710483551, + -0.07824219763278961, + 0.09914015978574753, + 0.44841501116752625, + 0.502975344657898, + 0.5024886131286621, + 0.38604170083999634, + 0.8946154713630676, + 0.9346277713775635, + 0.004612135700881481, + -0.6283481121063232, + 0.6691251993179321, + 0.5567365884780884, + 0.7437829375267029, + 0.6066128015518188, + -0.6871744394302368, + 0.4355708062648773, + 0.1420619636774063, + 0.25829219818115234, + -0.22035910189151764, + -0.38775962591171265, + -0.6642442345619202, + 0.8170205950737, + 0.49835702776908875, + -0.22967806458473206, + 0.8586680889129639, + -0.0231960229575634, + 0.631873369216919, + 0.1405947506427765, + 0.4230232834815979, + 0.014657549560070038, + 0.636929988861084, + 0.8541913628578186, + -0.029359793290495872, + 0.8847939968109131, + 0.9192421436309814, + -0.5979225039482117, + 0.48590341210365295, + 0.4526309669017792, + 0.7534998655319214 + ], + "mmd": [ + 0.0809143516169093, + 0.10588783521307521, + 0.02815118788882398, + 0.2527909504739875, + 0.3570001915815567, + 0.02241923448308869, + 0.05712625178175679, + 0.1083705554345693, + 0.13857612679904074, + 0.06244041649200072, + 0.10638810836247914, + 0.041412748848822845, + 0.6913780205505058, + 0.09585122935459756, + 0.06830665458839591, + 0.35169678557338213, + 0.17390920915552355, + 0.2666237997121147, + 0.04476341536819051, + 0.1555678813967648, + 0.06566832249449339, + 0.14166246904831148, + 0.2000638018402262, + 0.09070905201493906, + 0.1809746983246231, + 0.11698364922825433, + 0.15641764428484906, + 0.06597259075933104, + 0.15238316265569363, + 0.05631616640217996, + 0.28882902770010355, + 0.20164829461155642, + 0.18997509206920837, + 0.16717023345626947, + 0.11507045994792442, + 0.20263163619730828, + 0.05960045931489044, + 0.36131602161932164, + 0.3423693597510471, + 0.09503090709883877, + 0.11297322198565818, + 0.01945287433600973, + 0.4062361118970391, + 0.1463318294361018, + 0.3485198043269099, + 0.121233175181654, + 0.16237900199542765, + 0.19019277117000644, + 0.14202143166593928, + 0.13257433618514303, + 0.07303888847335238, + 0.1253977368614112, + 0.06966961642637692, + 0.05546865845565796, + 0.11893769788392161, + 0.5992419972333981, + 0.044546423932852686, + 0.1004200228136306, + 0.15818403465273034, + 0.10077193528978101, + 0.11414741060725386, + 0.08347654725606657, + 0.281421654236773, + 0.12010791754956063, + 0.024241049079596788, + 0.47815373161088826, + 0.13581965738559987, + 0.0658093569123035, + 0.2103817519492598, + 0.06554846533720038, + 0.2017760922740024, + 0.08559098396727027, + 0.07601663342241816, + 0.26321095308948517, + 0.06932827705118738, + 0.07228630687977344, + 0.1379388971621901, + 0.15498256334127436, + 0.13466325588443329, + 0.12707303296345485 + ], + "wasserstein": [ + 0.2652066467685652, + 0.37399134076421753, + 0.16734133526496447, + 0.4255766143211449, + 0.6453680302483691, + 0.15132115378438635, + 0.20443826926424233, + 0.31368083616586145, + 0.3381372990633475, + 0.274038125236219, + 0.33299961435916997, + 0.1919495077097343, + 0.9799792150519271, + 0.33537069728501956, + 0.23409305885683993, + 0.7778874911336777, + 0.42799276272655584, + 0.5360810875311652, + 0.19475707305772677, + 0.38778176666521874, + 0.28164829148264386, + 0.344889030274866, + 0.3464288452422015, + 0.2924875933810338, + 0.42429805939244875, + 0.3265834254057435, + 0.3549672363800947, + 0.2515531022803798, + 0.42602756776003825, + 0.21505773752172833, + 0.4689532342094136, + 0.46607501680584634, + 0.365827156635943, + 0.36209436369188575, + 0.35382014598908446, + 0.48375852141022757, + 0.2395031840446391, + 0.7108998474374776, + 0.6739819414840466, + 0.2767550755163291, + 0.40637535421504445, + 0.1439145138559787, + 0.6557894159637516, + 0.3881383334056901, + 0.6342545818012664, + 0.3441210021271051, + 0.40868147948198114, + 0.45052487097450067, + 0.34682116674131097, + 0.4027359922962763, + 0.22081378749382596, + 0.3300649244792871, + 0.26731230548086865, + 0.205896381926555, + 0.30556426048508434, + 0.7931256598688856, + 0.2041052274536696, + 0.2816024826384169, + 0.31240927744890906, + 0.29397843440961574, + 0.2902744366571957, + 0.2585257156381198, + 0.5339397824199847, + 0.32678902211372324, + 0.15070922860545785, + 0.6353298976816326, + 0.33165910301242796, + 0.27644656925361594, + 0.416049188409383, + 0.20803500084541324, + 0.4307615311707081, + 0.2891896376201367, + 0.25560148528564436, + 0.5743985757102005, + 0.2518945278059192, + 0.233498755738341, + 0.33096669134400614, + 0.37122319800976217, + 0.36949387588202676, + 0.35065841196768427 + ] + }, + "n_perts": 80 + } + } +} \ No newline at end of file diff --git a/experiments/results/norman_forward_combination.json b/experiments/results/norman_forward_combination.json new file mode 100644 index 0000000000000000000000000000000000000000..7020a3d45fc06d3274526952388d35460b1f3b1f --- /dev/null +++ b/experiments/results/norman_forward_combination.json @@ -0,0 +1,1669 @@ +{ + "split": "combination", + "models": { + "PIVOT": { + "mse": 0.0011608018271195202, + "r2": 0.7034275646393116, + "pearson": 0.9094234865445358, + "spearman": 0.7422449962055115, + "de_corr": 0.9502354585207425, + "mmd": 0.30546490356834105, + "wasserstein": 0.4372436159231492, + "_per_pert": { + "mse": [ + 0.0004416794399730861, + 0.00033747247653082013, + 0.0004790037346538156, + 0.0006593593861907721, + 0.0004620901891030371, + 0.0019741933792829514, + 0.0037908623926341534, + 0.001514257164672017, + 0.00016269889601971954, + 0.0012954982230439782, + 0.0012997586745768785, + 0.0005281568737700582, + 0.0017190042417496443, + 0.0003682815295178443, + 0.0005863922997377813, + 0.0018724900437518954, + 0.00015000069106463343, + 0.00014376378385350108, + 0.0016738612903282046, + 0.0009087433572858572, + 0.001752177020534873, + 0.0003636993351392448, + 0.0020773408468812704, + 0.0024218547623604536, + 0.0026488068979233503, + 0.0005494005745276809 + ], + "r2": [ + 0.3020176291465759, + 0.9467234015464783, + 0.9203470945358276, + 0.8184603452682495, + 0.9147472381591797, + 0.5887033939361572, + 0.6299736499786377, + 0.938554048538208, + 0.9600407481193542, + 0.4582985043525696, + 0.6972436904907227, + 0.9535616636276245, + 0.6415849924087524, + 0.7591631412506104, + 0.8760426044464111, + 0.7594628930091858, + 0.9689065217971802, + 0.7848913073539734, + 0.18005633354187012, + -0.09197616577148438, + 0.5074808597564697, + 0.9264187812805176, + 0.7165902853012085, + 0.644216001033783, + 0.7914367914199829, + 0.6961709260940552 + ], + "pearson": [ + 0.6286964416503906, + 0.9733661413192749, + 0.9625992774963379, + 0.9056428670883179, + 0.9792198538780212, + 0.9477250576019287, + 0.9555656313896179, + 0.9880457520484924, + 0.9798969030380249, + 0.7223176956176758, + 0.9317120909690857, + 0.9887338280677795, + 0.8971215486526489, + 0.9108695983886719, + 0.9660959243774414, + 0.9386588931083679, + 0.9844915866851807, + 0.9293400049209595, + 0.7490290999412537, + 0.701932430267334, + 0.9207367897033691, + 0.9665771126747131, + 0.9677329659461975, + 0.8525486588478088, + 0.9600315690040588, + 0.9363229274749756 + ], + "spearman": [ + 0.40532229870442155, + 0.7695981981365192, + 0.7168365810557625, + 0.8223511142479002, + 0.7619216037663427, + 0.683857225754397, + 0.6145115214042277, + 0.9030108665319926, + 0.8265676943402535, + 0.671723228500967, + 0.6946859011442998, + 0.8733894077156299, + 0.7313993757164831, + 0.7732291276055234, + 0.89247673992973, + 0.8519675614383306, + 0.8543754041910453, + 0.7109876899782194, + 0.5988971571019566, + 0.571165396466269, + 0.5966738634873373, + 0.8095972592400602, + 0.7516057058029206, + 0.7712747612701276, + 0.919446070868832, + 0.7214981469437507 + ], + "de_corr": [ + 0.8499228954315186, + 0.9913710951805115, + 0.9797207117080688, + 0.9553540349006653, + 0.9917449355125427, + 0.9736267328262329, + 0.9834495186805725, + 0.9824050664901733, + 0.9921419620513916, + 0.8035094738006592, + 0.9676457047462463, + 0.9952547550201416, + 0.9743742942810059, + 0.9676927924156189, + 0.9835843443870544, + 0.9907316565513611, + 0.9951936602592468, + 0.9855618476867676, + 0.8434659838676453, + 0.6738124489784241, + 0.9695654511451721, + 0.9861806631088257, + 0.9885909557342529, + 0.9232415556907654, + 0.9831457734107971, + 0.974833607673645 + ], + "mmd": [ + 0.328915026572134, + 0.2825841063486447, + 0.294318340899585, + 0.3250979304211864, + 0.2985747716434175, + 0.3383447859580423, + 0.3599959851521064, + 0.2681938456592685, + 0.25684975999905224, + 0.2912402456812845, + 0.2955167755485727, + 0.3050651947717583, + 0.33313845325313596, + 0.26395905100391637, + 0.3135891655418249, + 0.32101054965927234, + 0.2707002241951575, + 0.26835427290232927, + 0.262577515584691, + 0.27464787193905105, + 0.3378149350365237, + 0.32967945988472336, + 0.3673476519636928, + 0.3334991821371782, + 0.34538214127359135, + 0.2756902497467274 + ], + "wasserstein": [ + 0.38160696400595895, + 0.4137511200401532, + 0.42309166187224867, + 0.4809079089532095, + 0.43214627874244366, + 0.46716996259261256, + 0.5414571257715426, + 0.5464605045876217, + 0.3775509286352024, + 0.41904336459342434, + 0.4467201824337285, + 0.4392330632078002, + 0.4642640756035464, + 0.3592642491196178, + 0.4572561755616178, + 0.45067202461840195, + 0.3663159121643666, + 0.3444241465740951, + 0.3829438513983993, + 0.3762235401968839, + 0.4657051158337028, + 0.4276944648936299, + 0.48315019492358374, + 0.509800186190512, + 0.5455922505392994, + 0.36588876094827727 + ] + }, + "n_perts": 26 + }, + "MeanControl": { + "mse": 0.0057471211117584835, + "r2": -0.011610170969596276, + "pearson": 0.03853007166001659, + "spearman": 0.01234826030847139, + "de_corr": 0.0049495832779659675, + "mmd": 0.13235756195115472, + "wasserstein": 0.3782550852396613, + "_per_pert": { + "mse": [ + 0.0007026545936241746, + 0.00680745393037796, + 0.006161879748106003, + 0.0036344523541629314, + 0.005524597596377134, + 0.004370624199509621, + 0.010359177365899086, + 0.02515377104282379, + 0.004014274105429649, + 0.0023135184310376644, + 0.004061258863657713, + 0.011318846605718136, + 0.004867125768214464, + 0.0015870737843215466, + 0.004860573913902044, + 0.007731732912361622, + 0.004952032584697008, + 0.0006888253847137094, + 0.001931409235112369, + 0.0008697910816408694, + 0.0035772169940173626, + 0.004746480844914913, + 0.007469563279300928, + 0.00716841034591198, + 0.012592131271958351, + 0.00196027266792953 + ], + "r2": [ + -0.11039936542510986, + -0.07468903064727783, + -0.0246506929397583, + -0.0006638765335083008, + -0.019253849983215332, + 0.08943921327590942, + -0.011159896850585938, + -0.020696282386779785, + 0.01408451795578003, + 0.03262203931808472, + 0.05399984121322632, + 0.004787087440490723, + -0.014803051948547363, + -0.03786289691925049, + -0.027476072311401367, + 0.006793856620788574, + -0.02650141716003418, + -0.030665278434753418, + 0.05389589071273804, + -0.045169830322265625, + -0.005519390106201172, + 0.039723873138427734, + -0.0190657377243042, + -0.05307948589324951, + 0.008513808250427246, + -0.0840684175491333 + ], + "pearson": [ + -0.06629537045955658, + -0.3966469168663025, + -0.12479717284440994, + 0.05818164348602295, + -0.07915422320365906, + 0.4844202995300293, + -0.050288137048482895, + 0.02881428971886635, + 0.17225602269172668, + 0.21480095386505127, + 0.30700379610061646, + 0.08251570165157318, + 0.0075753843411803246, + -0.06009141728281975, + -0.041397422552108765, + 0.11025799065828323, + -0.029640095308423042, + 0.032515063881874084, + 0.27604448795318604, + 0.041267454624176025, + 0.045245762914419174, + 0.3109055161476135, + -0.07629498839378357, + -0.18719244003295898, + 0.14141690731048584, + -0.19964122772216797 + ], + "spearman": [ + 0.04353725647206709, + -0.09233009961360504, + -0.17818829737082245, + -0.12565003140590206, + -0.11508361946294095, + 0.3707466312197973, + 0.16069679863595812, + 0.00032152283484183805, + -0.06183391666424053, + 0.025292726933603096, + 0.14768816927659872, + -0.05732592687687896, + 0.1341233180441605, + -0.07679896723652788, + 0.03204796381481832, + 0.11720680407351838, + -0.18430154293547427, + 0.01611448405055293, + 0.28831915469517444, + 0.10102569790815605, + -0.17818684883817582, + 0.08296864864116403, + -0.12180483509557406, + -0.21667246809319538, + 0.24106632652097706, + -0.03192418150779438 + ], + "de_corr": [ + -0.22437791526317596, + -0.5656072497367859, + 0.1137598380446434, + 0.23108525574207306, + -0.14381539821624756, + 0.7256581783294678, + -0.21843844652175903, + -0.5244619846343994, + 0.6701590418815613, + 0.3335712254047394, + 0.7878367304801941, + 0.3845817446708679, + -0.5479661822319031, + -0.3156905472278595, + -0.11130276322364807, + 0.4411935806274414, + -0.19360119104385376, + 0.0018395064398646355, + 0.42608770728111267, + -0.3652479946613312, + 0.27641230821609497, + 0.6019760966300964, + -0.43252789974212646, + -0.5774248242378235, + -0.24419526755809784, + -0.40081438422203064 + ], + "mmd": [ + 0.028212891729032075, + 0.17668493966754995, + 0.1551985733846034, + 0.09521743587710563, + 0.14369480219229347, + 0.11806902970722166, + 0.2255878636555937, + 0.31327826181329843, + 0.08330115947701278, + 0.07780785262113332, + 0.12215313731187416, + 0.26980985180555317, + 0.12880152264843836, + 0.056574219217009114, + 0.10613605760518652, + 0.2244930029396509, + 0.1676700115849149, + 0.024761279159099425, + 0.06210051842261466, + 0.029301764564502952, + 0.10289429100830971, + 0.13306709379342874, + 0.19283435202935384, + 0.14094517867540513, + 0.21403328006563993, + 0.04866823977419643 + ], + "wasserstein": [ + 0.15157757597968538, + 0.44020539156686445, + 0.4400235845707495, + 0.3607232100291323, + 0.3959694677861648, + 0.3512882785085011, + 0.5483459678380452, + 0.7275385804687302, + 0.29424641074921526, + 0.31549036084220616, + 0.395544556183622, + 0.6093649157257086, + 0.37510882071090257, + 0.22370848710780677, + 0.38830919854877083, + 0.5176736723988615, + 0.4112948989728112, + 0.1353916600626855, + 0.2595916480410795, + 0.16596665988480586, + 0.3249977138314772, + 0.3761359704317928, + 0.4527599415596157, + 0.4134669910667745, + 0.5471746097544797, + 0.21273364361070476 + ] + }, + "n_perts": 26 + }, + "AvgPerturbationEffect": { + "mse": 0.0044067181834879406, + "r2": 0.0399630069732666, + "pearson": 0.4885468087159097, + "spearman": 0.4042017919807703, + "de_corr": 0.46895806663311446, + "mmd": 0.11475870164747928, + "wasserstein": 0.327025581030484, + "_per_pert": { + "mse": [ + 0.001267462968826294, + 0.004218101501464844, + 0.003647373989224434, + 0.0017841884400695562, + 0.003240479389205575, + 0.0029262227471917868, + 0.006986001040786505, + 0.021046752110123634, + 0.004038598854094744, + 0.0022836565040051937, + 0.0026010992005467415, + 0.00744654843583703, + 0.005217214580625296, + 0.0011601600563153625, + 0.0025345073081552982, + 0.007477449253201485, + 0.004120431840419769, + 0.002499110298231244, + 0.0016265985323116183, + 0.0012811636552214622, + 0.0017465396085754037, + 0.0027206726372241974, + 0.004879365675151348, + 0.004667452536523342, + 0.009905749000608921, + 0.0032517726067453623 + ], + "r2": [ + -1.0029613971710205, + 0.3340906500816345, + 0.3934832215309143, + 0.5087642669677734, + 0.4021516442298889, + 0.39036089181900024, + 0.3180961012840271, + 0.14595919847488403, + 0.0081101655960083, + 0.04510855674743652, + 0.3941187858581543, + 0.34526002407073975, + -0.08779716491699219, + 0.24131619930267334, + 0.46423083543777466, + 0.03945869207382202, + 0.14588022232055664, + -2.739330291748047, + 0.2032080888748169, + -0.539488673210144, + 0.5090654492378235, + 0.44957178831100464, + 0.3343126177787781, + 0.3143252730369568, + 0.22003579139709473, + -0.7982927560806274 + ], + "pearson": [ + 0.3604811429977417, + 0.6058692932128906, + 0.6697478294372559, + 0.7310587167739868, + 0.6734141111373901, + 0.6347340941429138, + 0.6224637031555176, + 0.4709996283054352, + 0.28053054213523865, + 0.40062958002090454, + 0.6288027167320251, + 0.687199592590332, + 0.1971733123064041, + 0.5907549262046814, + 0.7059608697891235, + 0.24624839425086975, + 0.3904935121536255, + -0.28574952483177185, + 0.5447945594787598, + 0.44264665246009827, + 0.7273120880126953, + 0.6910309195518494, + 0.6204204559326172, + 0.5835672616958618, + 0.49274808168411255, + -0.01111543271690607 + ], + "spearman": [ + -0.005572146720350216, + 0.4049987336190409, + 0.48805908653659147, + 0.7226606894210917, + 0.3415249398944683, + 0.34360029244832807, + 0.41841257715861885, + 0.5685465523120167, + 0.45542969537201383, + 0.5155350259801306, + 0.4223747608625305, + 0.4883545880796307, + 0.46737821638369603, + 0.4138993150624696, + 0.7499677014287711, + 0.3411502579224064, + 0.07087614538849275, + 0.012027375959805959, + 0.398113071496426, + 0.4623981108419023, + 0.4823332341005564, + 0.4203638150922295, + 0.24832150593271687, + 0.5368074709487687, + 0.5605109316946306, + 0.18117464428304272 + ], + "de_corr": [ + 0.7283901572227478, + 0.7522621154785156, + 0.8509905934333801, + 0.7136895656585693, + 0.9145503640174866, + 0.8487303256988525, + 0.8070111274719238, + -0.658985435962677, + -0.14219823479652405, + 0.4001394510269165, + 0.8591786026954651, + 0.7900312542915344, + -0.12902699410915375, + 0.7461597919464111, + 0.5489757061004639, + 0.3416403830051422, + 0.4684438407421112, + -0.4527846872806549, + 0.5128503441810608, + 0.27120205760002136, + 0.838915228843689, + 0.858460009098053, + 0.8084298372268677, + 0.5075000524520874, + 0.2661384046077728, + -0.2577841281890869 + ], + "mmd": [ + 0.05001841963404674, + 0.1243999716578329, + 0.10577308021160592, + 0.06989012677420037, + 0.10020200287508596, + 0.08902702227400527, + 0.1724092508063214, + 0.2911497647074187, + 0.10117398552149726, + 0.07722481336758369, + 0.08886264226803986, + 0.20220013329751296, + 0.14174360407776765, + 0.04268554686292081, + 0.07242410458620052, + 0.22036877414645706, + 0.15004829617682658, + 0.08273019820860938, + 0.054775785790464826, + 0.046795093836730195, + 0.06360700855966939, + 0.09240974717766504, + 0.1414321989080809, + 0.11456708778898483, + 0.1887216315143322, + 0.09908595180460111 + ], + "wasserstein": [ + 0.19751347198629352, + 0.32000839841210266, + 0.3027903984557414, + 0.28081650843299333, + 0.29404827368705755, + 0.27698321473773685, + 0.42078244654825653, + 0.6576770736997155, + 0.31715481247827343, + 0.2870991900162981, + 0.31031270858544463, + 0.4710812588113828, + 0.3964049639929377, + 0.17318106225096142, + 0.2922675958777241, + 0.46861032907271394, + 0.362768244503924, + 0.2712758174607812, + 0.22606261821831544, + 0.21748638476438, + 0.22856126139946967, + 0.2629585976820611, + 0.34642352983307817, + 0.32831178931941957, + 0.48006523217092956, + 0.31201992439459203 + ] + }, + "n_perts": 26 + }, + "Additive": { + "mse": 0.0016535368653981446, + "r2": 0.573949531866954, + "pearson": 0.9300772272623502, + "spearman": 0.7897218029307761, + "de_corr": 0.9704181047586294, + "mmd": 0.059320614216664985, + "wasserstein": 0.2123106030642661, + "_per_pert": { + "mse": [ + 0.00015997662558220327, + 0.0005177605198696256, + 0.001115658087655902, + 0.002104917075484991, + 0.0004899363848380744, + 0.000492266786750406, + 0.001916507724672556, + 0.013743879273533821, + 0.0027093931566923857, + 0.0013348587090149522, + 0.0014990427298471332, + 0.0015931000234559178, + 0.0004326222406234592, + 0.0010920573258772492, + 0.000530772958882153, + 0.0008865304989740252, + 0.0003558291064109653, + 0.0004061798972543329, + 0.0022346421610563993, + 0.0011294002179056406, + 0.0008639924344606698, + 0.0004070299328304827, + 0.000788895704317838, + 0.0006908114883117378, + 0.0008649550727568567, + 0.004630942363291979 + ], + "r2": [ + 0.7471902370452881, + 0.9182614088058472, + 0.8144786953926086, + 0.42045891284942627, + 0.9096097946166992, + 0.8974428176879883, + 0.8129295706748962, + 0.442297101020813, + 0.33456629514694214, + 0.4418402910232544, + 0.6508238315582275, + 0.8599262237548828, + 0.9097976088523865, + 0.2858518362045288, + 0.8877999782562256, + 0.886117696762085, + 0.9262405633926392, + 0.3922472596168518, + -0.094643235206604, + -0.35712456703186035, + 0.7571404576301575, + 0.917652428150177, + 0.892371654510498, + 0.8985159397125244, + 0.9318946599960327, + -1.5609996318817139 + ], + "pearson": [ + 0.8696218132972717, + 0.980620265007019, + 0.9799976944923401, + 0.89031982421875, + 0.954730212688446, + 0.959190845489502, + 0.933533251285553, + 0.9897459149360657, + 0.9803408980369568, + 0.8745859265327454, + 0.8584630489349365, + 0.9879471659660339, + 0.9558621048927307, + 0.7636538743972778, + 0.9562665820121765, + 0.9557057023048401, + 0.9704350233078003, + 0.9516004920005798, + 0.8845965266227722, + 0.8278399109840393, + 0.8941793441772461, + 0.9762647747993469, + 0.9550750851631165, + 0.9652448892593384, + 0.9663643836975098, + 0.8998223543167114 + ], + "spearman": [ + 0.6874229546359486, + 0.7854118225005448, + 0.8183332800064441, + 0.8041241358867101, + 0.7646113842689339, + 0.7176599546225205, + 0.688587194942726, + 0.8911605604049929, + 0.9065612036885211, + 0.8493612223751926, + 0.6984521228159735, + 0.8898582004776359, + 0.8367760643784031, + 0.6926421240891262, + 0.8416687892109902, + 0.8655633996685086, + 0.744568430661192, + 0.779303185064447, + 0.8037996979768682, + 0.6723784886738736, + 0.6205288290863296, + 0.8576187371998566, + 0.753541923090875, + 0.9028960344131053, + 0.9308670020760561, + 0.7290701339844011 + ], + "de_corr": [ + 0.965126633644104, + 0.9906924366950989, + 0.9944162964820862, + 0.9439982771873474, + 0.9877852201461792, + 0.9841868877410889, + 0.965343713760376, + 0.9861399531364441, + 0.9939308166503906, + 0.9483758211135864, + 0.9121333360671997, + 0.9934125542640686, + 0.9876887798309326, + 0.9351226091384888, + 0.9721375107765198, + 0.9926180243492126, + 0.9922538995742798, + 0.9765479564666748, + 0.9229952692985535, + 0.8940964341163635, + 0.9726075530052185, + 0.9875404834747314, + 0.9810596108436584, + 0.9823089838027954, + 0.9831912517547607, + 0.9851604104042053 + ], + "mmd": [ + 0.00869688189204465, + 0.027503277129409254, + 0.043831266173979144, + 0.11028029639922188, + 0.02509863159265069, + 0.022246669756055937, + 0.06793987733277973, + 0.26240850873556854, + 0.11436110231190588, + 0.047902975380531565, + 0.058675209843414455, + 0.05252364628086359, + 0.024561532527645036, + 0.04192828122725545, + 0.03601482869048189, + 0.03251179337071386, + 0.015227312009062022, + 0.01838109460109527, + 0.06996368945967801, + 0.04406407510763066, + 0.031062906260774903, + 0.022564580166961834, + 0.03708039271306218, + 0.062240337885000274, + 0.08853296664586607, + 0.17673383613963678 + ], + "wasserstein": [ + 0.10009500007567787, + 0.1606439628824183, + 0.20915482549802097, + 0.30069878232411873, + 0.15155494231937924, + 0.15263269444013755, + 0.25857470502279023, + 0.5519766869965191, + 0.2578558580673624, + 0.22031762226370546, + 0.22408907335770042, + 0.24397260914311938, + 0.1512310505937172, + 0.17671375569923664, + 0.16866251349216616, + 0.16318874698223187, + 0.14428973430566278, + 0.12879515475904585, + 0.25087982720080154, + 0.205900811513284, + 0.17909877213727576, + 0.14794281224682468, + 0.1865181168327959, + 0.21668269487861277, + 0.2571256575227321, + 0.31147926911558194 + ] + }, + "n_perts": 26 + }, + "LinearResponse": { + "mse": 0.0007943772249676001, + "r2": 0.7440571349400741, + "pearson": 0.9199725710428678, + "spearman": 0.7693219288655411, + "de_corr": 0.9511560132870307, + "mmd": 0.04490069678916941, + "wasserstein": 0.1850864126089564, + "_per_pert": { + "mse": [ + 0.0002673232520464808, + 0.0004400323086883873, + 0.0005671681137755513, + 0.0006946227513253689, + 0.00020604993915185332, + 0.0004807524965144694, + 0.0011033053742721677, + 0.0013256630627438426, + 0.00016457772289868444, + 0.0014632371021434665, + 0.0010216188384220004, + 0.00042015797225758433, + 0.0011820258805528283, + 0.0008684452041052282, + 0.0002365340042160824, + 0.001679806038737297, + 0.00034780616988427937, + 0.0003253179893363267, + 0.0013649225002154708, + 0.0006130867404863238, + 0.0014941217377781868, + 0.00046094926074147224, + 0.00033666467061266303, + 0.0011592293158173561, + 0.000986011466011405, + 0.0014443779364228249 + ], + "r2": [ + 0.5775511264801025, + 0.9305323958396912, + 0.9056863784790039, + 0.8087513446807861, + 0.9619851112365723, + 0.8998416662216187, + 0.8923063278198242, + 0.9462068676948547, + 0.9595792889595032, + 0.38815993070602417, + 0.7620314955711365, + 0.9630575180053711, + 0.7535457611083984, + 0.4320824146270752, + 0.9499990940093994, + 0.7842147946357727, + 0.9279036521911621, + 0.5132381916046143, + 0.3313904404640198, + 0.263294517993927, + 0.5800174474716187, + 0.9067438244819641, + 0.9540691375732422, + 0.8297027349472046, + 0.9223629236221313, + 0.20123112201690674 + ], + "pearson": [ + 0.7870759963989258, + 0.9787894487380981, + 0.9782218933105469, + 0.9218029975891113, + 0.9844422340393066, + 0.9751392006874084, + 0.988003671169281, + 0.9864732623100281, + 0.9834932684898376, + 0.7187127470970154, + 0.8731866478919983, + 0.9890651702880859, + 0.938484787940979, + 0.781520664691925, + 0.9848747849464417, + 0.9313053488731384, + 0.9756849408149719, + 0.8324065804481506, + 0.8292630314826965, + 0.7894912958145142, + 0.9063573479652405, + 0.9756512641906738, + 0.9891635775566101, + 0.9665358066558838, + 0.9667638540267944, + 0.8873770236968994 + ], + "spearman": [ + 0.551747035385344, + 0.7370737997453489, + 0.803836848773959, + 0.8292184235600115, + 0.7613014349223204, + 0.7675304115846785, + 0.7789766806394016, + 0.8832378448143863, + 0.8493184084598053, + 0.7242364798579012, + 0.6666711304178595, + 0.8587951388645674, + 0.8209920731601313, + 0.6643807105643174, + 0.94185369178596, + 0.8130837590466344, + 0.7475316814597881, + 0.7205471445258141, + 0.7121962959539753, + 0.7126376637972636, + 0.47444913454653115, + 0.8554636495518614, + 0.79350591958881, + 0.9104760445018664, + 0.9143626239345516, + 0.7089461210609797 + ], + "de_corr": [ + 0.9269323945045471, + 0.9903609752655029, + 0.9926550984382629, + 0.969302773475647, + 0.9927334785461426, + 0.9951089024543762, + 0.9945881366729736, + 0.9869873523712158, + 0.9948418736457825, + 0.8022546768188477, + 0.9485812187194824, + 0.9966453909873962, + 0.9563841819763184, + 0.9618679881095886, + 0.9917111396789551, + 0.9792109131813049, + 0.9953674077987671, + 0.7746251821517944, + 0.8504496216773987, + 0.73276686668396, + 0.9748659133911133, + 0.9905813932418823, + 0.9938662052154541, + 0.9811793565750122, + 0.9787013530731201, + 0.9774865508079529 + ], + "mmd": [ + 0.013021821112970566, + 0.02689526231028827, + 0.02820178421391406, + 0.06906585385531339, + 0.020487769660918698, + 0.023150655193067582, + 0.050137506492634265, + 0.16251804350257926, + 0.0258187441534784, + 0.049896416209351147, + 0.045910940850522386, + 0.035428211191668835, + 0.04205538802670716, + 0.03411345341440386, + 0.03430092272019736, + 0.06699429318588046, + 0.022095179005238497, + 0.014287880451578472, + 0.04600966690867225, + 0.02442238698581134, + 0.04895633861652493, + 0.025632502027059134, + 0.026054079367394012, + 0.0659416347612718, + 0.08807082656125143, + 0.07795055573970688 + ], + "wasserstein": [ + 0.11233875102879669, + 0.1382577469116224, + 0.16257367499728104, + 0.22345839197149164, + 0.13100025399928808, + 0.1473105598750808, + 0.21374490149100697, + 0.378033049583956, + 0.12087458622020454, + 0.21157240183543238, + 0.19642191220685326, + 0.1745468951781186, + 0.1953054238124035, + 0.15487670067352413, + 0.15586411645888687, + 0.2547957024119093, + 0.14254234915454567, + 0.10974410507796854, + 0.19968812475020362, + 0.1647749943071228, + 0.21860856110177715, + 0.15151733043827764, + 0.14455454923654443, + 0.23581756988551764, + 0.26925133427962333, + 0.2047727409454289 + ] + }, + "n_perts": 26 + }, + "NearestPerturbationCentroid": { + "mse": 0.002478486358505996, + "r2": 0.23386846138880804, + "pearson": 0.7247861949010537, + "spearman": 0.5799124729318662, + "de_corr": 0.7635814500614427, + "mmd": 0.08080473789892954, + "wasserstein": 0.2582930905262011, + "_per_pert": { + "mse": [ + 0.000533635844476521, + 0.0013795195845887065, + 0.0013664873549714684, + 0.0007153262849897146, + 0.0016368427313864231, + 0.0010332483798265457, + 0.004374958109110594, + 0.0006444199243560433, + 0.001957371598109603, + 0.006869337521493435, + 0.0009074644767679274, + 0.0017874609911814332, + 0.0010570818558335304, + 0.0036049378104507923, + 0.003051120787858963, + 0.0032517763320356607, + 0.0004848494427278638, + 0.0002504873846191913, + 0.002362786792218685, + 0.0025162675883620977, + 0.001352915191091597, + 0.00018224639643449336, + 0.003386517520993948, + 0.00904778577387333, + 0.005635064095258713, + 0.0050507355481386185 + ], + "r2": [ + 0.1566997766494751, + 0.7822160124778748, + 0.7727686762809753, + 0.8030511736869812, + 0.6980127096176147, + 0.7847366333007812, + 0.5729600787162781, + 0.9738505482673645, + 0.5192646980285645, + -1.8723547458648682, + 0.7886219024658203, + 0.8428369760513306, + 0.7795968055725098, + -1.3574402332305908, + 0.35502398014068604, + 0.5822820067405701, + 0.8994961380958557, + 0.6252045631408691, + -0.15741503238677979, + -2.0236306190490723, + 0.6197092533111572, + 0.9631291031837463, + 0.5379804372787476, + -0.3291703462600708, + 0.5563032627105713, + -1.7931537628173828 + ], + "pearson": [ + 0.40049001574516296, + 0.929872989654541, + 0.9087874293327332, + 0.9011523723602295, + 0.9123130440711975, + 0.9401535987854004, + 0.9237536787986755, + 0.9924522042274475, + 0.9656147956848145, + -0.08230501413345337, + 0.9518076181411743, + 0.9306371808052063, + 0.8863422870635986, + 0.09355726093053818, + 0.7735655307769775, + 0.778367280960083, + 0.948794424533844, + 0.8171080946922302, + 0.05523582175374031, + 0.627107560634613, + 0.8792233467102051, + 0.9813959002494812, + 0.9108834266662598, + -0.29144471883773804, + 0.8878174424171448, + 0.8217574954032898 + ], + "spearman": [ + 0.3810437789050358, + 0.6065274617974341, + 0.5486168655162754, + 0.7831034787407409, + 0.5216215074472673, + 0.6799291849212872, + 0.6649632422127042, + 0.9339662157003352, + 0.8787702436922701, + 0.014809978657983117, + 0.7728194244675329, + 0.7233591536263967, + 0.7574225069434324, + 0.3961334579681166, + 0.6307828225901657, + 0.7217318566328387, + 0.6870010005857667, + 0.5758524374306192, + 0.08045635799333074, + 0.5319892656791716, + 0.5764579665922199, + 0.8967433429046583, + 0.620134526905448, + -0.3594185817850521, + 0.836594005006243, + 0.6163127950962983 + ], + "de_corr": [ + 0.5721431970596313, + 0.9719492197036743, + 0.939814567565918, + 0.9490275382995605, + 0.9564734101295471, + 0.9784294366836548, + 0.957977294921875, + 0.9853857755661011, + 0.9904815554618835, + -0.3254048228263855, + 0.9855234622955322, + 0.9800997972488403, + 0.9723494648933411, + 0.010173377580940723, + 0.856938362121582, + 0.9231246709823608, + 0.9796337485313416, + 0.855949342250824, + 0.5607375502586365, + 0.6474562287330627, + 0.9740811586380005, + 0.9901545643806458, + 0.9644842743873596, + -0.7426186800003052, + 0.9561570286750793, + 0.9625961780548096 + ], + "mmd": [ + 0.02204469496328021, + 0.04624911196675574, + 0.04746513316097101, + 0.05850050663460027, + 0.05384147191517019, + 0.0361109265783357, + 0.11663521270909372, + 0.15518572284447507, + 0.09348214564328561, + 0.1778393454296321, + 0.029855567470220667, + 0.06980892692864027, + 0.04091339127139526, + 0.11479473976791221, + 0.07370843238077363, + 0.11546226816321503, + 0.022746616132742092, + 0.011514855898313026, + 0.07266941986915876, + 0.0854665723722755, + 0.045003382837002004, + 0.018198857259373757, + 0.10215822264566865, + 0.1671924776037782, + 0.13503237873100937, + 0.18904280419509 + ], + "wasserstein": [ + 0.1402848677767436, + 0.22384115974220262, + 0.22574395983493822, + 0.22839451319865708, + 0.23123111355692186, + 0.18999337168742156, + 0.3651302607002648, + 0.33065099107446744, + 0.21426254452904867, + 0.44905922384330976, + 0.20683663195158197, + 0.2605474445830111, + 0.20645871223374182, + 0.28734785374330934, + 0.3018896834928422, + 0.2903402758335888, + 0.15180851176513208, + 0.09403153041927144, + 0.2821338821418673, + 0.2188255783476873, + 0.20926775740598483, + 0.11903806205432738, + 0.31814383882848457, + 0.4467690433555681, + 0.3799488600241454, + 0.3436406815567083 + ] + }, + "n_perts": 26 + }, + "EndpointMLP": { + "mse": 0.0008020812652270811, + "r2": 0.7639783162337083, + "pearson": 0.943373576952861, + "spearman": 0.7928224143378853, + "de_corr": 0.9737307039590982, + "mmd": 0.044485534927875284, + "wasserstein": 0.17595379308630796, + "_per_pert": { + "mse": [ + 0.0001764298212947324, + 0.000341860024491325, + 0.0006514676497317851, + 0.0016977416817098856, + 0.00020893856708426028, + 0.0003437062550801784, + 0.0006340914987958968, + 0.0032982639968395233, + 0.001114899292588234, + 0.0005150196957401931, + 0.0002106309839291498, + 0.001079932670108974, + 0.0007576587377116084, + 0.0005042431876063347, + 0.00016221677651628852, + 0.00108602293767035, + 0.00026029604487121105, + 0.0003001794684678316, + 0.0018586951773613691, + 0.0004277301486581564, + 0.0008145751780830324, + 0.00017303899221587926, + 0.000495580374263227, + 0.000787517346907407, + 0.0008026810828596354, + 0.0021506953053176403 + ], + "r2": [ + 0.7211893200874329, + 0.9460307359695435, + 0.8916683197021484, + 0.5325654745101929, + 0.9614521265029907, + 0.9283934235572815, + 0.9381062984466553, + 0.866162121295929, + 0.7261780500411987, + 0.7846488952636719, + 0.9509371519088745, + 0.9050465822219849, + 0.8420270085334778, + 0.6702514290809631, + 0.9657090306282043, + 0.8604912161827087, + 0.9460435509681702, + 0.5508520007133484, + 0.08951520919799805, + 0.4860253930091858, + 0.7710312008857727, + 0.9649919271469116, + 0.932388424873352, + 0.8843092918395996, + 0.9367980360984802, + -0.18937599658966064 + ], + "pearson": [ + 0.8538834452629089, + 0.9834492802619934, + 0.9840121269226074, + 0.9230494499206543, + 0.9823115468025208, + 0.977793276309967, + 0.9880353212356567, + 0.9933536052703857, + 0.9654573202133179, + 0.8889068961143494, + 0.9755520224571228, + 0.9825723171234131, + 0.9482463002204895, + 0.8223302364349365, + 0.9833102822303772, + 0.929058849811554, + 0.9731715321540833, + 0.9565269351005554, + 0.875830888748169, + 0.855214536190033, + 0.8972867727279663, + 0.9853354692459106, + 0.9754910469055176, + 0.952223539352417, + 0.967992901802063, + 0.9073171019554138 + ], + "spearman": [ + 0.527013020367931, + 0.7515835285741037, + 0.8457713827450878, + 0.8362780284228648, + 0.7574566203962355, + 0.8095630693734549, + 0.8643096912456038, + 0.8981804707573718, + 0.8548378028787879, + 0.7521596037261155, + 0.8067627081345097, + 0.8789945026946745, + 0.8338661967880412, + 0.7013842708277744, + 0.9425145511111144, + 0.8359182966491646, + 0.6965080796194033, + 0.761916213550671, + 0.757682484740644, + 0.7902816421564133, + 0.5547945336832226, + 0.8580117556276767, + 0.806298656459696, + 0.8274195420853317, + 0.9179559408813905, + 0.7459201792877279 + ], + "de_corr": [ + 0.9722740650177002, + 0.9957163333892822, + 0.9919744729995728, + 0.9635945558547974, + 0.9935960173606873, + 0.9915443658828735, + 0.99666827917099, + 0.9915331602096558, + 0.9873785376548767, + 0.9613246917724609, + 0.9909056425094604, + 0.9955531358718872, + 0.9637845158576965, + 0.9094120860099792, + 0.9885944128036499, + 0.9769296646118164, + 0.9854501485824585, + 0.9806379675865173, + 0.9181355834007263, + 0.8516497611999512, + 0.969434380531311, + 0.9968209266662598, + 0.9937450885772705, + 0.979896605014801, + 0.9899572730064392, + 0.9804866313934326 + ], + "mmd": [ + 0.00939074502599535, + 0.023945967306793103, + 0.032513114908470864, + 0.10172367153089756, + 0.022291654025017982, + 0.019033009974969928, + 0.04078682089177177, + 0.17810661065121425, + 0.06605911452199564, + 0.023305841300289343, + 0.0179010890322433, + 0.04698349858770068, + 0.0320253415100229, + 0.02243332577424806, + 0.036360509155239695, + 0.04310328437344857, + 0.014676688852410136, + 0.014393517162749836, + 0.058695578526053116, + 0.017454215915711502, + 0.03021586689117839, + 0.018672551217580713, + 0.028699622044615958, + 0.06686701296800857, + 0.08779529986539847, + 0.10318995611073167 + ], + "wasserstein": [ + 0.09634433725809048, + 0.14457362822844394, + 0.17813129015134715, + 0.26963856392159335, + 0.1434556965459493, + 0.12590953731785043, + 0.1738429979290879, + 0.38652253694458105, + 0.19249031864272617, + 0.13392930952241766, + 0.1319038161119049, + 0.19968582500395804, + 0.16901401790296622, + 0.13802216887287094, + 0.16001044192442296, + 0.1879722848449178, + 0.1285070085429668, + 0.1189616944563749, + 0.22085639480056357, + 0.1295554748901658, + 0.16637893428918307, + 0.1253171778105432, + 0.1512992796521846, + 0.22283154374680045, + 0.2648075181248954, + 0.21483682280720123 + ] + }, + "n_perts": 26 + }, + "Random": { + "mse": 0.006530506056151353, + "r2": -0.6242899436217088, + "pearson": 0.33471211712234294, + "spearman": 0.30173074557310325, + "de_corr": 0.2762379719553372, + "mmd": 0.16339670384709448, + "wasserstein": 0.3892757439227088, + "_per_pert": { + "mse": [ + 0.0036576436832547188, + 0.0056915609166026115, + 0.0060289716348052025, + 0.004404988139867783, + 0.007452930323779583, + 0.004236144013702869, + 0.001192743075080216, + 0.024693042039871216, + 0.0013728041667491198, + 0.009539428167045116, + 0.002810685196891427, + 0.00863055419176817, + 0.005879695527255535, + 0.0028781192377209663, + 0.0029862020164728165, + 0.013793515972793102, + 0.015004010871052742, + 0.0038161741103976965, + 0.0020486859139055014, + 0.000368712906492874, + 0.007352271117269993, + 0.0023348368704319, + 0.017345810309052467, + 0.004993144888430834, + 0.007239057682454586, + 0.004041424486786127 + ], + "r2": [ + -4.780143737792969, + 0.10147649049758911, + -0.002549409866333008, + -0.2128133773803711, + -0.3750194311141968, + 0.11745631694793701, + 0.8835762739181519, + -0.0020008087158203125, + 0.6628358960151672, + -2.988830089569092, + 0.34529948234558105, + 0.24115610122680664, + -0.22592544555664062, + -0.8821392059326172, + 0.36874711513519287, + -0.7718935012817383, + -2.1101648807525635, + -4.710006237030029, + -0.0035523176193237305, + 0.5569422245025635, + -1.0666491985321045, + 0.5276314616203308, + -1.3664729595184326, + 0.2664791941642761, + 0.43000710010528564, + -1.234985589981079 + ], + "pearson": [ + -0.32098689675331116, + 0.6315445899963379, + 0.06893187761306763, + 0.17515534162521362, + 0.4494899809360504, + 0.376334011554718, + 0.9715424180030823, + 0.212151899933815, + 0.8740909099578857, + -0.14101824164390564, + 0.6391974091529846, + 0.696272611618042, + -0.16628801822662354, + 0.019388029351830482, + 0.6841511726379395, + 0.18468309938907623, + -0.35539254546165466, + 0.2860647141933441, + 0.5686036348342896, + 0.8090764880180359, + 0.717863917350769, + 0.7279661893844604, + -0.5468238592147827, + 0.520050585269928, + 0.7499146461486816, + -0.12944892048835754 + ], + "spearman": [ + -0.09682912051447327, + 0.22879759978816758, + 0.26566111845095053, + 0.19210605561994568, + 0.3072106109499961, + 0.18366238676489946, + 0.7787123272327153, + 0.38650271534346553, + 0.5770389367808767, + 0.109951854134633, + 0.41707697971382285, + 0.43196796653145103, + 0.12069515886786666, + 0.048372930159525734, + 0.5196168931176147, + 0.22551694443483453, + -0.05681824908382497, + 0.4219306056185222, + 0.485013799143099, + 0.680594747336716, + 0.44971425543613464, + 0.48711372039248285, + -0.23501064413033665, + 0.5138413323971912, + 0.6161601103392491, + -0.2136016499248397 + ], + "de_corr": [ + -0.7161344885826111, + 0.8492564558982849, + -0.1793336272239685, + 0.13276320695877075, + 0.776057243347168, + 0.5899722576141357, + 0.9700168371200562, + -0.3755188286304474, + 0.9809064269065857, + -0.4573112726211548, + 0.60968416929245, + 0.840336263179779, + -0.4690474271774292, + -0.006662833970040083, + 0.6662706732749939, + 0.1331259310245514, + -0.7366300225257874, + 0.07454127073287964, + 0.7414866089820862, + 0.8588231205940247, + 0.8517552614212036, + 0.7582389116287231, + -0.727645754814148, + 0.4333813190460205, + 0.8468313813209534, + -0.262975811958313 + ], + "mmd": [ + 0.1259100149447856, + 0.16946875132238604, + 0.15328670939502576, + 0.11925395481013612, + 0.21227452919845446, + 0.11639224942562032, + 0.05440252783775834, + 0.333752679237582, + 0.03177710284945923, + 0.24049995578142902, + 0.09254898518275667, + 0.2386030471134899, + 0.15347768850526877, + 0.09591408351052366, + 0.1002594879705514, + 0.3399638335367665, + 0.39018998027496965, + 0.1259245276964478, + 0.06979720367977793, + 0.015508065578698216, + 0.2094256725394087, + 0.08344996485088807, + 0.3747955517551732, + 0.12558276191202788, + 0.1565787000513006, + 0.11927627106377081 + ], + "wasserstein": [ + 0.32759463319893284, + 0.4425174979876795, + 0.4252578782643919, + 0.38039587629106897, + 0.4211703004133938, + 0.3463849288268218, + 0.20747240099971048, + 0.7083644932209476, + 0.18970051912524752, + 0.5432435331111674, + 0.32828753055254845, + 0.4381020627120035, + 0.3929922959591078, + 0.2821122568067752, + 0.34254322073800736, + 0.5808119012996298, + 0.6576707281875873, + 0.35540729146222566, + 0.25355683532358997, + 0.11177937542410157, + 0.48529021725106586, + 0.2419028528459981, + 0.5954052939235607, + 0.3152188587606063, + 0.4070771722904518, + 0.3409093870138086 + ] + }, + "n_perts": 26 + } + } +} \ No newline at end of file diff --git a/experiments/results/norman_forward_perturbation.json b/experiments/results/norman_forward_perturbation.json new file mode 100644 index 0000000000000000000000000000000000000000..508bc9da8ebd586346e4b21e0764c375b5d242ff --- /dev/null +++ b/experiments/results/norman_forward_perturbation.json @@ -0,0 +1,2845 @@ +{ + "split": "perturbation", + "models": { + "PIVOT": { + "mse": 0.0017456959687208043, + "r2": -0.8285323470196826, + "pearson": 0.76253557141791, + "spearman": 0.5823030614288411, + "de_corr": 0.8314210008433525, + "mmd": 0.3021512914899839, + "wasserstein": 0.43160508252391033, + "_per_pert": { + "mse": [ + 0.0015962913166731596, + 0.0008809863356873393, + 0.0019998853094875813, + 0.0003412741352804005, + 0.0008116157841868699, + 0.0006022303132340312, + 0.0007515754550695419, + 0.0007353638648055494, + 0.0009762277477420866, + 0.004156765062361956, + 0.0056117321364581585, + 0.0009224773966707289, + 0.0014235102571547031, + 0.000321702565997839, + 0.001231062808074057, + 0.0017586882458999753, + 0.002621252555400133, + 0.00047866388922557235, + 0.0004518006753642112, + 0.0009143200586549938, + 0.00048078721738420427, + 0.0035726698115468025, + 0.0006342866108752787, + 0.0007832234841771424, + 0.0029461875092238188, + 0.0012722138781100512, + 0.00037073460407555103, + 0.005331906024366617, + 0.007546290289610624, + 0.001551676308736205, + 0.0010973499156534672, + 0.0006320606917142868, + 0.00013778671564068645, + 0.0014407264534384012, + 0.0016012798296287656, + 0.001502867671661079, + 0.0015092769172042608, + 0.004813094157725573, + 0.0007678758120164275, + 0.002616616664454341, + 0.0011721006594598293, + 0.0018023933516815305, + 0.0007824173080734909, + 0.0014333735452964902, + 0.0012945494381710887, + 0.005306630861014128, + 0.0010599088855087757 + ], + "r2": [ + 0.5472016930580139, + -1.842552900314331, + 0.6095108389854431, + -4.876230716705322, + -20.437246322631836, + -2.7194979190826416, + -0.18770825862884521, + 0.8777173161506653, + 0.7312178611755371, + -1.9020304679870605, + -2.7078135013580322, + 0.8078142404556274, + 0.8908155560493469, + 0.8376991748809814, + 0.42308592796325684, + 0.1281682848930359, + 0.6018931865692139, + 0.9521409869194031, + 0.6058731079101562, + 0.9128751754760742, + 0.867173433303833, + -7.576755523681641, + 0.8659182786941528, + -0.182220458984375, + 0.27227669954299927, + 0.5839238166809082, + 0.9288119077682495, + 0.31507188081741333, + 0.601916491985321, + 0.6740221977233887, + -1.2246947288513184, + 0.9510142803192139, + 0.7938345074653625, + 0.7671914100646973, + -4.445379257202148, + 0.31845784187316895, + 0.8332926034927368, + 0.31394362449645996, + 0.4389669895172119, + 0.6129298210144043, + 0.8421118855476379, + -7.5136003494262695, + 0.4907867908477783, + -3.4536638259887695, + 0.8398243188858032, + 0.6412551403045654, + -1.7503633499145508 + ], + "pearson": [ + 0.7536920309066772, + 0.5233908891677856, + 0.8592501878738403, + 0.22083774209022522, + 0.2833574116230011, + 0.039258554577827454, + 0.7498257756233215, + 0.9401399493217468, + 0.8797203898429871, + 0.6033342480659485, + 0.9358171224594116, + 0.9723519682884216, + 0.9687368869781494, + 0.9374346137046814, + 0.6731510758399963, + 0.47313177585601807, + 0.9422309994697571, + 0.9758107662200928, + 0.803334653377533, + 0.965327799320221, + 0.9316220283508301, + 0.8100623488426208, + 0.9581425189971924, + 0.6218380331993103, + 0.8873342275619507, + 0.8127406239509583, + 0.9650146961212158, + 0.5918835997581482, + 0.846104085445404, + 0.8276202082633972, + 0.6287824511528015, + 0.9759727716445923, + 0.8941470384597778, + 0.983452558517456, + 0.18894581496715546, + 0.7274587154388428, + 0.9846226572990417, + 0.6012974977493286, + 0.7263070344924927, + 0.871539294719696, + 0.9778054356575012, + 0.8281512260437012, + 0.7819417715072632, + 0.39418646693229675, + 0.9719808101654053, + 0.9259516596794128, + 0.624131441116333 + ], + "spearman": [ + 0.6555050882255908, + 0.40542519904581553, + 0.4964159787310143, + 0.09228475010163843, + -0.15748521125132137, + -0.11657048285974761, + 0.21862650960808533, + 0.605807510152057, + 0.769920244183745, + 0.5469354385725889, + 0.5626566869480071, + 0.6832155535335042, + 0.9070905775510695, + 0.8487869280419458, + 0.45339023110868865, + 0.2868166739366436, + 0.7518973609126082, + 0.8643221348230034, + 0.6194693370885784, + 0.7732447700697314, + 0.8543534221221268, + 0.6644175705672826, + 0.9216471532145842, + 0.5517182833777677, + 0.7707817801400423, + 0.6253514731561677, + 0.85791655113273, + 0.5972728738711756, + 0.6745397902412349, + 0.680843485444789, + 0.45267624472772633, + 0.8136672703384545, + 0.6424177851315931, + 0.8843047114275199, + 0.11669099443566826, + 0.5227399671880087, + 0.7238119692458834, + 0.5503434385154962, + 0.46469522726553875, + 0.717613957110383, + 0.8244220387943655, + 0.39631782092974593, + 0.6393433529567272, + 0.24420713551435697, + 0.704496413148991, + 0.8780034344331563, + 0.3258944642007685 + ], + "de_corr": [ + 0.9333991408348083, + 0.7491828203201294, + 0.9241585731506348, + 0.24465131759643555, + 0.4710516333580017, + 0.03841133415699005, + 0.9465099573135376, + 0.9783341288566589, + 0.9323738813400269, + 0.7223357558250427, + 0.9700398445129395, + 0.9917811155319214, + 0.9845224022865295, + 0.95426344871521, + 0.8516639471054077, + 0.539853572845459, + 0.9640456438064575, + 0.9839518070220947, + 0.92417311668396, + 0.979576051235199, + 0.9622992277145386, + 0.9482614398002625, + 0.9775505661964417, + 0.8960217237472534, + 0.9404195547103882, + 0.9081787467002869, + 0.988193929195404, + 0.698651909828186, + 0.7528803944587708, + 0.7510294318199158, + 0.5872046947479248, + 0.9912537336349487, + 0.9690126776695251, + 0.997524082660675, + 0.25434747338294983, + 0.8284298181533813, + 0.9940236806869507, + 0.6034884452819824, + 0.9004725813865662, + 0.8926483392715454, + 0.9886642098426819, + 0.959450364112854, + 0.8362482190132141, + 0.6785160303115845, + 0.9831262230873108, + 0.9680206775665283, + 0.7365893721580505 + ], + "mmd": [ + 0.31358329356625925, + 0.2829083437860266, + 0.3268910257886851, + 0.3034275433088548, + 0.26561077532549326, + 0.27375241880245993, + 0.2764328706803504, + 0.30042128408200697, + 0.3065089420728595, + 0.3461828757735623, + 0.36859794732202045, + 0.2710819231170011, + 0.2931891493477259, + 0.2320166530035307, + 0.2678655791247254, + 0.3100671682066971, + 0.2947334531122461, + 0.27877327345627356, + 0.26099842907739523, + 0.2544545857509505, + 0.3034685334443069, + 0.34142556532541024, + 0.2865900872098376, + 0.2621079957448894, + 0.33341331852705725, + 0.2986212856731577, + 0.3046254001186489, + 0.4049631989245549, + 0.29687186899197815, + 0.2892757718681802, + 0.2832749104171687, + 0.2963613126384349, + 0.262936228194131, + 0.28750202458966534, + 0.31853469099007437, + 0.3254136872440071, + 0.3229344827643106, + 0.394373763081333, + 0.27312924977788144, + 0.3664557058728112, + 0.2771543305879254, + 0.32460494136859086, + 0.2727312048226932, + 0.27398726155845665, + 0.3571783400626488, + 0.3426648703072518, + 0.27301313521871284 + ], + "wasserstein": [ + 0.4821041697967327, + 0.37169123004200516, + 0.46441134658798594, + 0.37558352939015793, + 0.35424735165224336, + 0.34667920523386314, + 0.36154464792525415, + 0.42276941312530963, + 0.46618256614801645, + 0.46860028466716114, + 0.5220581034096756, + 0.4019005545926686, + 0.5063314914221153, + 0.3512468343961775, + 0.37955504571508614, + 0.40661420355491534, + 0.4442106491816762, + 0.44726791691187456, + 0.3572826731205464, + 0.4066724517513526, + 0.42381805290126495, + 0.43329532088579326, + 0.43324303879231296, + 0.36331708800584595, + 0.5153061605365753, + 0.4254621708617453, + 0.4270177856138994, + 0.5573708062094781, + 0.5803057499360839, + 0.4368545237685248, + 0.38260245122802966, + 0.4435057205256277, + 0.3411261893862858, + 0.39030869652872263, + 0.4198432339215978, + 0.5112260641620169, + 0.4723184218096585, + 0.5083531046129299, + 0.3921637349952938, + 0.5017335441630584, + 0.4175469696081974, + 0.4156747659203838, + 0.3698995877111973, + 0.3685795259443726, + 0.4717643181925033, + 0.573698411676562, + 0.3721497721010105 + ] + }, + "n_perts": 47 + }, + "MeanControl": { + "mse": 0.004514833508050713, + "r2": -0.10789477064254437, + "pearson": -0.022738046513157004, + "spearman": -0.04880332349662557, + "de_corr": -0.029544741906067158, + "mmd": 0.10804938097705778, + "wasserstein": 0.3255573552861375, + "_per_pert": { + "mse": [ + 0.003526095300912857, + 0.00047110902960412204, + 0.005013410467654467, + 0.00011254555283812806, + 8.388989954255521e-05, + 0.0002046082663582638, + 0.0007276904652826488, + 0.006099091377109289, + 0.0038916850462555885, + 0.0013538715429604053, + 0.0013269009068608284, + 0.00479864701628685, + 0.013018874451518059, + 0.0020068504381924868, + 0.0022634572815150023, + 0.002172177890315652, + 0.006917996797710657, + 0.01052918191999197, + 0.0010125469416379929, + 0.010958677157759666, + 0.003653470892459154, + 0.0005656962166540325, + 0.0047027855180203915, + 0.0007785499328747392, + 0.0039011535700410604, + 0.003163485322147608, + 0.005225673783570528, + 0.0077210054732859135, + 0.020093467086553574, + 0.00497997272759676, + 0.00054371845908463, + 0.013299396261572838, + 0.0007702374714426696, + 0.006292510777711868, + 0.00042728715925477445, + 0.002570908050984144, + 0.008804382756352425, + 0.007052035070955753, + 0.0014501134864985943, + 0.0073930020444095135, + 0.007186289876699448, + 0.0002132457448169589, + 0.0015514662954956293, + 0.00042830329039134085, + 0.00801896583288908, + 0.014559644274413586, + 0.0003610997518990189 + ], + "r2": [ + -0.0001995563507080078, + -0.5200603008270264, + 0.021102607250213623, + -0.9378657341003418, + -1.2157888412475586, + -0.26370251178741455, + -0.1499631404876709, + -0.014209747314453125, + -0.07148706912994385, + 0.054799675941467285, + 0.12328284978866577, + 0.0002665519714355469, + 0.0014414191246032715, + -0.012468218803405762, + -0.060726165771484375, + -0.07681047916412354, + -0.0506819486618042, + -0.05275595188140869, + 0.11670804023742676, + -0.044243812561035156, + -0.009341120719909668, + -0.35804271697998047, + 0.005878746509552002, + -0.17516624927520752, + 0.03639495372772217, + -0.03461432456970215, + -0.0034291744232177734, + 0.00817185640335083, + -0.05997490882873535, + -0.04619753360748291, + -0.10229921340942383, + -0.030724048614501953, + -0.15247929096221924, + -0.01681363582611084, + -0.45305097103118896, + -0.16589272022247314, + 0.027510643005371094, + -0.005194187164306641, + -0.059496164321899414, + -0.09362983703613281, + 0.03196924924850464, + -0.0072650909423828125, + -0.00972592830657959, + -0.3307896852493286, + 0.007806599140167236, + 0.01572251319885254, + 0.06298035383224487 + ], + "pearson": [ + 0.0591709278523922, + -0.40775421261787415, + 0.15382766723632812, + 0.0747164785861969, + -0.1517912894487381, + 0.05104249715805054, + -0.1530401110649109, + -0.05691283568739891, + -0.2880525290966034, + 0.2644456923007965, + 0.3980618417263031, + 0.06142786517739296, + 0.2099093198776245, + 0.09384554624557495, + -0.09174783527851105, + -0.22845198214054108, + -0.2895209491252899, + -0.370250940322876, + 0.39623144268989563, + -0.19269369542598724, + 0.05708274617791176, + -0.14504733681678772, + 0.15338094532489777, + -0.22136132419109344, + 0.2530163526535034, + -0.08538380265235901, + 0.051351871341466904, + 0.11927977204322815, + -0.27650654315948486, + -0.08860388398170471, + -0.06261344999074936, + -0.1181451827287674, + -0.15346801280975342, + -0.0022375716362148523, + -0.3507479429244995, + -0.5506516695022583, + 0.2766454517841339, + 0.07293259352445602, + -0.049403443932533264, + -0.5229116082191467, + 0.24387092888355255, + 0.3320770561695099, + 0.07020393013954163, + -0.13814716041088104, + 0.09130217880010605, + 0.1804584115743637, + 0.2624756097793579 + ], + "spearman": [ + 0.19216284713491025, + -0.19284188589236262, + -0.11938483946898354, + 0.08759452212678809, + -0.14483278589106036, + -0.23891023950917015, + -0.09278604301227522, + -0.07776816930023873, + -0.34219772716910113, + 0.08997476599209138, + 0.1571429676875295, + -0.009334050216522768, + 0.30276350005000907, + 0.11369565713516436, + 0.026765146902985506, + -0.11985193137592841, + -0.0676444480055454, + -0.26492315410202055, + -0.022761245256123594, + -0.17927895359507073, + 0.12467714385242742, + -0.06812438063506722, + 0.14233023016137084, + -0.13192931206284725, + 0.2202828745445375, + -0.04506938627789246, + 0.1044222268044151, + 0.05574005331374939, + -0.13509633039854418, + -0.08472786457997593, + 0.06745851827828901, + -0.02750546917303849, + -0.14517513649361455, + -0.11361011361230283, + -0.3363138725037647, + -0.30150052130845006, + 0.03761056068859066, + -0.11917355169692095, + 0.040746374501949574, + -0.33333291383322283, + 0.0914415356877504, + 0.06168918202823291, + -0.1441669084879804, + -0.21287281650164175, + -0.11814292463066067, + -0.039590558196958545, + 0.01859322195509365 + ], + "de_corr": [ + -0.300130695104599, + -0.5648529529571533, + 0.3370738923549652, + 0.3860224485397339, + -0.1955641210079193, + 0.02822083979845047, + -0.48389118909835815, + -0.009495597332715988, + -0.05394879728555679, + 0.36357206106185913, + 0.667620062828064, + 0.43334126472473145, + -0.4320434629917145, + 0.14437812566757202, + -0.32033661007881165, + -0.3400384485721588, + -0.2038392722606659, + -0.5935828685760498, + 0.541037917137146, + -0.2471340298652649, + -0.10499667376279831, + -0.17938752472400665, + 0.077311210334301, + -0.3787427544593811, + 0.4311549961566925, + -0.1292337328195572, + -0.12586845457553864, + 0.415390282869339, + -0.5604172945022583, + -0.011644965037703514, + -0.18901412189006805, + -0.47924914956092834, + -0.5018155574798584, + 0.33002686500549316, + -0.3823208510875702, + -0.8062375783920288, + 0.5056836009025574, + 0.12952904403209686, + -0.22419269382953644, + -0.4344070255756378, + 0.3809707760810852, + 0.8074125647544861, + 0.16500452160835266, + -0.31204333901405334, + 0.142561137676239, + 0.25846385955810547, + 0.6310514211654663 + ], + "mmd": [ + 0.09349272787495821, + 0.017079164586266127, + 0.14250329469241785, + 0.00759453203977023, + 0.006482170008059729, + 0.011063732850183694, + 0.026731542733261282, + 0.1470853808210062, + 0.09653127548836427, + 0.04204111040549552, + 0.04216490529019157, + 0.1312443358607276, + 0.21005125605035002, + 0.05309926180032265, + 0.06683692811031039, + 0.0707329364249959, + 0.19890545534435256, + 0.26540914314875486, + 0.03516672967215262, + 0.24847255678051916, + 0.0821541846284356, + 0.025634852829435117, + 0.1032416212736893, + 0.02939766576232139, + 0.10044225887905478, + 0.09454968542461961, + 0.144971177320813, + 0.2158981101013302, + 0.27529296477662657, + 0.13301308819063173, + 0.022128561707196348, + 0.2808152270527752, + 0.033273759910667056, + 0.18338728148350747, + 0.016317289022899972, + 0.06928072801802743, + 0.2114590654919788, + 0.18688661728037947, + 0.04694983392849206, + 0.15847260958398424, + 0.18867237244125679, + 0.009225637311758073, + 0.051785343083803004, + 0.017391647303436564, + 0.20626368571513176, + 0.26286667188565194, + 0.01586052553135231 + ], + "wasserstein": [ + 0.3465150925077128, + 0.14262462636439255, + 0.3924961748765501, + 0.0884045304954344, + 0.07540012800379445, + 0.11246360100660709, + 0.14956275073881106, + 0.4246831496977201, + 0.36262283630451864, + 0.20913232861350933, + 0.2027074007927328, + 0.37186873605567927, + 0.5165355363724826, + 0.21696055667376182, + 0.23970034947649663, + 0.2468377396448319, + 0.47336123278609643, + 0.6069115027749394, + 0.1988019244084769, + 0.5360744081970711, + 0.32131733750831143, + 0.15920445814820125, + 0.37406449459488234, + 0.1906735867984297, + 0.3921992392111027, + 0.3386888100758347, + 0.4107306759792381, + 0.5186042173434964, + 0.648326146552633, + 0.4215655626262298, + 0.1460777869166568, + 0.6562208847516053, + 0.1565663488804904, + 0.4559409539989952, + 0.14973875231061093, + 0.29716214661285395, + 0.5065901284321037, + 0.46066045826130547, + 0.20351251804672624, + 0.44478038092501976, + 0.4880487648741447, + 0.08884758015177945, + 0.23486717757267328, + 0.11477118067833636, + 0.4918381143601012, + 0.5930188602971916, + 0.1235145267478904 + ] + }, + "n_perts": 47 + }, + "AvgPerturbationEffect": { + "mse": 0.0036481301476089085, + "r2": -1.6605322170764842, + "pearson": 0.4622683938830457, + "spearman": 0.39142583024932337, + "de_corr": 0.4883508604971018, + "mmd": 0.10014484937386135, + "wasserstein": 0.3018542549345404, + "_per_pert": { + "mse": [ + 0.00167435547336936, + 0.0009507954237051308, + 0.0027328734286129475, + 0.0014651004457846284, + 0.0011944141006097198, + 0.0018723756074905396, + 0.0013803355395793915, + 0.0035241846926510334, + 0.002143725287169218, + 0.0022780015133321285, + 0.001303242170251906, + 0.003029764164239168, + 0.010649318806827068, + 0.0014153380179777741, + 0.001925644464790821, + 0.001829093205742538, + 0.0048235333524644375, + 0.006736383307725191, + 0.0006359592662192881, + 0.010419798083603382, + 0.0021609030663967133, + 0.0011679689632728696, + 0.0025179528165608644, + 0.0008555786334909499, + 0.001846952480264008, + 0.0012071691453456879, + 0.0025123802479356527, + 0.0076608723029494286, + 0.016836199909448624, + 0.003412527497857809, + 0.0010437192395329475, + 0.00940390769392252, + 0.0026134364306926727, + 0.007003629580140114, + 0.0014786251122131944, + 0.0009856611723080277, + 0.005891507025808096, + 0.004652815405279398, + 0.0011924020946025848, + 0.0059386445209383965, + 0.006750002969056368, + 0.0009650677675381303, + 0.0024069820065051317, + 0.0010960380313917994, + 0.005309190601110458, + 0.01162755023688078, + 0.0009401956340298057 + ], + "r2": [ + 0.5250583291053772, + -2.067795991897583, + 0.4663906693458557, + -24.22682762145996, + -30.548120498657227, + -10.564176559448242, + -1.181333065032959, + 0.41396796703338623, + 0.40977394580841064, + -0.5903782844543457, + 0.13891476392745972, + 0.3687893748283386, + 0.1831883192062378, + 0.2859535217285156, + 0.09758329391479492, + 0.0932665467262268, + 0.26741814613342285, + 0.3264654874801636, + 0.44522279500961304, + 0.007105767726898193, + 0.4030093550682068, + -1.8038930892944336, + 0.46773022413253784, + -0.29143571853637695, + 0.5437933206558228, + 0.6051965951919556, + 0.5175750255584717, + 0.01589643955230713, + 0.11185312271118164, + 0.28309279680252075, + -1.1159677505493164, + 0.2711823582649231, + -2.910392999649048, + -0.13172423839569092, + -4.0282745361328125, + 0.5530080795288086, + 0.34925276041030884, + 0.33678966760635376, + 0.12879538536071777, + 0.12150990962982178, + 0.09073930978775024, + -3.5584936141967773, + -0.5665128231048584, + -2.405522108078003, + 0.34308922290802, + 0.21394115686416626, + -1.4397189617156982 + ], + "pearson": [ + 0.737379252910614, + 0.4586031138896942, + 0.7093303799629211, + -0.25234323740005493, + 0.1738933026790619, + -0.15050499141216278, + 0.3571605682373047, + 0.679621696472168, + 0.6435927152633667, + 0.19227777421474457, + 0.5869457721710205, + 0.6202855110168457, + 0.45253705978393555, + 0.5960014462471008, + 0.4905829429626465, + 0.44439244270324707, + 0.5275831818580627, + 0.6465961337089539, + 0.7708851099014282, + 0.1986197978258133, + 0.6349406838417053, + 0.35338643193244934, + 0.7023636102676392, + 0.6127575039863586, + 0.7472073435783386, + 0.7900751829147339, + 0.7679491639137268, + 0.22588463127613068, + 0.36706048250198364, + 0.538213849067688, + 0.5286682844161987, + 0.5769003033638, + -0.30524545907974243, + 0.09802873432636261, + 0.11854038387537003, + 0.7439865469932556, + 0.6508612036705017, + 0.6041733622550964, + 0.5806814432144165, + 0.35568973422050476, + 0.3291589915752411, + 0.5059716701507568, + 0.11591579020023346, + 0.4430306851863861, + 0.641900360584259, + 0.4907100796699524, + 0.6243635416030884 + ], + "spearman": [ + 0.7083933706773007, + 0.3079713100159294, + 0.4046327512363655, + -0.041920830546138856, + -0.22250455279841613, + -0.37446157279400916, + -0.04115601111004243, + 0.4428028882606632, + 0.7038113920796653, + 0.5201007455465465, + 0.1764553940253625, + 0.307479988808722, + 0.6482464654386071, + 0.6252430897360204, + 0.3229249814306044, + 0.3245433173888679, + 0.2560018710704993, + 0.4606582793062035, + 0.5833880686915266, + 0.45562647697843667, + 0.6309124784820304, + 0.4308533873224278, + 0.767816622161061, + 0.5198515634258314, + 0.6736379330035954, + 0.5083028549033244, + 0.6489180099832399, + 0.346059653642849, + 0.48145509380778256, + 0.4494448301253817, + 0.4366564155926501, + 0.4280707717755995, + -0.024840797120072432, + 0.396796492645064, + 0.0785348149733743, + 0.705523408801131, + 0.299715903337534, + 0.5683499054652498, + 0.46037392128493043, + 0.6424110492753116, + 0.25206411135794143, + 0.15070971116413803, + 0.3665107979630368, + 0.3772544999395016, + 0.35032410981040296, + 0.5469437560747836, + 0.33612529907739225 + ], + "de_corr": [ + 0.8540282845497131, + 0.6691981554031372, + 0.8922524452209473, + -0.49096253514289856, + 0.3278610110282898, + -0.16388878226280212, + 0.7121587991714478, + 0.8541043996810913, + 0.6396335363388062, + 0.0007307860068976879, + 0.8476625680923462, + 0.8329525589942932, + -0.3111203908920288, + 0.7493950724601746, + 0.7377703785896301, + 0.5266879200935364, + 0.8043926358222961, + 0.7881635427474976, + 0.9095393419265747, + -0.4482191503047943, + 0.5458834767341614, + 0.16728194057941437, + 0.5472290515899658, + 0.9184627532958984, + 0.9280699491500854, + 0.8740137219429016, + 0.8739995360374451, + 0.3085290193557739, + -0.3118397295475006, + 0.36631032824516296, + 0.5417255163192749, + 0.7066925168037415, + -0.5113534331321716, + -0.17085237801074982, + 0.16569198668003082, + 0.872644305229187, + 0.8713144063949585, + 0.6834990978240967, + 0.8228168487548828, + 0.15404105186462402, + 0.19959162175655365, + 0.8399051427841187, + -0.00472659757360816, + 0.7255598306655884, + 0.9051570296287537, + 0.33812928199768066, + 0.8623735904693604 + ], + "mmd": [ + 0.06291116145751907, + 0.03737636644246678, + 0.09266723026850843, + 0.049567228025216625, + 0.044329834391393486, + 0.06264545216692041, + 0.05133777735970935, + 0.09621605241284636, + 0.07391450792826637, + 0.07196999240626378, + 0.046156650204655536, + 0.09682136583351053, + 0.19489274801003653, + 0.044353686639593715, + 0.0586054047280552, + 0.06642907820275523, + 0.15141614292000194, + 0.19550864448962824, + 0.028198343313148078, + 0.24949023789575764, + 0.06256788804255775, + 0.04464470220389738, + 0.07173329658399596, + 0.03284182947036085, + 0.06417759206185203, + 0.043663762429208886, + 0.08845362106376908, + 0.21566233510805466, + 0.25596716829163724, + 0.11002364394215791, + 0.040609433315430654, + 0.22463460265151292, + 0.09658945849103306, + 0.1956928415381134, + 0.057826830020943465, + 0.059529163109169336, + 0.16081938583966682, + 0.1424068295197991, + 0.04660008390879111, + 0.14559845988534936, + 0.18544416480316483, + 0.039382917173956145, + 0.07997849287093939, + 0.043427717128807, + 0.15654692956076366, + 0.23068043688661477, + 0.03649642957368271 + ], + "wasserstein": [ + 0.25153837911396004, + 0.16893252381456805, + 0.2676053666094325, + 0.20834732128994043, + 0.20906201319240444, + 0.2623580543810338, + 0.2082874359997082, + 0.29089588786363046, + 0.2932813172795775, + 0.28382051087695415, + 0.17286301008204027, + 0.2768469751638227, + 0.47183194502653286, + 0.1827742416065256, + 0.20242473866032043, + 0.23531629986454738, + 0.3880936894164569, + 0.4650012666968979, + 0.13480157794115927, + 0.5289359817374468, + 0.2794908138904269, + 0.22313267381781063, + 0.28506071168270364, + 0.17023104447518222, + 0.2860487361756974, + 0.19420078673364963, + 0.2737223791372607, + 0.467083203653169, + 0.6013220910540901, + 0.3559960638660447, + 0.19122034399507742, + 0.5544341593328391, + 0.2896746096005146, + 0.456163411743442, + 0.23632673230202164, + 0.24449735425823316, + 0.3823542699831418, + 0.33566691132404364, + 0.20076542659868996, + 0.4174490782135223, + 0.4888898951437967, + 0.17955707688635514, + 0.2753026664949679, + 0.1872511026375303, + 0.3795474592512466, + 0.5413690636213984, + 0.18737337943358137 + ] + }, + "n_perts": 47 + }, + "Additive": { + "mse": 0.002130915904962557, + "r2": -1.452756440385859, + "pearson": 0.6206745877069362, + "spearman": 0.5072803608982661, + "de_corr": 0.6938813036862523, + "mmd": 0.07430983260676954, + "wasserstein": 0.2578024455719882, + "_per_pert": { + "mse": [ + 0.00167435547336936, + 0.0009507954237051308, + 0.0027328734286129475, + 0.0014651004457846284, + 0.0011944141006097198, + 0.0018723756074905396, + 0.0013803355395793915, + 0.0011990240309387445, + 0.0011195228435099125, + 0.0022780015133321285, + 0.001303242170251906, + 0.003029764164239168, + 0.0012679604114964604, + 0.0037433153484016657, + 0.001925644464790821, + 0.001829093205742538, + 0.0048235333524644375, + 0.0035266857594251633, + 0.0006359592662192881, + 0.002117374911904335, + 0.0016054949956014752, + 0.0011679689632728696, + 0.0025179528165608644, + 0.0008555786334909499, + 0.001846952480264008, + 0.0012071691453456879, + 0.002857340034097433, + 0.0031821466982364655, + 0.003948783501982689, + 0.0011508818715810776, + 0.0010437192395329475, + 0.013433117419481277, + 0.00039268890395760536, + 0.007003629580140114, + 0.0014786251122131944, + 0.0007619631942361593, + 0.002539267996326089, + 0.001459421357139945, + 0.0011924020946025848, + 0.0012287272838875651, + 0.000827369571197778, + 0.0009650677675381303, + 0.0024069820065051317, + 0.0010960380313917994, + 0.0009213167359121144, + 0.002052875002846122, + 0.0009401956340298057 + ], + "r2": [ + 0.5250583291053772, + -2.067795991897583, + 0.4663906693458557, + -24.22682762145996, + -30.548120498657227, + -10.564176559448242, + -1.181333065032959, + 0.8006158471107483, + 0.6917648315429688, + -0.5903782844543457, + 0.13891476392745972, + 0.3687893748283386, + 0.9027464389801025, + -0.8885250091552734, + 0.09758329391479492, + 0.0932665467262268, + 0.26741814613342285, + 0.6473857760429382, + 0.44522279500961304, + 0.7982370257377625, + 0.5564513206481934, + -1.8038930892944336, + 0.46773022413253784, + -0.29143571853637695, + 0.5437933206558228, + 0.6051965951919556, + 0.4513360857963562, + 0.5912264585494995, + 0.7916929721832275, + 0.7582215666770935, + -1.1159677505493164, + -0.04108762741088867, + 0.41243356466293335, + -0.13172423839569092, + -4.0282745361328125, + 0.6544538736343384, + 0.7195248007774353, + 0.791974663734436, + 0.12879538536071777, + 0.8182371854782104, + 0.8885490298271179, + -3.5584936141967773, + -0.5665128231048584, + -2.405522108078003, + 0.8860046863555908, + 0.8612192273139954, + -1.4397189617156982 + ], + "pearson": [ + 0.737379252910614, + 0.4586031138896942, + 0.7093303799629211, + -0.25234323740005493, + 0.1738933026790619, + -0.15050499141216278, + 0.3571605682373047, + 0.977320671081543, + 0.8485552668571472, + 0.19227777421474457, + 0.5869457721710205, + 0.6202855110168457, + 0.9806216955184937, + 0.907036304473877, + 0.4905829429626465, + 0.44439244270324707, + 0.5275831818580627, + 0.8898630142211914, + 0.7708851099014282, + 0.897230327129364, + 0.7903505563735962, + 0.35338643193244934, + 0.7023636102676392, + 0.6127575039863586, + 0.7472073435783386, + 0.7900751829147339, + 0.9672414660453796, + 0.7713102698326111, + 0.9467152953147888, + 0.8849393725395203, + 0.5286682844161987, + 0.018820714205503464, + 0.943000853061676, + 0.09802873432636261, + 0.11854038387537003, + 0.8176720142364502, + 0.9195569157600403, + 0.9476909041404724, + 0.5806814432144165, + 0.905232310295105, + 0.9894416332244873, + 0.5059716701507568, + 0.11591579020023346, + 0.4430306851863861, + 0.9485238790512085, + 0.9331204295158386, + 0.6243635416030884 + ], + "spearman": [ + 0.7083933706773007, + 0.3079713100159294, + 0.4046327512363655, + -0.041920830546138856, + -0.22250455279841613, + -0.37446157279400916, + -0.04115601111004243, + 0.8302420688301261, + 0.758864041237267, + 0.5201007455465465, + 0.1764553940253625, + 0.307479988808722, + 0.9300612434600944, + 0.8013651787102366, + 0.3229249814306044, + 0.3245433173888679, + 0.2560018710704993, + 0.6411851049522752, + 0.5833880686915266, + 0.758357528861593, + 0.6538860797606403, + 0.4308533873224278, + 0.767816622161061, + 0.5198515634258314, + 0.6736379330035954, + 0.5083028549033244, + 0.8299614037560654, + 0.63705683850344, + 0.8853876420378269, + 0.7477887642857057, + 0.4366564155926501, + 0.04160304135281166, + 0.750667553052903, + 0.396796492645064, + 0.0785348149733743, + 0.8229722542204181, + 0.7481139514391116, + 0.9005541928837376, + 0.46037392128493043, + 0.8153443278432305, + 0.8967476883792962, + 0.15070971116413803, + 0.3665107979630368, + 0.3772544999395016, + 0.7829543318084666, + 0.8737905817438184, + 0.33612529907739225 + ], + "de_corr": [ + 0.8540282845497131, + 0.6691981554031372, + 0.8922524452209473, + -0.49096253514289856, + 0.3278610110282898, + -0.16388878226280212, + 0.7121587991714478, + 0.9931616187095642, + 0.9208729863166809, + 0.0007307860068976879, + 0.8476625680923462, + 0.8329525589942932, + 0.9927356243133545, + 0.9374369978904724, + 0.7377703785896301, + 0.5266879200935364, + 0.8043926358222961, + 0.9638471007347107, + 0.9095393419265747, + 0.9674869179725647, + 0.8717171549797058, + 0.16728194057941437, + 0.5472290515899658, + 0.9184627532958984, + 0.9280699491500854, + 0.8740137219429016, + 0.9877463579177856, + 0.929778516292572, + 0.9343132972717285, + 0.7840830087661743, + 0.5417255163192749, + 0.014756222255527973, + 0.9695466160774231, + -0.17085237801074982, + 0.16569198668003082, + 0.8839684724807739, + 0.9486202597618103, + 0.973825991153717, + 0.8228168487548828, + 0.918163537979126, + 0.9966879487037659, + 0.8399051427841187, + -0.00472659757360816, + 0.7255598306655884, + 0.9777360558509827, + 0.968001663684845, + 0.8623735904693604 + ], + "mmd": [ + 0.06291116145751907, + 0.03737636644246678, + 0.09266723026850843, + 0.049567228025216625, + 0.044329834391393486, + 0.06264545216692041, + 0.05133777735970935, + 0.05021868456888656, + 0.061060278690874825, + 0.07196999240626378, + 0.046156650204655536, + 0.09682136583351053, + 0.10767813138675597, + 0.13701120582615078, + 0.0586054047280552, + 0.06642907820275523, + 0.15141614292000194, + 0.12208139388607153, + 0.028198343313148078, + 0.09468476215576449, + 0.042705668881369396, + 0.04464470220389738, + 0.07173329658399596, + 0.03284182947036085, + 0.06417759206185203, + 0.043663762429208886, + 0.08901107561180033, + 0.10592565083802308, + 0.19750044167399128, + 0.03350597964301849, + 0.040609433315430654, + 0.2806191491336122, + 0.015222367381501956, + 0.1956928415381134, + 0.057826830020943465, + 0.05387107038760042, + 0.08132990830542441, + 0.059559122262586306, + 0.04660008390879111, + 0.06415637938945573, + 0.035576516207533215, + 0.039382917173956145, + 0.07997849287093939, + 0.043427717128807, + 0.045977810793240925, + 0.09735857949440396, + 0.03649642957368271 + ], + "wasserstein": [ + 0.25153837911396004, + 0.16893252381456805, + 0.2676053666094325, + 0.20834732128994043, + 0.20906201319240444, + 0.2623580543810338, + 0.2082874359997082, + 0.208885073570133, + 0.23475329927753966, + 0.28382051087695415, + 0.17286301008204027, + 0.2768469751638227, + 0.29428721718537315, + 0.30712842512790184, + 0.20242473866032043, + 0.23531629986454738, + 0.3880936894164569, + 0.3733510470160322, + 0.13480157794115927, + 0.29645300056511287, + 0.21274011479809748, + 0.22313267381781063, + 0.28506071168270364, + 0.17023104447518222, + 0.2860487361756974, + 0.19420078673364963, + 0.29555100498454245, + 0.3762041377967754, + 0.4745980007385362, + 0.18575707833168842, + 0.19122034399507742, + 0.6938497058349854, + 0.12119612743338175, + 0.456163411743442, + 0.23632673230202164, + 0.236194979054538, + 0.2834715800082889, + 0.2572407973833032, + 0.20076542659868996, + 0.21113540905871203, + 0.19100342704708737, + 0.17955707688635514, + 0.2753026664949679, + 0.1872511026375303, + 0.2113030277750908, + 0.3086794995132667, + 0.18737337943358137 + ] + }, + "n_perts": 47 + }, + "LinearResponse": { + "mse": 0.0011560855510129415, + "r2": -0.2554005422490708, + "pearson": 0.7328052140296774, + "spearman": 0.5781660967292104, + "de_corr": 0.7979922754333374, + "mmd": 0.04799841794265553, + "wasserstein": 0.20276722761569496, + "_per_pert": { + "mse": [ + 0.001815351890400052, + 0.0005269621615298092, + 0.0030712697189301252, + 0.000511285150423646, + 0.000643533596303314, + 0.0007793476688675582, + 0.0008428554865531623, + 0.00044451531721279025, + 0.0006145142251625657, + 0.0024074052926152945, + 0.0012071673991158605, + 0.00037343817530199885, + 0.000748060061596334, + 0.00042762543307617307, + 0.0017397834453731775, + 0.0017684008926153183, + 0.0010283575393259525, + 0.0005933258798904717, + 0.0004704850143752992, + 0.0010955922771245241, + 0.00029082701075822115, + 0.0007098543574102223, + 0.0003000711731147021, + 0.00047978051588870585, + 0.001113002304919064, + 0.001506912987679243, + 0.0008496627560816705, + 0.004119977355003357, + 0.00357564864680171, + 0.0008324038935825229, + 0.0006465500337071717, + 0.0017127461032941937, + 0.00030324782710522413, + 0.0010518929921090603, + 0.0008361907093785703, + 0.0013509067939594388, + 0.0015559562016278505, + 0.0013129838043823838, + 0.0009852976072579622, + 0.0016814854461699724, + 0.0006546838558278978, + 0.000686608487740159, + 0.0010450928239151835, + 0.0005867264699190855, + 0.0008671642863191664, + 0.003665788099169731, + 0.0005052817286923528 + ], + "r2": [ + 0.48506367206573486, + -0.7002733945846558, + 0.40031683444976807, + -7.803562164306641, + -15.997682571411133, + -3.813410758972168, + -0.33195745944976807, + 0.9260821342468262, + 0.8308074474334717, + -0.6807211637496948, + 0.20239382982254028, + 0.9221991896629333, + 0.9426231980323792, + 0.7842604517936707, + 0.18468362092971802, + 0.12335318326950073, + 0.8438165187835693, + 0.9406765699386597, + 0.5895739793777466, + 0.8956018686294556, + 0.9196535348892212, + -0.704116702079773, + 0.9365679621696472, + 0.27580517530441284, + 0.7250826358795166, + 0.507165789604187, + 0.8368484973907471, + 0.47075414657592773, + 0.811376690864563, + 0.8251277804374695, + -0.31077301502227783, + 0.8672595024108887, + 0.5462608337402344, + 0.830023467540741, + -1.8435852527618408, + 0.38737112283706665, + 0.8281366229057312, + 0.8128478527069092, + 0.280112087726593, + 0.7512617111206055, + 0.9118106365203857, + -2.243192195892334, + 0.3198322653770447, + -0.823029637336731, + 0.892704963684082, + 0.7521812319755554, + -0.3111581802368164 + ], + "pearson": [ + 0.7579284310340881, + 0.47559016942977905, + 0.6993327140808105, + -0.16080106794834137, + 0.11650417000055313, + -0.12152967602014542, + 0.5724650621414185, + 0.9785683155059814, + 0.9157998561859131, + 0.4514901638031006, + 0.8907166123390198, + 0.9634212851524353, + 0.9832399487495422, + 0.9335516691207886, + 0.4725910425186157, + 0.38926422595977783, + 0.9449183940887451, + 0.9823147058486938, + 0.7734150886535645, + 0.9583873152732849, + 0.9650200605392456, + 0.6997309923171997, + 0.9710015654563904, + 0.659519374370575, + 0.891148567199707, + 0.7662409543991089, + 0.9433495402336121, + 0.7203361392021179, + 0.9030676484107971, + 0.910010576248169, + 0.5172854661941528, + 0.9742609858512878, + 0.8469299674034119, + 0.9616471529006958, + 0.17740903794765472, + 0.7472275495529175, + 0.982086718082428, + 0.9204921722412109, + 0.5689943432807922, + 0.9189297556877136, + 0.9800853133201599, + 0.6968837976455688, + 0.6960882544517517, + 0.4919852316379547, + 0.9659656286239624, + 0.9531323313713074, + 0.6358475089073181 + ], + "spearman": [ + 0.7267308305160979, + 0.31736740644742656, + 0.4216959907901176, + 0.09782925927800983, + -0.24098214056210873, + -0.3451894804929966, + 0.07573042623336623, + 0.8199193010659636, + 0.8175295774756172, + 0.5655961559966857, + 0.3984229661797205, + 0.6557342397818725, + 0.9402929366906774, + 0.8674973284861689, + 0.3116749904282631, + 0.2980009207789957, + 0.6792444185233542, + 0.8402760939127509, + 0.5834434652061444, + 0.8207432128236153, + 0.8749760061811058, + 0.573012032926743, + 0.9297289170092395, + 0.5504182864916254, + 0.8073274992307962, + 0.5025536668690098, + 0.6564830794832915, + 0.657349432398684, + 0.8396238562408863, + 0.8014734601432296, + 0.4272860214421892, + 0.7321032153263551, + 0.6238218472324555, + 0.8380126318805834, + 0.11462644599705345, + 0.794577246419071, + 0.7311134626725649, + 0.7269334100695536, + 0.4811152217042791, + 0.8358231282393944, + 0.8521301566622356, + 0.2780840219636733, + 0.645078593970114, + 0.39831319285489375, + 0.598538622214927, + 0.8899792725932806, + 0.3617659184959151 + ], + "de_corr": [ + 0.8566803932189941, + 0.6914318203926086, + 0.8387675881385803, + -0.32153546810150146, + 0.26463887095451355, + -0.16012905538082123, + 0.8890593647956848, + 0.9942091107368469, + 0.9700576663017273, + 0.4726371765136719, + 0.9673143029212952, + 0.9935668706893921, + 0.9963659644126892, + 0.9523894190788269, + 0.7570591568946838, + 0.49530255794525146, + 0.9838354587554932, + 0.9930711984634399, + 0.920901894569397, + 0.9879890084266663, + 0.9676618576049805, + 0.714035153388977, + 0.967126727104187, + 0.8958656787872314, + 0.9540151953697205, + 0.8432382941246033, + 0.9835994839668274, + 0.8748316764831543, + 0.8785965442657471, + 0.7334020733833313, + 0.5192469358444214, + 0.977804958820343, + 0.820503294467926, + 0.9924158453941345, + 0.18588325381278992, + 0.8143969774246216, + 0.9902132153511047, + 0.9613940119743347, + 0.754327654838562, + 0.9321264624595642, + 0.9903842210769653, + 0.9279942512512207, + 0.7527039051055908, + 0.7175959348678589, + 0.9930542707443237, + 0.983783483505249, + 0.8358222842216492 + ], + "mmd": [ + 0.0629487132017087, + 0.022456076923069057, + 0.09968608333981799, + 0.019345235876682798, + 0.025701186718797286, + 0.028099133125839115, + 0.03359164574618889, + 0.022820628018136446, + 0.054755285262679254, + 0.08060966045136686, + 0.04327910366022525, + 0.02597645571513474, + 0.07836909986412388, + 0.026435072263026216, + 0.052843560263926115, + 0.06282847958579918, + 0.04689447736599561, + 0.04571262204721249, + 0.021987667452409987, + 0.05925346373925522, + 0.03021151442244563, + 0.029661855849053675, + 0.03843123932103254, + 0.0208589208184653, + 0.045885523907673864, + 0.05085836046271952, + 0.036318519873277166, + 0.13032579795668975, + 0.16314296583544763, + 0.030986573677355023, + 0.027234458699533115, + 0.07404027342328645, + 0.012739657915447844, + 0.04155824406909081, + 0.036207635881859934, + 0.05428925728897305, + 0.06009881585171151, + 0.0534873625868717, + 0.038367872652006696, + 0.0633094090432128, + 0.029446081378519118, + 0.028128208862154613, + 0.03978929112492802, + 0.025372501140748516, + 0.04561530973775052, + 0.11377756668985495, + 0.022188774213305895 + ], + "wasserstein": [ + 0.2566886884580639, + 0.12852174171606898, + 0.28999688116767314, + 0.13220750878374615, + 0.15637176144463816, + 0.1822112176075181, + 0.1794160322713854, + 0.1422491776338578, + 0.2094718904535178, + 0.27663250696543273, + 0.20272781105528417, + 0.14300537525789722, + 0.25418636412861717, + 0.1418415124534268, + 0.18949076792692993, + 0.22958166388786372, + 0.19895359836274593, + 0.21322797355669365, + 0.12038133791801668, + 0.21999133516795236, + 0.1506127703629451, + 0.17365526534875048, + 0.1664007394540281, + 0.13894913695800695, + 0.2369690471481899, + 0.2270707735174547, + 0.1809110964820951, + 0.4044117331140134, + 0.4124102136696584, + 0.18091131274702593, + 0.15158340322557998, + 0.2896965232193937, + 0.10218128314947407, + 0.19067208907009128, + 0.18465850656604055, + 0.2487455741256824, + 0.2383106791602504, + 0.21613429193883057, + 0.17311263449183453, + 0.21913956834357698, + 0.17800456359408123, + 0.15993518777288698, + 0.19096271242439403, + 0.13678648921740166, + 0.20984909332081356, + 0.3547931218713146, + 0.14603674142651785 + ] + }, + "n_perts": 47 + }, + "NearestPerturbationCentroid": { + "mse": 0.0033090435918371017, + "r2": -4.196858470744275, + "pearson": 0.6274464218381871, + "spearman": 0.503322270605923, + "de_corr": 0.7151544857572051, + "mmd": 0.1042386035709379, + "wasserstein": 0.307973293298589, + "_per_pert": { + "mse": [ + 0.0023796367458999157, + 0.001801011385396123, + 0.0033724126406013966, + 0.0035735154524445534, + 0.002037221100181341, + 0.0038693780079483986, + 0.0034680429380387068, + 0.0013926805695518851, + 0.0011195228435099125, + 0.0152303921058774, + 0.008728939108550549, + 0.004362241830676794, + 0.0018618378089740872, + 0.005347156431525946, + 0.0028414963744580746, + 0.002719290554523468, + 0.003768590511754155, + 0.0035266857594251633, + 0.001328550628386438, + 0.002117374911904335, + 0.0016054949956014752, + 0.00422050291672349, + 0.0006051911623217165, + 0.0016853106208145618, + 0.001079079112969339, + 0.0017575172241777182, + 0.0007794360863044858, + 0.0031821466982364655, + 0.002119982149451971, + 0.004216709174215794, + 0.0020710572134703398, + 0.013433117419481277, + 0.00028920292970724404, + 0.00193432392552495, + 0.002310030395165086, + 0.0016017381567507982, + 0.0030285550747066736, + 0.0034309562761336565, + 0.002545519033446908, + 0.003481273539364338, + 0.0024392546620219946, + 0.00397268682718277, + 0.0022698568645864725, + 0.0019501668866723776, + 0.0026432841550558805, + 0.009829113259911537, + 0.002197564346715808 + ], + "r2": [ + 0.32500070333480835, + -4.8110671043396, + 0.3415168523788452, + -60.53059387207031, + -52.80923843383789, + -22.8980712890625, + -4.480518341064453, + 0.7684130668640137, + 0.6917648315429688, + -9.633040428161621, + -4.767431735992432, + 0.0911855697631836, + 0.8571954965591431, + -1.6976723670959473, + -0.33161354064941406, + -0.34802961349487305, + 0.4276392459869385, + 0.6473857760429382, + -0.1589568853378296, + 0.7982370257377625, + 0.5564513206481934, + -9.131980895996094, + 0.8720687031745911, + -1.5438568592071533, + 0.7334619164466858, + 0.4252058267593384, + 0.8503333926200867, + 0.5912264585494995, + 0.8881662487983704, + 0.11414945125579834, + -3.1987247467041016, + -0.04108762741088867, + 0.5672757625579834, + 0.6874305009841919, + -6.85558557510376, + 0.2736205458641052, + 0.6654804348945618, + 0.5109529495239258, + -0.8598324060440063, + 0.4850231409072876, + 0.671419620513916, + -17.764972686767578, + -0.4772692918777466, + -5.059403419494629, + 0.672944188117981, + 0.33552122116088867, + -4.7024712562561035 + ], + "pearson": [ + 0.6041916608810425, + 0.4075838625431061, + 0.5931739807128906, + -0.13428668677806854, + 0.24753394722938538, + 0.12699802219867706, + 0.561170756816864, + 0.9018423557281494, + 0.8485552668571472, + 0.19876433908939362, + 0.9028801321983337, + 0.9268650412559509, + 0.9389623403549194, + 0.7554796934127808, + 0.40001431107521057, + 0.3590351641178131, + 0.8593475818634033, + 0.8898630142211914, + 0.6816719174385071, + 0.897230327129364, + 0.7903505563735962, + 0.8397958278656006, + 0.952285647392273, + 0.5144321322441101, + 0.8642792701721191, + 0.6805150508880615, + 0.9423225522041321, + 0.7713102698326111, + 0.9771626591682434, + 0.3689722418785095, + 0.42257338762283325, + 0.018820714205503464, + 0.7846940755844116, + 0.9069423079490662, + 0.1806703507900238, + 0.5792919993400574, + 0.934908390045166, + 0.7277368307113647, + 0.36097484827041626, + 0.7531649470329285, + 0.8293632864952087, + 0.6741036772727966, + 0.28217679262161255, + 0.4049389064311981, + 0.9165318012237549, + 0.6722699403762817, + 0.37251633405685425 + ], + "spearman": [ + 0.6203252538166802, + 0.34102383585978086, + 0.3837774128219293, + -0.08538268075448442, + -0.15712193365328184, + -0.08690643216056418, + 0.12051850525662616, + 0.5728070483326835, + 0.758864041237267, + 0.4063638602959689, + 0.46510206544970045, + 0.6587072622895157, + 0.8623349338856354, + 0.6791235634911009, + 0.24173554726317414, + 0.3196232471795954, + 0.67926463806182, + 0.6411851049522752, + 0.5718394343952093, + 0.758357528861593, + 0.6538860797606403, + 0.6796146821610123, + 0.8869795504626674, + 0.5045848809718708, + 0.7226048945742045, + 0.5395662119130941, + 0.8068636251611392, + 0.63705683850344, + 0.8851632168224438, + 0.4674815784115909, + 0.37613841940140774, + 0.04160304135281166, + 0.5045291544570197, + 0.7242173938528648, + 0.19138985182716672, + 0.7090978056290951, + 0.7526828356464393, + 0.7120370146217535, + 0.3447675147598305, + 0.703218443421422, + 0.6379013429899208, + 0.23381386660995387, + 0.3517490441858732, + 0.3820320385587042, + 0.6665511931362395, + 0.53366749407151, + 0.2554064683320432 + ], + "de_corr": [ + 0.7687333226203918, + 0.7302901148796082, + 0.8511683940887451, + -0.4226294755935669, + 0.40809565782546997, + 0.12300767749547958, + 0.712519109249115, + 0.9314944744110107, + 0.9208729863166809, + 0.1709519922733307, + 0.9414488077163696, + 0.9556061625480652, + 0.9622368812561035, + 0.6859473586082458, + 0.7796323895454407, + 0.34363195300102234, + 0.887298583984375, + 0.9638471007347107, + 0.901648223400116, + 0.9674869179725647, + 0.8717171549797058, + 0.9460306763648987, + 0.9477066397666931, + 0.860784649848938, + 0.9132860898971558, + 0.866118311882019, + 0.9768178462982178, + 0.929778516292572, + 0.9468079209327698, + -0.11106303334236145, + 0.5682790279388428, + 0.014756222255527973, + 0.8187680244445801, + 0.9813927412033081, + 0.2747507095336914, + 0.649765133857727, + 0.9683998227119446, + 0.7584554553031921, + 0.6796631217002869, + 0.8098202347755432, + 0.8685473799705505, + 0.8888605833053589, + 0.33092421293258667, + 0.7125753164291382, + 0.9640641808509827, + 0.8201625943183899, + 0.7718026638031006 + ], + "mmd": [ + 0.08293167912239963, + 0.06567023866872024, + 0.10981149427315096, + 0.11360836991853707, + 0.07356138077538543, + 0.13257402776857463, + 0.12208309135675088, + 0.045923174426250934, + 0.061060278690874825, + 0.37495171939497807, + 0.23522206539320523, + 0.1299205046141153, + 0.09585080507822352, + 0.17690753395243897, + 0.08517639343377525, + 0.09609675296496789, + 0.11969412337570717, + 0.12208139388607153, + 0.05038438010190227, + 0.09468476215576449, + 0.042705668881369396, + 0.1344707578483153, + 0.052099511988847436, + 0.06088613102868312, + 0.04568076626253059, + 0.06267252242363996, + 0.03607533595689161, + 0.10592565083802308, + 0.15829487043475998, + 0.127898550949471, + 0.07343368933446792, + 0.2806191491336122, + 0.01344139054898863, + 0.06217937882347979, + 0.08381984764257366, + 0.05800418421911402, + 0.09118554255760081, + 0.11214983798306177, + 0.08993828549492211, + 0.09231938033546683, + 0.07981969305115844, + 0.13099192564222595, + 0.07517288418616264, + 0.072506674779657, + 0.08886136637627329, + 0.2019740941021999, + 0.07789310765879187 + ], + "wasserstein": [ + 0.27050395334556, + 0.24584534168150252, + 0.3082609425935583, + 0.3408674339484363, + 0.2866592989331655, + 0.37534452611573565, + 0.3489738084317255, + 0.21457563399528223, + 0.23475329927753966, + 0.6088018751232922, + 0.47906488228095306, + 0.3323204870813563, + 0.27302225057812024, + 0.31507033117063266, + 0.2563207208449249, + 0.3075557417708538, + 0.36553640982299584, + 0.3733510470160322, + 0.1946241162930515, + 0.29645300056511287, + 0.21274011479809748, + 0.3394227367247307, + 0.20071874020825067, + 0.2262191027235575, + 0.23233900694510506, + 0.22184497430691644, + 0.18679896885264202, + 0.3762041377967754, + 0.3768296638704943, + 0.39835683509802455, + 0.28071070301355494, + 0.6938497058349854, + 0.10078415560965251, + 0.20926892369380642, + 0.29832451347657224, + 0.256509956026794, + 0.31141232467383717, + 0.31393154163828746, + 0.27114600673178285, + 0.28744571184910006, + 0.3105869786357878, + 0.3670474614914913, + 0.23118492268390065, + 0.27029257202798895, + 0.3050042117050033, + 0.4946549211788823, + 0.27321079256782954 + ] + }, + "n_perts": 47 + }, + "EndpointMLP": { + "mse": 0.0011665141643421289, + "r2": 0.05987304575899814, + "pearson": 0.5941063728223138, + "spearman": 0.5126150765634386, + "de_corr": 0.6435572355589334, + "mmd": 0.04903882547947358, + "wasserstein": 0.2003736154331534, + "_per_pert": { + "mse": [ + 0.0036682309582829475, + 0.0005103750736452639, + 0.005402616690844297, + 0.00031863481854088604, + 0.00032014859607443213, + 0.00048404073459096253, + 0.0007227485184557736, + 0.0007629882893525064, + 0.0011065156431868672, + 0.001901013427414, + 0.0007759711588732898, + 0.0003969112003687769, + 0.0005158786661922932, + 0.00045928655890747905, + 0.0023647623602300882, + 0.002461922587826848, + 0.0005208771326579154, + 0.0002854724589269608, + 0.0010409415699541569, + 0.0006360114784911275, + 0.0003024112666025758, + 0.0009925079066306353, + 0.0013474463485181332, + 0.0006413684459403157, + 0.0006882536690682173, + 0.0035410255659371614, + 0.0013137771748006344, + 0.0031697379890829325, + 0.003913164138793945, + 0.0009889702778309584, + 0.0007571351598016918, + 0.0007665202720090747, + 0.0002901284606195986, + 0.00045556839904747903, + 0.0004817750887013972, + 0.0006223590462468565, + 0.0015162485651671886, + 0.0010664068395271897, + 0.001473718206398189, + 0.0011742303613573313, + 0.0003918787115253508, + 0.00031622310052625835, + 0.0003426342736929655, + 0.0003908651415258646, + 0.0005885711871087551, + 0.0020876340568065643, + 0.000550258147995919 + ], + "r2": [ + -0.040517330169677734, + -0.646754264831543, + -0.05489206314086914, + -4.486414432525635, + -7.45610237121582, + -1.989534854888916, + -0.14215362071990967, + 0.8731237053871155, + 0.6953460574150085, + -0.3271852731704712, + 0.4872961640357971, + 0.9173088669776917, + 0.960431694984436, + 0.768287181854248, + -0.10820090770721436, + -0.2204451560974121, + 0.9208909273147583, + 0.9714571237564087, + 0.0919378399848938, + 0.9393949508666992, + 0.9164531230926514, + -1.3826713562011719, + 0.7151634693145752, + 0.03189957141876221, + 0.8299977779388428, + -0.1580885648727417, + 0.7477296590805054, + 0.5928204655647278, + 0.793571949005127, + 0.7922360897064209, + -0.534965991973877, + 0.9405934810638428, + 0.565890908241272, + 0.9263842105865479, + -0.6383448839187622, + 0.7177635431289673, + 0.8325225114822388, + 0.8479948043823242, + -0.07674252986907959, + 0.8262987732887268, + 0.9472119212150574, + -0.49367833137512207, + 0.7770066261291504, + -0.21446502208709717, + 0.9271755814552307, + 0.8588693737983704, + -0.4278682470321655 + ], + "pearson": [ + 0.05270738527178764, + 0.03536360338330269, + -0.010150463320314884, + 0.29591673612594604, + -0.5409607291221619, + -0.2308097779750824, + 0.4392741024494171, + 0.9674931764602661, + 0.851591944694519, + 0.7005103826522827, + 0.929423987865448, + 0.9622913002967834, + 0.9821222424507141, + 0.939292311668396, + 0.030473805963993073, + -0.2070801705121994, + 0.9726182222366333, + 0.9865130186080933, + 0.3205333650112152, + 0.9783572554588318, + 0.9621458649635315, + 0.7361637353897095, + 0.9669621586799622, + 0.28028619289398193, + 0.9313540458679199, + -0.20505286753177643, + 0.9817032814025879, + 0.7723673582077026, + 0.8908286094665527, + 0.8989760875701904, + -0.08951489627361298, + 0.975121021270752, + 0.912742018699646, + 0.980613648891449, + -0.0990881398320198, + 0.8734300136566162, + 0.9732058644294739, + 0.9366763830184937, + 0.07404083758592606, + 0.9202995896339417, + 0.986939549446106, + 0.6866685152053833, + 0.9149501919746399, + 0.2164500653743744, + 0.9744678735733032, + 0.946538507938385, + 0.06824231147766113 + ], + "spearman": [ + 0.18519241717322635, + 0.10822405351703812, + 0.0099834334442791, + 0.32957079734190514, + -0.38282363941605874, + -0.19851410140400802, + 0.06293497227217011, + 0.684183466545481, + 0.7633651219730669, + 0.6090042402780066, + 0.5550195144127006, + 0.6680776607895218, + 0.9226776819896063, + 0.8633905096673422, + 0.038745875090305415, + 0.03024947597073692, + 0.7993649769288612, + 0.8616925520559876, + 0.2007840903342541, + 0.8679031971924926, + 0.8521447921661414, + 0.5938497877609684, + 0.9097774986173688, + 0.1485962736009069, + 0.8294586601117799, + -0.06403498575742866, + 0.9133512546680557, + 0.6379728270089983, + 0.8564610276774808, + 0.7783468319713108, + 0.15165596073502338, + 0.8064451534176335, + 0.742932961749033, + 0.9156179799489049, + -0.19875691269698034, + 0.8384051489841586, + 0.8271939265073791, + 0.7626775754655413, + 0.19578851391369415, + 0.7616423377283719, + 0.9087424025444016, + 0.32851076506258664, + 0.7594768336569494, + 0.18198143633859454, + 0.7260797373442908, + 0.8718316408045742, + 0.07773287299496047 + ], + "de_corr": [ + -0.047485753893852234, + 0.08712006360292435, + 0.020298732444643974, + 0.6543987393379211, + -0.7934816479682922, + -0.44739097356796265, + 0.7694762945175171, + 0.9939884543418884, + 0.9243724346160889, + 0.7755008339881897, + 0.976364016532898, + 0.9942675828933716, + 0.9900578260421753, + 0.9717514514923096, + 0.2907801866531372, + -0.23932424187660217, + 0.9863185882568359, + 0.9952530264854431, + 0.6756848096847534, + 0.9973934292793274, + 0.9712522625923157, + 0.9157187342643738, + 0.9714534282684326, + 0.5391358733177185, + 0.9711427092552185, + -0.18987883627414703, + 0.9913809299468994, + 0.9305169582366943, + 0.856482207775116, + 0.7723619341850281, + -0.1443070024251938, + 0.9835894703865051, + 0.9258934855461121, + 0.9954570531845093, + -0.10771752148866653, + 0.9215916395187378, + 0.9929366707801819, + 0.9812349677085876, + 0.09604117274284363, + 0.9337543845176697, + 0.9960479140281677, + 0.9483177661895752, + 0.9571083784103394, + 0.3764186203479767, + 0.9903275370597839, + 0.9790841937065125, + 0.11650128662586212 + ], + "mmd": [ + 0.09884149292600153, + 0.01946630129346827, + 0.15171916808125996, + 0.01677440707837663, + 0.015576968228695032, + 0.022819728212541968, + 0.028901073634166274, + 0.03629310212660142, + 0.060784358970313646, + 0.06759542769611904, + 0.029645911324162944, + 0.026939055914784404, + 0.08743045767279123, + 0.028307923566447735, + 0.06905576237887412, + 0.0797458918350391, + 0.030019970001558538, + 0.034842742497081414, + 0.03638062109455609, + 0.05321159242782125, + 0.037715922203221064, + 0.03972560217839849, + 0.08046538346937837, + 0.026080472603711158, + 0.03700166800847815, + 0.10378621415481315, + 0.051635032985873575, + 0.10556357239049574, + 0.17011389888447925, + 0.02957054053553898, + 0.029268021363971553, + 0.05678986274996556, + 0.011811875833486196, + 0.022807423737228105, + 0.020770561428410272, + 0.04979912365457795, + 0.0582438261768079, + 0.04700165123440936, + 0.046960950863845774, + 0.05820412759803262, + 0.022515691973779672, + 0.015578848498585263, + 0.016934224570258505, + 0.016198152371246666, + 0.03738652011899268, + 0.09475845081580081, + 0.023785220170810417 + ], + "wasserstein": [ + 0.34860048423544365, + 0.12762712183614455, + 0.399214916609018, + 0.1257260244166984, + 0.10169126561146175, + 0.13718531649965465, + 0.1631261575339426, + 0.17884611453706611, + 0.2338028239218351, + 0.24464420344797677, + 0.16695945350802588, + 0.13668125218964633, + 0.2636733644328416, + 0.13194942639478316, + 0.2295172279942127, + 0.2662629058790926, + 0.15312860954232194, + 0.17961483128945008, + 0.20226673508647297, + 0.1821283675702408, + 0.15910928740642663, + 0.15789000268005446, + 0.23752961041334839, + 0.1657625071861199, + 0.1998985368209983, + 0.3469739391319118, + 0.2211719215833392, + 0.3753918946984671, + 0.4377202495175173, + 0.18825635686929837, + 0.15371424038611559, + 0.24215011123070174, + 0.09309381660995744, + 0.1430783438146134, + 0.14201223355852183, + 0.22237483417978077, + 0.2216233619866504, + 0.21794066446238272, + 0.19986639207741955, + 0.19920056674123057, + 0.15563173184666473, + 0.10774408592838654, + 0.11859925892772133, + 0.10275211762736548, + 0.17996026728308198, + 0.31309201486930155, + 0.14237497498450202 + ] + }, + "n_perts": 47 + }, + "Random": { + "mse": 0.006797554055625137, + "r2": -6.335227725353647, + "pearson": 0.2715044714351918, + "spearman": 0.2418118923013008, + "de_corr": 0.285452477891553, + "mmd": 0.16974875210115886, + "wasserstein": 0.3916861021971288, + "_per_pert": { + "mse": [ + 0.002127312822267413, + 0.002602711785584688, + 0.004285301081836224, + 0.00466095469892025, + 0.00258551724255085, + 0.0011789415730163455, + 0.0020540133118629456, + 0.005653779953718185, + 0.00194260582793504, + 0.004209674894809723, + 0.008063239976763725, + 0.009813507087528706, + 0.019128642976284027, + 0.0022924779914319515, + 0.015366286039352417, + 0.0021281992085278034, + 0.005382271949201822, + 0.012778623029589653, + 0.0021838033571839333, + 0.024319134652614594, + 0.004210384096950293, + 0.0021094647236168385, + 0.0014473391929641366, + 0.0016853106208145618, + 0.002427374944090843, + 0.005390360951423645, + 0.006245764903724194, + 0.012452896684408188, + 0.02416934259235859, + 0.004013997968286276, + 0.003113203914836049, + 0.012817734852433205, + 0.006395484320819378, + 0.0062553430907428265, + 0.0007524136453866959, + 0.0029931014869362116, + 0.004285393748432398, + 0.01057181041687727, + 0.0013701937859877944, + 0.004784707445651293, + 0.008943222463130951, + 0.002139439107850194, + 0.003122832393273711, + 0.025033049285411835, + 0.015380092896521091, + 0.010265622287988663, + 0.004352159332484007 + ], + "r2": [ + 0.3965739607810974, + -7.397797584533691, + 0.1632699966430664, + -79.25465393066406, + -67.2914047241211, + -6.281386375427246, + -2.2459397315979004, + 0.05984055995941162, + 0.46514761447906494, + -1.9389686584472656, + -4.327587604522705, + -1.0445117950439453, + -0.467182993888855, + -0.1565690040588379, + -6.201117515563965, + -0.055008888244628906, + 0.1825588345527649, + -0.27766525745391846, + -0.9050338268280029, + -1.3173508644104004, + -0.1631988286972046, + -4.06410026550293, + 0.694047212600708, + -1.5438568592071533, + 0.4004262089729309, + -0.7629115581512451, + -0.19930613040924072, + -0.5996795892715454, + -0.27498626708984375, + 0.15673530101776123, + -5.311505317687988, + 0.006605386734008789, + -8.56933879852295, + -0.010807633399963379, + -1.558689832687378, + -0.35735511779785156, + 0.5266561508178711, + -0.5069013833999634, + -0.0011047124862670898, + 0.2922091484069824, + -0.20469868183135986, + -9.105634689331055, + -1.0324029922485352, + -76.78069305419922, + -0.9029921293258667, + 0.3060118556022644, + -10.293442726135254 + ], + "pearson": [ + 0.6368151903152466, + 0.3726290762424469, + 0.7447395920753479, + 0.315721333026886, + -0.38576221466064453, + 0.03760609030723572, + 0.5410704016685486, + 0.5128589272499084, + 0.6846206784248352, + -0.1384371668100357, + -0.5415480732917786, + 0.6354108452796936, + -0.14010487496852875, + 0.09047293663024902, + 0.041458435356616974, + 0.3128522038459778, + 0.481003075838089, + 0.11361993104219437, + -0.208737313747406, + -0.398562490940094, + 0.3834177851676941, + 0.19152754545211792, + 0.8739404082298279, + 0.5144321322441101, + 0.8524073362350464, + 0.4876151978969574, + -0.09158381074666977, + 0.1261187642812729, + -0.09059347212314606, + 0.4108235836029053, + -0.15390656888484955, + 0.2239699810743332, + -0.30447566509246826, + 0.14265793561935425, + 0.126038059592247, + 0.36229193210601807, + 0.7881902456283569, + 0.42433008551597595, + 0.3949683904647827, + 0.544305682182312, + 0.25457534193992615, + 0.7993372678756714, + 0.16079279780387878, + 0.24126645922660828, + 0.40922462940216064, + 0.5611183047294617, + 0.42019322514533997 + ], + "spearman": [ + 0.5310529498815971, + 0.29072441046490377, + 0.43642513838461816, + 0.17381459232158886, + -0.3283585753806948, + -0.22055620227822795, + 0.2055008514765372, + 0.05186900211199081, + 0.6403380137191934, + 0.21481422410354808, + -0.21637657866435825, + 0.3798212132572192, + -0.03674943755750615, + -0.014436720064903277, + 0.18363924632758857, + 0.2092071265821519, + 0.361022396795536, + 0.20171339015147832, + -0.219319054448528, + -0.08815865322980515, + 0.43652040983268925, + 0.2756451397863652, + 0.8317345702191635, + 0.5045848809718708, + 0.7590550197881629, + 0.36163291987517315, + -0.016590193371515953, + 0.253648292442388, + -0.11070730267137427, + 0.40015038854013496, + 0.17864666796226952, + 0.34988195724354165, + -0.018705086741920025, + 0.24596091498305447, + 0.08549474959847182, + 0.3821888723899761, + 0.4397286239482889, + 0.36892227620613244, + 0.3143309932549579, + 0.6068896338930021, + 0.23259526412607534, + 0.3464865572186, + 0.2943930949557598, + 0.2548029642505411, + 0.18254030716782355, + 0.4849836537250617, + 0.16435603461251672 + ], + "de_corr": [ + 0.6353400945663452, + 0.6368319392204285, + 0.883208155632019, + 0.6014145016670227, + -0.7253379225730896, + 0.06181507557630539, + 0.8785457015037537, + 0.5250102281570435, + 0.781560480594635, + -0.31299322843551636, + -0.7867901921272278, + 0.8307375907897949, + -0.5707265734672546, + 0.26218822598457336, + 0.12379666417837143, + 0.569264829158783, + 0.4927721321582794, + 0.025338536128401756, + -0.27684250473976135, + -0.8801300525665283, + 0.1953013688325882, + 0.0615214928984642, + 0.9277873039245605, + 0.860784649848938, + 0.9239442944526672, + 0.6461829543113708, + 0.07855163514614105, + 0.08836401253938675, + -0.41076961159706116, + 0.17942115664482117, + -0.3638710081577301, + -0.07429542392492294, + -0.5127584934234619, + 0.43892529606819153, + 0.1457577645778656, + 0.3521004617214203, + 0.9495858550071716, + 0.6043353080749512, + 0.5511282086372375, + 0.40757182240486145, + 0.2117714136838913, + 0.9158341884613037, + -0.010485046543180943, + 0.5453157424926758, + 0.663602352142334, + 0.6413303017616272, + 0.644324779510498 + ], + "mmd": [ + 0.07582423070000577, + 0.09066030471428443, + 0.13440939834259258, + 0.15669938595753508, + 0.09433798250725844, + 0.04083013235400712, + 0.07541903320460452, + 0.16280151297449774, + 0.07923091625328016, + 0.12258262992934121, + 0.21741939603791371, + 0.2719747586976766, + 0.2989449496887061, + 0.05989172056223657, + 0.34906586657617056, + 0.07681751019169347, + 0.16409425708629988, + 0.3096559604587591, + 0.06942023545417886, + 0.4613971666419753, + 0.10886920379904819, + 0.0753278417420915, + 0.07004345768384057, + 0.06088613102868312, + 0.09035913843038934, + 0.15901598591416166, + 0.1676486447176112, + 0.31342991469262405, + 0.3228716332238504, + 0.12478109170540197, + 0.10392395112691877, + 0.28367725308136127, + 0.20236387850499327, + 0.18093193948842523, + 0.0314208226916749, + 0.10333577865562027, + 0.12568169023156728, + 0.2704420928984306, + 0.050206667875308475, + 0.1331245515640146, + 0.2390971665902627, + 0.06865022393785758, + 0.10264098900531793, + 0.5416579725882449, + 0.3738631913442223, + 0.2291659573858711, + 0.13329683051365715 + ], + "wasserstein": [ + 0.2642483104273426, + 0.22755625862254336, + 0.3795218265785536, + 0.36722831406082085, + 0.21921312521240682, + 0.2055172542355999, + 0.24338368856814907, + 0.3743023131000601, + 0.2866021605998024, + 0.36364981599250074, + 0.3876888131099121, + 0.45845710413365504, + 0.6548866648130107, + 0.22070921078712658, + 0.5413695945606194, + 0.24476616319496178, + 0.4192243219339754, + 0.6093798144485473, + 0.28708298785192027, + 0.7928895882157252, + 0.35499459780271536, + 0.29159671334039333, + 0.24400529341732097, + 0.2262191027235575, + 0.2968901788601761, + 0.41169307158882873, + 0.46124014647731826, + 0.5479934202071135, + 0.7132902073724848, + 0.36284336809629314, + 0.30780573662145594, + 0.6427933537474344, + 0.4184564232621895, + 0.44394911221452515, + 0.19738676814739847, + 0.31440434152639307, + 0.3352521747253045, + 0.4557445354499354, + 0.20604127144019885, + 0.38820798613802365, + 0.501121909574595, + 0.2250441501159834, + 0.3195775018898895, + 0.7079157195686178, + 0.5903799193185655, + 0.5201768708734071, + 0.37654559831770007 + ] + }, + "n_perts": 47 + } + } +} \ No newline at end of file diff --git a/experiments/results/norman_nom_cell.json b/experiments/results/norman_nom_cell.json new file mode 100644 index 0000000000000000000000000000000000000000..053a29f5511b456d3dc020789bab56c1f0cc0ae0 --- /dev/null +++ b/experiments/results/norman_nom_cell.json @@ -0,0 +1,2773 @@ +{ + "split": "cell", + "reward": "centroid", + "n_candidates": 105, + "n_targets": 40, + "methods": { + "PIVOT-ranking": { + "top1": 0.55, + "top5": 0.9, + "ndcg": 0.7454003731910971, + "func_top5": 1.0, + "func_ndcg": 0.7362368327199266, + "rank": 2.6, + "endpoint_dist": 0.9490216635167599, + "spec_margin": 1.730977939069271, + "off_dist": 2.6799996107816697, + "clf": null, + "_per": { + "top1": [ + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 0.0, + 1.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0 + ], + "top5": [ + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "ndcg": [ + 0.9999999999989999, + 0.0, + 0.3868528072341547, + 0.9999999999989999, + 0.9999999999989999, + 0.6309297535708266, + 0.49999999999949996, + 0.9999999999989999, + 0.3868528072341547, + 0.9999999999989999, + 0.6309297535708266, + 0.30102999566368016, + 0.9999999999989999, + 0.49999999999949996, + 0.6309297535708266, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.6309297535708266, + 0.43067655807296235, + 0.9999999999989999, + 0.49999999999949996, + 0.49999999999949996, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.35620718710766597, + 0.0, + 0.49999999999949996, + 0.49999999999949996, + 0.43067655807296235, + 0.9999999999989999, + 0.9999999999989999 + ], + "func_top5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.7719858642580791, + 0.5452792644002579, + 0.4622969259439367, + 0.9999999999996392, + 0.7601875334312905, + 0.6309297535708266, + 0.49999999999949996, + 0.9999999999996392, + 0.6585618697746796, + 0.9999999999996392, + 0.5679946984137687, + 0.761639033082953, + 0.9430934427597557, + 0.7985960275052596, + 0.7492069087978183, + 0.865959765373318, + 0.7723904789459194, + 0.8814413010323419, + 0.5679946984137687, + 0.7729141359198264, + 0.7209064905476281, + 0.909805240729282, + 0.909805240729282, + 0.9478557351531002, + 0.7101872472819487, + 0.7229846523326642, + 0.9540734464802791, + 0.629042953814712, + 0.9999999999996392, + 0.5449154948697388, + 0.7601875334312905, + 0.803358319898604, + 0.6980754691453294, + 0.5905990152147871, + 0.305935146935109, + 0.7209064905476281, + 0.4384685740390607, + 0.7239931533469753, + 0.6792194063292641, + 0.6686819963485252 + ], + "rank": [ + 1, + 12, + 5, + 1, + 1, + 2, + 3, + 1, + 5, + 1, + 2, + 9, + 1, + 3, + 2, + 1, + 1, + 1, + 2, + 4, + 1, + 3, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 6, + 11, + 3, + 3, + 4, + 1, + 1 + ], + "endpoint_dist": [ + 1.2624379396438599, + 1.987898588180542, + 0.6130089163780212, + 0.7837377190589905, + 0.3801593482494354, + 0.5316212177276611, + 1.1274526119232178, + 0.9317931532859802, + 0.8158573508262634, + 0.5724427103996277, + 1.043930172920227, + 0.9461696147918701, + 1.0851073265075684, + 0.9249292612075806, + 1.3508139848709106, + 1.8004765510559082, + 0.8150457739830017, + 0.7767999768257141, + 1.0714013576507568, + 1.3535343408584595, + 1.0559062957763672, + 1.2047194242477417, + 1.0925180912017822, + 1.2413640022277832, + 1.438982605934143, + 0.7611419558525085, + 0.5610932111740112, + 0.6496677398681641, + 0.5188019871711731, + 0.8329150676727295, + 0.8286756277084351, + 0.6340292692184448, + 0.37346237897872925, + 0.7785728573799133, + 1.0523473024368286, + 1.1261812448501587, + 0.8302287459373474, + 0.9894613027572632, + 1.143169641494751, + 0.6730098724365234 + ], + "spec_margin": [ + 0.923587441444397, + -0.014548420906066895, + 1.6584539413452148, + 1.815046787261963, + 1.5790126323699951, + 1.7291734218597412, + 1.253880262374878, + 2.382495403289795, + 1.2740118503570557, + 1.9212472438812256, + 1.310372233390808, + 0.9761004447937012, + 1.247694969177246, + 1.690231442451477, + 1.1252025365829468, + 4.583049297332764, + 3.685917377471924, + 3.4045183658599854, + 1.1300947666168213, + 0.2355557680130005, + 1.5858967304229736, + 2.10831880569458, + 1.798520803451538, + 1.1007182598114014, + 1.2903398275375366, + 2.7685585021972656, + 1.9255796670913696, + 2.0334508419036865, + 2.7833263874053955, + 1.098803997039795, + 2.4604926109313965, + 3.0710902214050293, + 3.5961661338806152, + 2.221989154815674, + 1.0602291822433472, + 0.5479577779769897, + 0.813274085521698, + 0.7865604162216187, + 1.0803568363189697, + 1.1963895559310913 + ], + "off_dist": [ + 2.186025381088257, + 1.973350167274475, + 2.271462917327881, + 2.5987844467163086, + 1.959172010421753, + 2.2607946395874023, + 2.3813328742980957, + 3.31428861618042, + 2.089869260787964, + 2.493690013885498, + 2.354302406311035, + 1.9222700595855713, + 2.3328022956848145, + 2.6151607036590576, + 2.4760165214538574, + 6.383525848388672, + 4.50096321105957, + 4.181318283081055, + 2.201496124267578, + 1.58909010887146, + 2.641803026199341, + 3.3130383491516113, + 2.8910388946533203, + 2.3420822620391846, + 2.7293224334716797, + 3.529700517654419, + 2.486672878265381, + 2.6831185817718506, + 3.302128314971924, + 1.9317190647125244, + 3.289168119430542, + 3.7051196098327637, + 3.9696285724639893, + 3.0005619525909424, + 2.112576484680176, + 1.6741390228271484, + 1.6435028314590454, + 1.7760217189788818, + 2.2235264778137207, + 1.8693994283676147 + ], + "clf": [] + }, + "n_targets": 40, + "random_top5": 0.047619047619047616 + }, + "PIVOT-guidance": { + "top1": 0.525, + "top5": 0.875, + "ndcg": 0.7150776971677019, + "func_top5": 0.975, + "func_ndcg": 0.6276044333475127, + "rank": 2.925, + "endpoint_dist": 0.982820986956358, + "spec_margin": 1.6728850707411766, + "off_dist": 2.655706062912941, + "clf": null, + "_per": { + "top1": [ + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 0.0, + 1.0, + 0.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0 + ], + "top5": [ + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0 + ], + "ndcg": [ + 0.0, + 0.3868528072341547, + 0.49999999999949996, + 0.9999999999989999, + 0.9999999999989999, + 0.6309297535708266, + 0.49999999999949996, + 0.9999999999989999, + 0.0, + 0.9999999999989999, + 0.6309297535708266, + 0.9999999999989999, + 0.9999999999989999, + 0.49999999999949996, + 0.6309297535708266, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.6309297535708266, + 0.0, + 0.9999999999989999, + 0.49999999999949996, + 0.6309297535708266, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.0, + 0.9999999999989999, + 0.6309297535708266, + 0.43067655807296235, + 0.49999999999949996, + 0.49999999999949996, + 0.0, + 0.9999999999989999, + 0.9999999999989999 + ], + "func_top5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.3949701603078018, + 0.5088566520086942, + 0.4851649195781592, + 0.9478557351531002, + 0.7601875334312905, + 0.6309297535708266, + 0.49999999999949996, + 0.642476447789185, + 0.3843973922908389, + 0.642476447789185, + 0.5583403462841774, + 0.9999999999996392, + 0.8814413010323419, + 0.6766675710155576, + 0.7058140748849717, + 0.75951284454264, + 0.5647869108315535, + 0.7122606394323479, + 0.5713692255708102, + 0.3718921699781131, + 0.6920970407325434, + 0.909805240729282, + 0.7700701178681301, + 0.642476447789185, + 0.3607790370814292, + 0.5267364164077353, + 0.9540734464802791, + 0.6169311756780926, + 0.9478557351531002, + 0.5916118404088373, + 0.7601875334312905, + 0.0, + 0.5347219910747678, + 0.4080157475002485, + 0.604636395515731, + 0.6220658801619905, + 0.5241325151519908, + 0.6562570033883632, + 0.5957547517842381, + 0.686568892072539 + ], + "rank": [ + 11, + 5, + 3, + 1, + 1, + 2, + 3, + 1, + 12, + 1, + 2, + 1, + 1, + 3, + 2, + 1, + 1, + 1, + 2, + 12, + 1, + 3, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 11, + 1, + 2, + 4, + 3, + 3, + 11, + 1, + 1 + ], + "endpoint_dist": [ + 1.626929521560669, + 1.987898588180542, + 0.6130089163780212, + 0.7837377190589905, + 0.3801593482494354, + 0.5316212177276611, + 1.1274526119232178, + 0.9317931532859802, + 0.8158573508262634, + 0.5724427103996277, + 1.043930172920227, + 0.8083254098892212, + 1.0851073265075684, + 0.9249292612075806, + 1.3508139848709106, + 1.8004765510559082, + 0.8150457739830017, + 0.7767999768257141, + 1.0714013576507568, + 1.3535343408584595, + 1.0559062957763672, + 1.2047194242477417, + 1.0925180912017822, + 1.2413640022277832, + 1.438982605934143, + 0.7611419558525085, + 0.5610932111740112, + 0.6496677398681641, + 0.5188019871711731, + 0.8329150676727295, + 0.8286756277084351, + 1.759354829788208, + 0.37346237897872925, + 0.7785728573799133, + 1.0523473024368286, + 1.1261812448501587, + 0.8302287459373474, + 0.9894613027572632, + 1.143169641494751, + 0.6730098724365234 + ], + "spec_margin": [ + 0.13825678825378418, + -0.014548420906066895, + 1.6584539413452148, + 1.815046787261963, + 1.5790126323699951, + 1.7291734218597412, + 1.253880262374878, + 2.382495403289795, + 1.2740118503570557, + 1.9212472438812256, + 1.310372233390808, + 1.7914849519729614, + 1.247694969177246, + 1.690231442451477, + 1.1252025365829468, + 4.583049297332764, + 3.685917377471924, + 3.4045183658599854, + 1.1300947666168213, + 0.2355557680130005, + 1.5858967304229736, + 2.10831880569458, + 1.798520803451538, + 1.1007182598114014, + 1.2903398275375366, + 2.7685585021972656, + 1.9255796670913696, + 2.0334508419036865, + 2.7833263874053955, + 1.098803997039795, + 2.4604926109313965, + 0.7173216342926025, + 3.5961661338806152, + 2.221989154815674, + 1.0602291822433472, + 0.5479577779769897, + 0.813274085521698, + 0.7865604162216187, + 1.0803568363189697, + 1.1963895559310913 + ], + "off_dist": [ + 1.7651863098144531, + 1.973350167274475, + 2.271462917327881, + 2.5987844467163086, + 1.959172010421753, + 2.2607946395874023, + 2.3813328742980957, + 3.31428861618042, + 2.089869260787964, + 2.493690013885498, + 2.354302406311035, + 2.5998103618621826, + 2.3328022956848145, + 2.6151607036590576, + 2.4760165214538574, + 6.383525848388672, + 4.50096321105957, + 4.181318283081055, + 2.201496124267578, + 1.58909010887146, + 2.641803026199341, + 3.3130383491516113, + 2.8910388946533203, + 2.3420822620391846, + 2.7293224334716797, + 3.529700517654419, + 2.486672878265381, + 2.6831185817718506, + 3.302128314971924, + 1.9317190647125244, + 3.289168119430542, + 2.4766764640808105, + 3.9696285724639893, + 3.0005619525909424, + 2.112576484680176, + 1.6741390228271484, + 1.6435028314590454, + 1.7760217189788818, + 2.2235264778137207, + 1.8693994283676147 + ], + "clf": [] + }, + "n_targets": 40, + "random_top5": 0.047619047619047616 + }, + "Additive+ranking": { + "top1": 1.0, + "top5": 1.0, + "ndcg": 0.9999999999989999, + "func_top5": 1.0, + "func_ndcg": 0.8479507485118546, + "rank": 1.0, + "endpoint_dist": 0.42116108611226083, + "spec_margin": 2.4543789327144623, + "off_dist": 2.8755400240421296, + "clf": null, + "_per": { + "top1": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "top5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "ndcg": [ + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999 + ], + "func_top5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.7644049042788869, + 0.7623315121284877, + 0.7691671966722313, + 0.9999999999996392, + 0.7601875334312905, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999996392, + 0.7209064905476281, + 0.9999999999996392, + 0.7122606394323479, + 0.9999999999996392, + 0.9999999999996392, + 0.9999999999996392, + 0.9999999999996392, + 0.8525262713988451, + 0.7490035979724269, + 0.8580544200588494, + 0.7122606394323479, + 0.9223104630420079, + 0.8700859688429049, + 0.9999999999996392, + 0.9999999999996392, + 0.9478557351531002, + 0.6588088387020696, + 0.6655389434274009, + 0.8460101051759085, + 0.6734442887418696, + 0.9999999999996392, + 0.3607790370814292, + 0.7601875334312905, + 0.8935530791689612, + 0.6319672012189719, + 0.8366465219290776, + 0.7923964318852734, + 0.9478557351531002, + 0.6719052786222132, + 0.8887907867756167, + 0.9456973440155002, + 0.9430934427597557 + ], + "rank": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "endpoint_dist": [ + 0.4336645305156708, + 0.4492749273777008, + 0.36839205026626587, + 0.5400750041007996, + 0.4056673049926758, + 0.5053470730781555, + 0.3870347738265991, + 0.46958184242248535, + 0.3555386960506439, + 0.49682140350341797, + 0.3571249842643738, + 0.448895126581192, + 0.416703999042511, + 0.43106698989868164, + 0.44865232706069946, + 0.44056838750839233, + 0.38007888197898865, + 0.38654324412345886, + 0.33925434947013855, + 0.4254252016544342, + 0.7614416480064392, + 0.44820210337638855, + 0.3869432210922241, + 0.4388890266418457, + 0.370621919631958, + 0.49089911580085754, + 0.3214934170246124, + 0.42563191056251526, + 0.3777088224887848, + 0.4537658989429474, + 0.3759283721446991, + 0.34280818700790405, + 0.4167364537715912, + 0.43289661407470703, + 0.37268662452697754, + 0.39969366788864136, + 0.3186255097389221, + 0.38318541646003723, + 0.4199433922767639, + 0.42263102531433105 + ], + "spec_margin": [ + 1.8226473331451416, + 2.429075002670288, + 2.0126900672912598, + 2.4777753353118896, + 1.6256582736968994, + 1.7526733875274658, + 2.1037702560424805, + 2.804387331008911, + 1.832645058631897, + 2.120689868927002, + 2.2210195064544678, + 1.8804901838302612, + 1.9542350769042969, + 1.9912099838256836, + 2.1478829383850098, + 7.4463677406311035, + 3.817469596862793, + 4.136026859283447, + 2.065373182296753, + 1.8025559186935425, + 1.9463722705841064, + 2.922027587890625, + 2.956066131591797, + 2.1518471240997314, + 2.761521577835083, + 3.247483968734741, + 2.3110668659210205, + 2.3323137760162354, + 3.1342692375183105, + 1.6014208793640137, + 2.703058958053589, + 2.9958693981170654, + 3.7247540950775146, + 2.5874547958374023, + 1.8483490943908691, + 1.554269552230835, + 1.318281650543213, + 1.786970853805542, + 2.3373780250549316, + 1.509738564491272 + ], + "off_dist": [ + 2.2563118934631348, + 2.878350019454956, + 2.381082057952881, + 3.017850399017334, + 2.031325578689575, + 2.2580204010009766, + 2.49080491065979, + 3.2739691734313965, + 2.1881837844848633, + 2.61751127243042, + 2.5781445503234863, + 2.329385280609131, + 2.370939016342163, + 2.4222769737243652, + 2.5965352058410645, + 7.886936187744141, + 4.1975483894348145, + 4.5225701332092285, + 2.404627561569214, + 2.2279810905456543, + 2.7078139781951904, + 3.370229721069336, + 3.3430094718933105, + 2.590736150741577, + 3.132143497467041, + 3.7383830547332764, + 2.6325602531433105, + 2.7579457759857178, + 3.5119781494140625, + 2.0551867485046387, + 3.0789873600006104, + 3.3386776447296143, + 4.141490459442139, + 3.0203514099121094, + 2.2210357189178467, + 1.953963279724121, + 1.6369072198867798, + 2.170156240463257, + 2.757321357727051, + 1.932369589805603 + ], + "clf": [] + }, + "n_targets": 40, + "random_top5": 0.047619047619047616 + }, + "LinearResponse+ranking": { + "top1": 0.8, + "top5": 0.95, + "ndcg": 0.9022189725330112, + "func_top5": 1.0, + "func_ndcg": 0.7964563634648597, + "rank": 1.575, + "endpoint_dist": 0.9708886995911599, + "spec_margin": 1.5014912456274032, + "off_dist": 2.4723799377679825, + "clf": null, + "_per": { + "top1": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0 + ], + "top5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "ndcg": [ + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.6309297535708266, + 0.33333333333299997, + 0.30102999566368016, + 0.9999999999989999, + 0.49999999999949996, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.6309297535708266, + 0.6309297535708266, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.6309297535708266, + 0.43067655807296235, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999 + ], + "func_top5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.7778130477875116, + 0.7390855690096315, + 0.6888737584588553, + 0.9999999999996392, + 0.7601875334312905, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999996392, + 0.6988271454578748, + 0.821149491578581, + 0.691397430362036, + 0.8137832979294921, + 0.9223104630420079, + 0.8321157037716506, + 0.9223104630420079, + 0.8011478628189659, + 0.743621924561139, + 0.8131794116030213, + 0.7402521670387343, + 0.9398701604860678, + 0.8814413010323419, + 0.9999999999996392, + 0.9478557351531002, + 0.8877258956395286, + 0.6587286093495057, + 0.6500574077683771, + 0.9467676761260954, + 0.6165377315019861, + 0.9999999999996392, + 0.48270749357113113, + 0.47962493313589827, + 0.8269766751080135, + 0.6478990166307772, + 0.8935530791689612, + 0.5916118404088373, + 0.9334235959386915, + 0.20752333876180207, + 0.873309251116593, + 0.7509714103044139, + 0.8756141175029094 + ], + "rank": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 7, + 9, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 2, + 4, + 1, + 1, + 1 + ], + "endpoint_dist": [ + 0.6907550096511841, + 1.0634472370147705, + 0.5679367184638977, + 1.165585994720459, + 0.6991990804672241, + 0.5411589741706848, + 0.699982225894928, + 0.6999955773353577, + 0.7555252313613892, + 0.8050605654716492, + 1.4271050691604614, + 0.8213156461715698, + 0.7616903781890869, + 0.6038821339607239, + 0.862602710723877, + 2.178295850753784, + 1.4125244617462158, + 1.9672163724899292, + 0.8852179646492004, + 0.7257881760597229, + 1.1350444555282593, + 1.1267881393432617, + 0.8013559579849243, + 1.0217680931091309, + 1.4548118114471436, + 1.2549763917922974, + 0.7250621318817139, + 1.434755802154541, + 1.083292007446289, + 0.6659277081489563, + 0.8551198244094849, + 0.6937877535820007, + 1.2051607370376587, + 1.2373899221420288, + 0.8727580308914185, + 0.6869524121284485, + 0.7910900115966797, + 1.0167807340621948, + 0.83255934715271, + 0.6058813333511353 + ], + "spec_margin": [ + 1.4733346700668335, + 1.1693987846374512, + 1.6650640964508057, + 1.156996726989746, + 1.1377700567245483, + 1.5223705768585205, + 1.5267906188964844, + 2.104612350463867, + 1.323801875114441, + 1.8257782459259033, + 0.5402255058288574, + 1.453631043434143, + 1.3117954730987549, + 1.7336227893829346, + 1.2966454029083252, + 3.671785593032837, + 1.9243266582489014, + 1.1463512182235718, + 1.1231210231781006, + 1.0982592105865479, + 1.1551765203475952, + 1.6141057014465332, + 2.186246395111084, + 0.9928333759307861, + 0.7203879356384277, + 1.6999324560165405, + 1.4218502044677734, + 1.0667059421539307, + 1.8538975715637207, + 1.1988515853881836, + 1.804688572883606, + 2.4775850772857666, + 2.8387041091918945, + 1.6696089506149292, + 1.3214129209518433, + 1.2888331413269043, + 0.7697153091430664, + 1.1673275232315063, + 1.4035670757293701, + 1.2025375366210938 + ], + "off_dist": [ + 2.1640896797180176, + 2.2328460216522217, + 2.2330007553100586, + 2.322582721710205, + 1.8369691371917725, + 2.0635294914245605, + 2.2267727851867676, + 2.80460786819458, + 2.07932710647583, + 2.6308388710021973, + 1.9673305749893188, + 2.274946689605713, + 2.073485851287842, + 2.3375048637390137, + 2.159248113632202, + 5.850081443786621, + 3.336851119995117, + 3.113567590713501, + 2.0083389282226562, + 1.8240474462509155, + 2.2902209758758545, + 2.740893840789795, + 2.987602472305298, + 2.014601469039917, + 2.1751997470855713, + 2.954908847808838, + 2.1469123363494873, + 2.5014617443084717, + 2.9371895790100098, + 1.8647792339324951, + 2.659808397293091, + 3.171372890472412, + 4.043864727020264, + 2.906998872756958, + 2.1941709518432617, + 1.975785493850708, + 1.560805320739746, + 2.184108257293701, + 2.23612642288208, + 1.808418869972229 + ], + "clf": [] + }, + "n_targets": 40, + "random_top5": 0.047619047619047616 + }, + "NearestPerturbationCentroid+ranking": { + "top1": 1.0, + "top5": 1.0, + "ndcg": 0.9999999999989999, + "func_top5": 1.0, + "func_ndcg": 0.8479507485118546, + "rank": 1.0, + "endpoint_dist": 0.42116108611226083, + "spec_margin": 2.4543789327144623, + "off_dist": 2.8755400240421296, + "clf": null, + "_per": { + "top1": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "top5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "ndcg": [ + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999 + ], + "func_top5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.7644049042788869, + 0.7623315121284877, + 0.7691671966722313, + 0.9999999999996392, + 0.7601875334312905, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999996392, + 0.7209064905476281, + 0.9999999999996392, + 0.7122606394323479, + 0.9999999999996392, + 0.9999999999996392, + 0.9999999999996392, + 0.9999999999996392, + 0.8525262713988451, + 0.7490035979724269, + 0.8580544200588494, + 0.7122606394323479, + 0.9223104630420079, + 0.8700859688429049, + 0.9999999999996392, + 0.9999999999996392, + 0.9478557351531002, + 0.6588088387020696, + 0.6655389434274009, + 0.8460101051759085, + 0.6734442887418696, + 0.9999999999996392, + 0.3607790370814292, + 0.7601875334312905, + 0.8935530791689612, + 0.6319672012189719, + 0.8366465219290776, + 0.7923964318852734, + 0.9478557351531002, + 0.6719052786222132, + 0.8887907867756167, + 0.9456973440155002, + 0.9430934427597557 + ], + "rank": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "endpoint_dist": [ + 0.4336645305156708, + 0.4492749273777008, + 0.36839205026626587, + 0.5400750041007996, + 0.4056673049926758, + 0.5053470730781555, + 0.3870347738265991, + 0.46958184242248535, + 0.3555386960506439, + 0.49682140350341797, + 0.3571249842643738, + 0.448895126581192, + 0.416703999042511, + 0.43106698989868164, + 0.44865232706069946, + 0.44056838750839233, + 0.38007888197898865, + 0.38654324412345886, + 0.33925434947013855, + 0.4254252016544342, + 0.7614416480064392, + 0.44820210337638855, + 0.3869432210922241, + 0.4388890266418457, + 0.370621919631958, + 0.49089911580085754, + 0.3214934170246124, + 0.42563191056251526, + 0.3777088224887848, + 0.4537658989429474, + 0.3759283721446991, + 0.34280818700790405, + 0.4167364537715912, + 0.43289661407470703, + 0.37268662452697754, + 0.39969366788864136, + 0.3186255097389221, + 0.38318541646003723, + 0.4199433922767639, + 0.42263102531433105 + ], + "spec_margin": [ + 1.8226473331451416, + 2.429075002670288, + 2.0126900672912598, + 2.4777753353118896, + 1.6256582736968994, + 1.7526733875274658, + 2.1037702560424805, + 2.804387331008911, + 1.832645058631897, + 2.120689868927002, + 2.2210195064544678, + 1.8804901838302612, + 1.9542350769042969, + 1.9912099838256836, + 2.1478829383850098, + 7.4463677406311035, + 3.817469596862793, + 4.136026859283447, + 2.065373182296753, + 1.8025559186935425, + 1.9463722705841064, + 2.922027587890625, + 2.956066131591797, + 2.1518471240997314, + 2.761521577835083, + 3.247483968734741, + 2.3110668659210205, + 2.3323137760162354, + 3.1342692375183105, + 1.6014208793640137, + 2.703058958053589, + 2.9958693981170654, + 3.7247540950775146, + 2.5874547958374023, + 1.8483490943908691, + 1.554269552230835, + 1.318281650543213, + 1.786970853805542, + 2.3373780250549316, + 1.509738564491272 + ], + "off_dist": [ + 2.2563118934631348, + 2.878350019454956, + 2.381082057952881, + 3.017850399017334, + 2.031325578689575, + 2.2580204010009766, + 2.49080491065979, + 3.2739691734313965, + 2.1881837844848633, + 2.61751127243042, + 2.5781445503234863, + 2.329385280609131, + 2.370939016342163, + 2.4222769737243652, + 2.5965352058410645, + 7.886936187744141, + 4.1975483894348145, + 4.5225701332092285, + 2.404627561569214, + 2.2279810905456543, + 2.7078139781951904, + 3.370229721069336, + 3.3430094718933105, + 2.590736150741577, + 3.132143497467041, + 3.7383830547332764, + 2.6325602531433105, + 2.7579457759857178, + 3.5119781494140625, + 2.0551867485046387, + 3.0789873600006104, + 3.3386776447296143, + 4.141490459442139, + 3.0203514099121094, + 2.2210357189178467, + 1.953963279724121, + 1.6369072198867798, + 2.170156240463257, + 2.757321357727051, + 1.932369589805603 + ], + "clf": [] + }, + "n_targets": 40, + "random_top5": 0.047619047619047616 + }, + "EndpointMLP+ranking": { + "top1": 1.0, + "top5": 1.0, + "ndcg": 0.9999999999989999, + "func_top5": 1.0, + "func_ndcg": 0.844723911438565, + "rank": 1.0, + "endpoint_dist": 0.42520923763513563, + "spec_margin": 2.4454854100942613, + "off_dist": 2.8706946551799772, + "clf": null, + "_per": { + "top1": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "top5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "ndcg": [ + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999 + ], + "func_top5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.7644049042788869, + 0.7358622867359436, + 0.7691671966722313, + 0.9999999999996392, + 0.7601875334312905, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999996392, + 0.7209064905476281, + 0.9999999999996392, + 0.7122606394323479, + 0.9999999999996392, + 0.9999999999996392, + 0.9999999999996392, + 0.9999999999996392, + 0.8525262713988451, + 0.7490035979724269, + 0.8580544200588494, + 0.7122606394323479, + 0.9223104630420079, + 0.8700859688429049, + 0.9999999999996392, + 0.9999999999996392, + 0.9478557351531002, + 0.6045061827179307, + 0.6629350421716564, + 0.8460101051759085, + 0.6734442887418696, + 0.9999999999996392, + 0.3607790370814292, + 0.7601875334312905, + 0.8935530791689612, + 0.6351904834926599, + 0.8366465219290776, + 0.7434754493124225, + 0.9478557351531002, + 0.6719052786222132, + 0.8887907867756167, + 0.9456973440155002, + 0.9430934427597557 + ], + "rank": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "endpoint_dist": [ + 0.42910057306289673, + 0.4497443437576294, + 0.3689153492450714, + 0.5424296259880066, + 0.4387088418006897, + 0.4989120066165924, + 0.38678574562072754, + 0.44920316338539124, + 0.37661030888557434, + 0.48695677518844604, + 0.3633248805999756, + 0.4482707679271698, + 0.44536110758781433, + 0.45165324211120605, + 0.456876277923584, + 0.4695214033126831, + 0.38211068511009216, + 0.3952172100543976, + 0.34323543310165405, + 0.4283789098262787, + 0.7605032920837402, + 0.4499318599700928, + 0.4084526002407074, + 0.43640345335006714, + 0.3723512589931488, + 0.4967805743217468, + 0.3198273777961731, + 0.43172505497932434, + 0.3519124686717987, + 0.4496127665042877, + 0.38622480630874634, + 0.33023756742477417, + 0.4086756408214569, + 0.44142019748687744, + 0.3617817163467407, + 0.4287257194519043, + 0.3391049802303314, + 0.37814775109291077, + 0.41947412490844727, + 0.4257596433162689 + ], + "spec_margin": [ + 1.8297321796417236, + 2.4234132766723633, + 2.0120315551757812, + 2.474533796310425, + 1.5998640060424805, + 1.7592231035232544, + 2.1033191680908203, + 2.8135390281677246, + 1.8096857070922852, + 2.1264402866363525, + 2.2138192653656006, + 1.861198902130127, + 1.9200901985168457, + 1.95857834815979, + 2.123638391494751, + 7.403108596801758, + 3.7975454330444336, + 4.105343341827393, + 2.063859701156616, + 1.7980142831802368, + 1.9481453895568848, + 2.9155890941619873, + 2.932025671005249, + 2.1543331146240234, + 2.7645161151885986, + 3.2405927181243896, + 2.310844659805298, + 2.3246302604675293, + 3.131648063659668, + 1.6016348600387573, + 2.693023920059204, + 2.9935696125030518, + 3.7314066886901855, + 2.577932357788086, + 1.844446063041687, + 1.5249234437942505, + 1.2980587482452393, + 1.7848142385482788, + 2.3416037559509277, + 1.5086990594863892 + ], + "off_dist": [ + 2.2588326930999756, + 2.873157501220703, + 2.3809468746185303, + 3.016963481903076, + 2.0385727882385254, + 2.2581350803375244, + 2.490104913711548, + 3.262742280960083, + 2.186295986175537, + 2.6133971214294434, + 2.577144145965576, + 2.309469699859619, + 2.3654513359069824, + 2.410231590270996, + 2.580514669418335, + 7.8726301193237305, + 4.179656028747559, + 4.500560760498047, + 2.407095193862915, + 2.226393222808838, + 2.708648681640625, + 3.36552095413208, + 3.3404781818389893, + 2.5907366275787354, + 3.1368672847747803, + 3.7373733520507812, + 2.630671977996826, + 2.7563552856445312, + 3.483560562133789, + 2.0512475967407227, + 3.0792486667633057, + 3.3238072395324707, + 4.140082359313965, + 3.019352674484253, + 2.2062277793884277, + 1.9536491632461548, + 1.637163758277893, + 2.162961959838867, + 2.761077880859375, + 1.9344587326049805 + ], + "clf": [] + }, + "n_targets": 40, + "random_top5": 0.047619047619047616 + }, + "Random+ranking": { + "top1": 0.025, + "top5": 0.025, + "ndcg": 0.0484465759108753, + "func_top5": 0.625, + "func_ndcg": 0.20469424490855914, + "rank": 57.925, + "endpoint_dist": 3.470254364609718, + "spec_margin": -0.21521338820457458, + "off_dist": 3.2550409764051436, + "clf": null, + "_per": { + "top1": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0 + ], + "top5": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0 + ], + "ndcg": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.33333333333299997, + 0.2890648263175988, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.3154648767854133, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.9999999999989999 + ], + "func_top5": [ + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.43262589581815525, + 0.37924165572138796, + 0.3718921699781131, + 0.05430265598413898, + 0.0, + 0.0, + 0.0, + 0.1678842962279888, + 0.3144297531670358, + 0.3225665727174216, + 0.11381311447976694, + 0.23214033921114718, + 0.27853764988081753, + 0.3198776724744765, + 0.4919092042576051, + 0.11640030782969744, + 0.0, + 0.0, + 0.25017371018387746, + 0.0, + 0.1298338018041706, + 0.4378169904238303, + 0.19408984478732896, + 0.4615310696772767, + 0.05430265598413898, + 0.05430265598413898, + 0.0, + 0.052144264846539085, + 0.33384167555429484, + 0.20246886363046793, + 0.0, + 0.23819899697649688, + 0.2645101253542494, + 0.0, + 0.1841364577883096, + 0.18889875018165395, + 0.14194557994078985, + 0.32749083505095544, + 0.527340302734206, + 0.5491219276918895 + ], + "rank": [ + 14, + 74, + 102, + 28, + 89, + 23, + 94, + 92, + 56, + 30, + 43, + 93, + 48, + 7, + 10, + 14, + 78, + 59, + 59, + 97, + 67, + 56, + 57, + 74, + 63, + 67, + 105, + 28, + 74, + 38, + 102, + 8, + 68, + 72, + 87, + 51, + 66, + 94, + 29, + 1 + ], + "endpoint_dist": [ + 2.2824044227600098, + 4.309011936187744, + 3.309451103210449, + 3.564979076385498, + 1.453060507774353, + 2.8975343704223633, + 1.6756479740142822, + 2.8101394176483154, + 3.0139412879943848, + 3.0206949710845947, + 5.2664995193481445, + 2.2033324241638184, + 2.293858766555786, + 2.880345344543457, + 2.1053731441497803, + 8.31590461730957, + 6.106352806091309, + 4.284556865692139, + 2.4269845485687256, + 2.013439178466797, + 3.7241005897521973, + 2.720334053039551, + 4.638664722442627, + 2.371800422668457, + 4.18781852722168, + 5.791450500488281, + 5.960268497467041, + 4.852285861968994, + 5.4412407875061035, + 2.308344602584839, + 2.7184574604034424, + 2.7296082973480225, + 2.9228410720825195, + 4.18679141998291, + 2.3668174743652344, + 1.5269440412521362, + 1.1631391048431396, + 5.09120512008667, + 6.774564743041992, + 1.0999850034713745 + ], + "spec_margin": [ + -0.20399761199951172, + 0.09865903854370117, + 0.7027626037597656, + -0.2298269271850586, + 0.6164559125900269, + 0.7414908409118652, + 0.9587249755859375, + 0.38802433013916016, + 0.4304380416870117, + 0.06131625175476074, + -1.3636457920074463, + 0.02685999870300293, + 0.9651339054107666, + -0.0017824172973632812, + 0.412534236907959, + -5.367891311645508, + -2.511678457260132, + -1.9479377269744873, + -0.026017427444458008, + -0.077117919921875, + -0.0017910003662109375, + 1.7640347480773926, + -1.2810420989990234, + 0.16104459762573242, + -1.0742332935333252, + -1.1297025680541992, + -0.7748074531555176, + -0.8083209991455078, + -1.7950336933135986, + 0.09963798522949219, + 0.4017605781555176, + -0.3908803462982178, + -0.15664148330688477, + -0.5637521743774414, + 0.6465599536895752, + 0.42621946334838867, + 0.5508947372436523, + 0.8817577362060547, + -0.12642383575439453, + 0.8896790742874146 + ], + "off_dist": [ + 2.078406810760498, + 4.407670974731445, + 4.012213706970215, + 3.3351521492004395, + 2.06951642036438, + 3.6390252113342285, + 2.6343729496002197, + 3.1981637477874756, + 3.4443793296813965, + 3.0820112228393555, + 3.9028537273406982, + 2.2301924228668213, + 3.2589926719665527, + 2.8785629272460938, + 2.5179073810577393, + 2.9480133056640625, + 3.5946743488311768, + 2.3366191387176514, + 2.4009671211242676, + 1.9363212585449219, + 3.7223095893859863, + 4.484368801116943, + 3.3576226234436035, + 2.5328450202941895, + 3.1135852336883545, + 4.661747932434082, + 5.185461044311523, + 4.043964862823486, + 3.646207094192505, + 2.407982587814331, + 3.12021803855896, + 2.3387279510498047, + 2.7661995887756348, + 3.6230392456054688, + 3.0133774280548096, + 1.953163504600525, + 1.714033842086792, + 5.972962856292725, + 6.648140907287598, + 1.989664077758789 + ], + "clf": [] + }, + "n_targets": 40, + "random_top5": 0.047619047619047616 + } + } +} \ No newline at end of file diff --git a/experiments/results/norman_nom_improved.json b/experiments/results/norman_nom_improved.json new file mode 100644 index 0000000000000000000000000000000000000000..a6be64767fc324e4afb69cc94c8cd58e580d2efe --- /dev/null +++ b/experiments/results/norman_nom_improved.json @@ -0,0 +1,295 @@ +{ + "dataset": "norman", + "split": "perturbation", + "reward": "cosine", + "n_candidates": 105, + "n_targets": 22, + "random_top5": 0.047619047619047616, + "methods": { + "PIVOT-ranking": { + "top1": 0.18181818181818182, + "top5": 0.22727272727272727, + "ndcg": 0.237863170222463, + "func_top5": 0.8636363636363636, + "func_ndcg": 0.6379857109126424, + "top5_ci": [ + 0.22727272727272727, + 0.045454545454545456, + 0.4090909090909091 + ], + "_top5_per": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 1.0, + 0.0, + 1.0, + 0.0, + 1.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "med_rank": 26.0 + }, + "PIVOT-guidance(random-init)": { + "top1": 0.13636363636363635, + "top5": 0.22727272727272727, + "ndcg": 0.20036677015736315, + "func_top5": 0.8181818181818182, + "func_ndcg": 0.5717396066885763, + "top5_ci": [ + 0.22727272727272727, + 0.045454545454545456, + 0.4090909090909091 + ], + "_top5_per": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 1.0, + 0.0, + 1.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0 + ], + "med_rank": 27.0 + }, + "EndpointMLP+ranking": { + "top1": 0.045454545454545456, + "top5": 0.13636363636363635, + "ndcg": 0.13489139848118523, + "func_top5": 0.9545454545454546, + "func_ndcg": 0.5888636133223321, + "top5_ci": [ + 0.13636363636363635, + 0.0, + 0.2727272727272727 + ], + "_top5_per": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "med_rank": 74.5 + }, + "LinearResponse+ranking": { + "top1": 0.045454545454545456, + "top5": 0.13636363636363635, + "ndcg": 0.1028117957791206, + "func_top5": 0.8181818181818182, + "func_ndcg": 0.5703934469254858, + "top5_ci": [ + 0.13636363636363635, + 0.0, + 0.2727272727272727 + ], + "_top5_per": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "med_rank": 53.0 + }, + "Additive+ranking": { + "top1": 0.0, + "top5": 0.0, + "ndcg": 0.0, + "func_top5": 0.9545454545454546, + "func_ndcg": 0.5342699870208798, + "top5_ci": [ + 0.0, + 0.0, + 0.0 + ], + "_top5_per": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "med_rank": 40.5 + }, + "NearestPerturbationCentroid+ranking": { + "top1": 0.18181818181818182, + "top5": 0.18181818181818182, + "ndcg": 0.181818181818, + "func_top5": 0.9090909090909091, + "func_ndcg": 0.5876837637320901, + "top5_ci": [ + 0.18181818181818182, + 0.045454545454545456, + 0.36363636363636365 + ], + "_top5_per": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 1.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "med_rank": 35.5 + }, + "Random+ranking": { + "top1": 0.0, + "top5": 0.045454545454545456, + "ndcg": 0.030723528797806977, + "func_top5": 0.7727272727272727, + "func_ndcg": 0.24888537092571922, + "top5_ci": [ + 0.045454545454545456, + 0.0, + 0.13636363636363635 + ], + "_top5_per": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "med_rank": 57.5 + } + }, + "significance": { + "PIVOT-ranking_vs_PIVOT-guidance(random-init)": { + "p": 1.0, + "mean_diff_top5": 0.0 + }, + "PIVOT-ranking_vs_EndpointMLP+ranking": { + "p": 0.3268365817091454, + "mean_diff_top5": 0.09090909090909091 + }, + "PIVOT-ranking_vs_LinearResponse+ranking": { + "p": 0.12643678160919541, + "mean_diff_top5": 0.09090909090909091 + }, + "PIVOT-ranking_vs_Additive+ranking": { + "p": 0.010994502748625687, + "mean_diff_top5": 0.22727272727272727 + }, + "PIVOT-ranking_vs_NearestPerturbationCentroid+ranking": { + "p": 0.25287356321839083, + "mean_diff_top5": 0.045454545454545456 + }, + "PIVOT-ranking_vs_Random+ranking": { + "p": 0.03298350824587706, + "mean_diff_top5": 0.18181818181818182 + } + } +} \ No newline at end of file diff --git a/experiments/results/norman_nom_perturbation.json b/experiments/results/norman_nom_perturbation.json new file mode 100644 index 0000000000000000000000000000000000000000..c617ef9eb7a640696fa67baea1600d9c96837b29 --- /dev/null +++ b/experiments/results/norman_nom_perturbation.json @@ -0,0 +1,1639 @@ +{ + "split": "perturbation", + "reward": "centroid", + "n_candidates": 105, + "n_targets": 22, + "methods": { + "PIVOT-ranking": { + "top1": 0.0, + "top5": 0.13636363636363635, + "ndcg": 0.08727248090397387, + "func_top5": 0.9090909090909091, + "func_ndcg": 0.4865406670801178, + "rank": 35.40909090909091, + "endpoint_dist": 1.068045285615054, + "spec_margin": 1.3363434076309204, + "off_dist": 2.4043886986645786, + "clf": null, + "_per": { + "top1": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "top5": [ + 1.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "ndcg": [ + 0.49999999999949996, + 0.0, + 0.0, + 0.49999999999949996, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.2890648263175988, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.6309297535708266, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "func_top5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.733275093386013, + 0.37033766516698385, + 0.8196104814589247, + 0.38009376671564526, + 0.0, + 0.4387000482749779, + 0.7553544384757664, + 0.4749645636517497, + 0.558712758374731, + 0.8196104814589247, + 0.4960354077271919, + 0.15032459878392884, + 0.4567687772839323, + 0.8196104814589247, + 0.5059324046044343, + 0.30215600509022716, + 0.7910845718217951, + 0.0, + 0.6430803341156557, + 0.40257647075284236, + 0.34696627888496423, + 0.4387000482749779 + ], + "rank": [ + 3, + 35, + 17, + 3, + 25, + 74, + 84, + 26, + 23, + 19, + 10, + 75, + 36, + 63, + 31, + 35, + 2, + 48, + 23, + 52, + 48, + 47 + ], + "endpoint_dist": [ + 1.9384753704071045, + 0.7429381012916565, + 1.1510169506072998, + 0.7578966021537781, + 0.6557507514953613, + 0.9810452461242676, + 0.819040060043335, + 1.5743571519851685, + 1.7966794967651367, + 2.0639142990112305, + 1.1475152969360352, + 0.840554416179657, + 0.8537219762802124, + 0.8637474179267883, + 1.4352035522460938, + 0.8421792984008789, + 1.0563677549362183, + 0.7560780048370361, + 1.2075574398040771, + 0.6115195155143738, + 0.691246747970581, + 0.7101908326148987 + ], + "spec_margin": [ + -0.18687379360198975, + 1.2044005393981934, + 2.3071529865264893, + 1.7771611213684082, + 1.122977375984192, + 1.3516638278961182, + 1.9459350109100342, + 0.6711388826370239, + 0.42350029945373535, + 0.5177159309387207, + 1.2125356197357178, + 1.326828956604004, + 0.9825109243392944, + 2.5846638679504395, + 1.4229609966278076, + 0.9427473545074463, + 3.5186824798583984, + 1.3512330055236816, + 0.6580355167388916, + 1.3769090175628662, + 1.4731967449188232, + 1.4144783020019531 + ], + "off_dist": [ + 1.7516015768051147, + 1.9473387002944946, + 3.458169937133789, + 2.535057783126831, + 1.7787281274795532, + 2.3327090740203857, + 2.764975070953369, + 2.2454960346221924, + 2.220179796218872, + 2.581630229949951, + 2.360050916671753, + 2.1673834323883057, + 1.8362329006195068, + 3.448411226272583, + 2.8581645488739014, + 1.7849266529083252, + 4.575050354003906, + 2.1073110103607178, + 1.8655929565429688, + 1.9884284734725952, + 2.1644434928894043, + 2.124669075012207 + ], + "clf": [] + }, + "n_targets": 22, + "random_top5": 0.047619047619047616 + }, + "PIVOT-guidance": { + "top1": 0.0, + "top5": 0.13636363636363635, + "ndcg": 0.0741331706168103, + "func_top5": 0.9090909090909091, + "func_ndcg": 0.40010813422906144, + "rank": 37.72727272727273, + "endpoint_dist": 1.0795677466826006, + "spec_margin": 1.2815920818935742, + "off_dist": 2.361159833994779, + "clf": null, + "_per": { + "top1": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "top5": [ + 1.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "ndcg": [ + 0.49999999999949996, + 0.0, + 0.0, + 0.49999999999949996, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.6309297535708266, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "func_top5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.6220658801619905, + 0.3259363302398262, + 0.8196104814589247, + 0.38009376671564526, + 0.0, + 0.34634689786702066, + 0.4620869292484704, + 0.24464556152387298, + 0.5791066180961114, + 0.5920009604052048, + 0.34036846945423477, + 0.2714871990069707, + 0.46683251373600076, + 0.6441452252517439, + 0.1835973061229298, + 0.3181934003204448, + 0.5499613274410384, + 0.0, + 0.4445272318044105, + 0.5189767785825399, + 0.3047754010374445, + 0.3876206745645269 + ], + "rank": [ + 3, + 35, + 20, + 3, + 28, + 75, + 84, + 32, + 28, + 26, + 14, + 75, + 38, + 64, + 37, + 37, + 2, + 49, + 29, + 52, + 50, + 49 + ], + "endpoint_dist": [ + 1.9384753704071045, + 0.7429381012916565, + 1.1510169506072998, + 0.7578966021537781, + 0.6557507514953613, + 0.9810452461242676, + 0.819040060043335, + 1.5743571519851685, + 1.8661444187164307, + 2.0639142990112305, + 1.1475152969360352, + 0.840554416179657, + 0.8537219762802124, + 0.8637474179267883, + 1.6192327737808228, + 0.8421792984008789, + 1.0563677549362183, + 0.7560780048370361, + 1.2075574398040771, + 0.6115195155143738, + 0.691246747970581, + 0.7101908326148987 + ], + "spec_margin": [ + -0.18687379360198975, + 1.2044005393981934, + 2.3071529865264893, + 1.7771611213684082, + 1.122977375984192, + 1.3516638278961182, + 1.9459350109100342, + 0.6711388826370239, + 0.15226531028747559, + 0.5177159309387207, + 1.2125356197357178, + 1.326828956604004, + 0.9825109243392944, + 2.5846638679504395, + 0.48966681957244873, + 0.9427473545074463, + 3.5186824798583984, + 1.3512330055236816, + 0.6580355167388916, + 1.3769090175628662, + 1.4731967449188232, + 1.4144783020019531 + ], + "off_dist": [ + 1.7516015768051147, + 1.9473387002944946, + 3.458169937133789, + 2.535057783126831, + 1.7787281274795532, + 2.3327090740203857, + 2.764975070953369, + 2.2454960346221924, + 2.0184097290039062, + 2.581630229949951, + 2.360050916671753, + 2.1673834323883057, + 1.8362329006195068, + 3.448411226272583, + 2.1088995933532715, + 1.7849266529083252, + 4.575050354003906, + 2.1073110103607178, + 1.8655929565429688, + 1.9884284734725952, + 2.1644434928894043, + 2.124669075012207 + ], + "clf": [] + }, + "n_targets": 22, + "random_top5": 0.047619047619047616 + }, + "Additive+ranking": { + "top1": 0.045454545454545456, + "top5": 0.045454545454545456, + "ndcg": 0.0454545454545, + "func_top5": 0.8636363636363636, + "func_ndcg": 0.4675706692782015, + "rank": 31.227272727272727, + "endpoint_dist": 1.154505336826498, + "spec_margin": 1.2376486469398846, + "off_dist": 2.3921539837663826, + "clf": null, + "_per": { + "top1": [ + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "top5": [ + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "ndcg": [ + 0.9999999999989999, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "func_top5": [ + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.5314987088010797, + 0.5055599925138808, + 0.8196104814589247, + 0.0, + 0.0, + 0.4620869292484704, + 0.8196104814589247, + 0.5263429722316287, + 0.5765194247461809, + 0.8196104814589247, + 0.15445080225351568, + 0.4524003745215921, + 0.5806456282157678, + 0.49305477020115496, + 0.3348403207942303, + 0.4988298877885495, + 0.6207583442782515, + 0.0, + 0.7553544384757664, + 0.5059324046044343, + 0.5477508703614005, + 0.2816974107077558 + ], + "rank": [ + 1, + 40, + 31, + 58, + 31, + 47, + 29, + 29, + 14, + 25, + 11, + 37, + 22, + 19, + 22, + 22, + 45, + 52, + 38, + 44, + 34, + 36 + ], + "endpoint_dist": [ + 1.9496577978134155, + 0.5574938654899597, + 1.2164744138717651, + 0.7687604427337646, + 0.7320241332054138, + 0.9937659502029419, + 0.7439819574356079, + 1.4131525754928589, + 1.8808938264846802, + 1.99337899684906, + 1.1153759956359863, + 0.9441820979118347, + 0.8986158967018127, + 1.046272873878479, + 1.3573639392852783, + 0.8964778184890747, + 2.548943281173706, + 0.8479581475257874, + 1.0793635845184326, + 0.8028191328048706, + 0.87811279296875, + 0.7340478897094727 + ], + "spec_margin": [ + -0.06023609638214111, + 1.4850876331329346, + 2.124490261077881, + 1.9016368389129639, + 1.1379446983337402, + 1.4654606580734253, + 1.7133015394210815, + 0.9593864679336548, + 0.3259471654891968, + 0.7593246698379517, + 1.059178352355957, + 1.2604901790618896, + 0.9792718291282654, + 2.7629809379577637, + 1.251596450805664, + 1.0095385313034058, + 0.5528774261474609, + 1.3290481567382812, + 1.105086326599121, + 1.2595843076705933, + 1.3820161819458008, + 1.4642577171325684 + ], + "off_dist": [ + 1.8894217014312744, + 2.042581558227539, + 3.3409647941589355, + 2.6703972816467285, + 1.8699687719345093, + 2.459226608276367, + 2.4572834968566895, + 2.3725390434265137, + 2.206840991973877, + 2.7527036666870117, + 2.1745543479919434, + 2.204672336578369, + 1.8778877258300781, + 3.809253692626953, + 2.6089603900909424, + 1.9060163497924805, + 3.101820707321167, + 2.177006244659424, + 2.1844499111175537, + 2.062403440475464, + 2.260128974914551, + 2.198305606842041 + ], + "clf": [] + }, + "n_targets": 22, + "random_top5": 0.047619047619047616 + }, + "LinearResponse+ranking": { + "top1": 0.09090909090909091, + "top5": 0.13636363636363635, + "ndcg": 0.12667653387175581, + "func_top5": 0.8636363636363636, + "func_ndcg": 0.465389360745805, + "rank": 24.5, + "endpoint_dist": 1.0359658057039434, + "spec_margin": 1.2246569259600206, + "off_dist": 2.260622728954662, + "clf": null, + "_per": { + "top1": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "top5": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "ndcg": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.9999999999989999, + 0.35620718710766597, + 0.0, + 0.0, + 0.43067655807296235, + 0.0, + 0.0, + 0.9999999999989999, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "func_top5": [ + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.33860396794763925, + 0.3796350998974945, + 0.8196104814589247, + 0.0, + 0.0, + 0.3843973922908389, + 0.7594806419453533, + 0.5727402829012992, + 0.6976820249692228, + 0.9999999999996392, + 0.3926497992300126, + 0.18205829600327347, + 0.3214166825941328, + 0.7130833312756829, + 0.7498262898157618, + 0.34634689786702066, + 0.9478557351531002, + 0.0, + 0.6976820249692228, + 0.26128684308056127, + 0.3023179750304166, + 0.3718921699781131 + ], + "rank": [ + 32, + 16, + 43, + 11, + 31, + 68, + 16, + 23, + 20, + 1, + 6, + 35, + 19, + 4, + 41, + 21, + 1, + 37, + 25, + 38, + 25, + 26 + ], + "endpoint_dist": [ + 1.8301076889038086, + 0.5907474160194397, + 1.1353018283843994, + 0.9802330136299133, + 0.7048063278198242, + 1.0184342861175537, + 0.7584463357925415, + 1.399523377418518, + 1.6685357093811035, + 1.4839903116226196, + 1.0153692960739136, + 0.670589029788971, + 0.7878819704055786, + 0.9112664461135864, + 1.1367274522781372, + 0.9118342995643616, + 1.3724935054779053, + 0.9561153650283813, + 0.9823485016822815, + 0.8708151578903198, + 0.8896368741989136, + 0.7160435318946838 + ], + "spec_margin": [ + 0.11864185333251953, + 1.3417978286743164, + 1.9002373218536377, + 1.2397401332855225, + 1.1664302349090576, + 1.25345778465271, + 1.6654146909713745, + 0.9994257688522339, + 0.6466996669769287, + 1.0209730863571167, + 1.0659247636795044, + 1.268491506576538, + 1.0920313596725464, + 2.0039072036743164, + 1.3454638719558716, + 0.9079654812812805, + 1.8100285530090332, + 1.0981038808822632, + 1.0993499755859375, + 1.188269019126892, + 1.3159099817276, + 1.394188404083252 + ], + "off_dist": [ + 1.9487495422363281, + 1.9325453042984009, + 3.035539150238037, + 2.219973087310791, + 1.8712365627288818, + 2.2718920707702637, + 2.423861026763916, + 2.398949146270752, + 2.3152353763580322, + 2.5049633979797363, + 2.081294059753418, + 1.9390804767608643, + 1.879913330078125, + 2.9151737689971924, + 2.482191324234009, + 1.819799780845642, + 3.1825220584869385, + 2.0542192459106445, + 2.081698417663574, + 2.059084177017212, + 2.2055468559265137, + 2.110231876373291 + ], + "clf": [] + }, + "n_targets": 22, + "random_top5": 0.047619047619047616 + }, + "NearestPerturbationCentroid+ranking": { + "top1": 0.045454545454545456, + "top5": 0.09090909090909091, + "ndcg": 0.0741331706168103, + "func_top5": 0.8636363636363636, + "func_ndcg": 0.5080102766031381, + "rank": 46.5, + "endpoint_dist": 1.1265862611207096, + "spec_margin": 1.356340909546072, + "off_dist": 2.4829271652481775, + "clf": null, + "_per": { + "top1": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "top5": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "ndcg": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.6309297535708266, + 0.0, + 0.0, + 0.9999999999989999, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "func_top5": [ + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.2718596110975242, + 0.5055599925138808, + 0.8196104814589247, + 0.0, + 0.0, + 0.4620869292484704, + 0.8196104814589247, + 0.5806456282157678, + 0.7032101736292273, + 0.8196104814589247, + 0.2565245506872169, + 0.4524003745215921, + 0.5806456282157678, + 0.7571924834648507, + 0.7419209445012933, + 0.4988298877885495, + 0.8157833128587658, + 0.0, + 0.7553544384757664, + 0.5059324046044343, + 0.5477508703614005, + 0.2816974107077558 + ], + "rank": [ + 24, + 55, + 30, + 75, + 50, + 100, + 93, + 53, + 27, + 17, + 35, + 77, + 50, + 2, + 18, + 50, + 1, + 47, + 47, + 79, + 47, + 46 + ], + "endpoint_dist": [ + 2.040769577026367, + 0.5574938654899597, + 1.2164744138717651, + 0.7687604427337646, + 0.7320241332054138, + 0.9937659502029419, + 0.7439819574356079, + 1.4131525754928589, + 1.8808938264846802, + 1.99337899684906, + 1.1153759956359863, + 0.9441820979118347, + 0.8986158967018127, + 1.046272873878479, + 1.3573639392852783, + 0.8964778184890747, + 1.8436118364334106, + 0.8479581475257874, + 1.0793635845184326, + 0.8028191328048706, + 0.87811279296875, + 0.7340478897094727 + ], + "spec_margin": [ + -0.03342318534851074, + 1.4850876331329346, + 2.124490261077881, + 1.9016368389129639, + 1.1379446983337402, + 1.4654606580734253, + 1.7133015394210815, + 0.9593864679336548, + 0.3259471654891968, + 0.7593246698379517, + 1.059178352355957, + 1.2604901790618896, + 0.9792718291282654, + 2.7629809379577637, + 1.251596450805664, + 1.0095385313034058, + 3.137294292449951, + 1.3290481567382812, + 1.105086326599121, + 1.2595843076705933, + 1.3820161819458008, + 1.4642577171325684 + ], + "off_dist": [ + 2.0073463916778564, + 2.042581558227539, + 3.3409647941589355, + 2.6703972816467285, + 1.8699687719345093, + 2.459226608276367, + 2.4572834968566895, + 2.3725390434265137, + 2.206840991973877, + 2.7527036666870117, + 2.1745543479919434, + 2.204672336578369, + 1.8778877258300781, + 3.809253692626953, + 2.6089603900909424, + 1.9060163497924805, + 4.980906009674072, + 2.177006244659424, + 2.1844499111175537, + 2.062403440475464, + 2.260128974914551, + 2.198305606842041 + ], + "clf": [] + }, + "n_targets": 22, + "random_top5": 0.047619047619047616 + }, + "EndpointMLP+ranking": { + "top1": 0.13636363636363635, + "top5": 0.18181818181818182, + "ndcg": 0.2107243250360866, + "func_top5": 0.9090909090909091, + "func_ndcg": 0.5992360271056622, + "rank": 23.40909090909091, + "endpoint_dist": 1.027024125510996, + "spec_margin": 1.4001903235912323, + "off_dist": 2.4272144491022285, + "clf": null, + "_per": { + "top1": [ + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "top5": [ + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "ndcg": [ + 0.0, + 0.35620718710766597, + 0.0, + 0.9999999999989999, + 0.3154648767854133, + 0.0, + 0.33333333333299997, + 0.0, + 0.0, + 0.9999999999989999, + 0.0, + 0.0, + 0.0, + 0.6309297535708266, + 0.0, + 0.0, + 0.9999999999989999, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "func_top5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.48576880590462984, + 0.6917444919867412, + 0.8196104814589247, + 0.7601875334312905, + 0.3154648767854133, + 0.4620869292484704, + 0.8797403209724962, + 0.6407754677293394, + 0.7653078254747857, + 0.9999999999996392, + 0.27853764988081753, + 0.33392190490685864, + 0.5263429722316287, + 0.8636394042955287, + 0.7498262898157618, + 0.4445272318044105, + 0.8266931349300584, + 0.0, + 0.7594806419453533, + 0.618206508964545, + 0.55770425736042, + 0.40362586719745763 + ], + "rank": [ + 55, + 6, + 64, + 1, + 8, + 58, + 7, + 31, + 24, + 1, + 24, + 29, + 21, + 2, + 51, + 18, + 1, + 19, + 34, + 24, + 18, + 19 + ], + "endpoint_dist": [ + 1.9789191484451294, + 0.5568957924842834, + 1.1287992000579834, + 0.7257286906242371, + 0.7324892282485962, + 0.9938557147979736, + 0.7250792980194092, + 1.4069081544876099, + 1.8390849828720093, + 0.8711382150650024, + 1.1161667108535767, + 0.9455131888389587, + 0.8955139517784119, + 1.0437345504760742, + 1.3547587394714355, + 0.8966653943061829, + 1.0293209552764893, + 0.8490437269210815, + 1.0827422142028809, + 0.8027507066726685, + 0.8848638534545898, + 0.7345583438873291 + ], + "spec_margin": [ + 0.19798123836517334, + 1.48545503616333, + 2.4005796909332275, + 1.6699566841125488, + 1.137984037399292, + 1.465325117111206, + 1.727329969406128, + 0.9635328054428101, + 0.24142420291900635, + 2.109982967376709, + 1.0585142374038696, + 1.2584810256958008, + 0.9801564812660217, + 2.764212131500244, + 1.2506382465362549, + 1.0092215538024902, + 2.556349992752075, + 1.3282328844070435, + 1.1001033782958984, + 1.2599939107894897, + 1.3763890266418457, + 1.4623425006866455 + ], + "off_dist": [ + 2.1769003868103027, + 2.0423507690429688, + 3.529378890991211, + 2.3956854343414307, + 1.8704732656478882, + 2.4591808319091797, + 2.452409267425537, + 2.37044095993042, + 2.0805091857910156, + 2.981121063232422, + 2.1746809482574463, + 2.2039942741394043, + 1.8756704330444336, + 3.8079466819763184, + 2.6053969860076904, + 1.9058870077133179, + 3.5856709480285645, + 2.177276611328125, + 2.1828455924987793, + 2.062744617462158, + 2.2612528800964355, + 2.1969008445739746 + ], + "clf": [] + }, + "n_targets": 22, + "random_top5": 0.047619047619047616 + }, + "Random+ranking": { + "top1": 0.0, + "top5": 0.0, + "ndcg": 0.0, + "func_top5": 0.6818181818181818, + "func_ndcg": 0.26643206726700686, + "rank": 57.54545454545455, + "endpoint_dist": 2.9179966937411916, + "spec_margin": 0.07963230935010043, + "off_dist": 2.997629003091292, + "clf": null, + "_per": { + "top1": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "top5": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "ndcg": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "func_top5": [ + 1.0, + 0.0, + 1.0, + 0.0, + 0.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 0.0, + 1.0 + ], + "func_ndcg": [ + 0.4749645636517497, + 0.06425604298315837, + 0.6082531219655256, + 0.0, + 0.0, + 0.23253378338725372, + 0.12116260022304186, + 0.5791066180961114, + 0.4809429920645356, + 0.41382622312386713, + 0.13459609419751495, + 0.1444974152544963, + 0.4985662109553454, + 0.17070296381383643, + 0.18682058839661783, + 0.4131337440783325, + 0.4387000482749779, + 0.0, + 0.06978419164316285, + 0.15997895091352016, + 0.10644692083067808, + 0.5632324060204243 + ], + "rank": [ + 46, + 70, + 78, + 77, + 49, + 56, + 27, + 99, + 69, + 25, + 19, + 52, + 48, + 104, + 25, + 102, + 95, + 86, + 42, + 17, + 35, + 45 + ], + "endpoint_dist": [ + 3.9145050048828125, + 2.1377127170562744, + 3.022123336791992, + 2.1810457706451416, + 3.4453420639038086, + 2.9303030967712402, + 2.341578483581543, + 2.6385629177093506, + 3.595100164413452, + 3.311495065689087, + 2.3130178451538086, + 1.4427813291549683, + 2.7334203720092773, + 4.076140880584717, + 2.273077964782715, + 3.58622670173645, + 4.741064548492432, + 2.7654285430908203, + 5.706614971160889, + 1.9952198266983032, + 1.2292784452438354, + 1.815887212753296 + ], + "spec_margin": [ + -0.2119739055633545, + 0.597123384475708, + -0.8162951469421387, + 0.6589181423187256, + -0.03424239158630371, + -0.1356971263885498, + 0.1988682746887207, + 0.0703420639038086, + -0.07148480415344238, + -1.5779609680175781, + 0.6732721328735352, + 0.686278223991394, + 0.288144588470459, + -0.2951502799987793, + 1.7428030967712402, + 0.08529400825500488, + -2.770522356033325, + 0.42055368423461914, + -0.3490447998046875, + 0.4473620653152466, + 1.1670209169387817, + 0.978302001953125 + ], + "off_dist": [ + 3.702531099319458, + 2.7348361015319824, + 2.2058281898498535, + 2.839963912963867, + 3.411099672317505, + 2.7946059703826904, + 2.5404467582702637, + 2.708904981613159, + 3.5236153602600098, + 1.7335340976715088, + 2.9862899780273438, + 2.1290595531463623, + 3.0215649604797363, + 3.7809906005859375, + 4.015881061553955, + 3.671520709991455, + 1.9705421924591064, + 3.1859822273254395, + 5.357570171356201, + 2.44258189201355, + 2.396299362182617, + 2.794189214706421 + ], + "clf": [] + }, + "n_targets": 22, + "random_top5": 0.047619047619047616 + } + } +} \ No newline at end of file diff --git a/experiments/results/norman_timing_scaling.json b/experiments/results/norman_timing_scaling.json new file mode 100644 index 0000000000000000000000000000000000000000..71e82e040b77254fac40c5ac63d1324accba4552 --- /dev/null +++ b/experiments/results/norman_timing_scaling.json @@ -0,0 +1,89 @@ +{ + "combo_guidance": { + "top1": 0.5384615384615384, + "top5": 0.7307692307692307, + "ndcg": 0.6529941534069046, + "endpoint_dist": 1.2043300362733693 + }, + "search_timing": { + "ranking_only": { + "sec_per_target": 0.01, + "queries": 105, + "top5": 0.13636363636363635, + "ndcg": 0.08727248090397387, + "endpoint_dist": 1.068045285615054 + }, + "random_opt": { + "sec_per_target": 0.058, + "queries": 25, + "top5": 0.13636363636363635, + "ndcg": 0.11048529809413464, + "endpoint_dist": 1.6047562198205427 + }, + "mean_top_init": { + "sec_per_target": 0.058, + "queries": 35, + "top5": 0.13636363636363635, + "ndcg": 0.15070294894465516, + "endpoint_dist": 1.5476418218829415 + }, + "guidance_no_norm": { + "sec_per_target": 0.061, + "queries": 25, + "top5": 0.18181818181818182, + "ndcg": 0.10475755936295951, + "endpoint_dist": 2.317273969000036 + }, + "guidance_norm": { + "sec_per_target": 0.064, + "queries": 25, + "top5": 0.13636363636363635, + "ndcg": 0.0968604433440603, + "endpoint_dist": 1.5734560652212664 + }, + "guidance_rerank": { + "sec_per_target": 0.065, + "queries": 35, + "top5": 0.13636363636363635, + "ndcg": 0.0741331706168103, + "endpoint_dist": 1.0795677466826006 + } + }, + "guidance_step_time": { + "0": 0.01, + "5": 0.024, + "10": 0.034, + "25": 0.065, + "50": 0.112, + "100": 0.201 + }, + "data_scaling_counts": { + "0.1": { + "n_perts": 16, + "cells_per_pert": 354 + }, + "0.25": { + "n_perts": 41, + "cells_per_pert": 354 + }, + "0.5": { + "n_perts": 83, + "cells_per_pert": 354 + }, + "0.75": { + "n_perts": 124, + "cells_per_pert": 354 + }, + "1.0": { + "n_perts": 166, + "cells_per_pert": 354 + } + }, + "heldout_gene_mse": { + "op_only": 0.0034, + "gene_only": 0.0035, + "random_id": 0.0034, + "gene_op": 0.0035, + "gene_pathway_op": 0.0032 + } +} \ No newline at end of file diff --git a/experiments/results/pivot_vs_gears.json b/experiments/results/pivot_vs_gears.json new file mode 100644 index 0000000000000000000000000000000000000000..de391002697ecfaf0021d14817dd5eabb7b22ce1 --- /dev/null +++ b/experiments/results/pivot_vs_gears.json @@ -0,0 +1,7 @@ +{ + "n_common": 45, + "pivot_pearson_de_expr": 0.9713062047958374, + "pivot_delta_de_corr": 0.8417869806289673, + "gears_pearson_de_expr": 0.8981071008576287, + "gears_pearson_all": 0.9916384220123291 +} \ No newline at end of file diff --git a/experiments/results/replogle_k562_forward_cell.json b/experiments/results/replogle_k562_forward_cell.json new file mode 100644 index 0000000000000000000000000000000000000000..12bad81b016734e2f09219fbe86adc7bed79a233 --- /dev/null +++ b/experiments/results/replogle_k562_forward_cell.json @@ -0,0 +1,4693 @@ +{ + "split": "cell", + "models": { + "PIVOT": { + "mse": 0.0011191630950634136, + "r2": 0.20920453891158103, + "pearson": 0.5100029533728957, + "spearman": 0.4549569012656951, + "de_corr": 0.5945503734285011, + "mmd": 0.2736271738874284, + "wasserstein": 0.5047475567593812, + "_per_pert": { + "mse": [ + 0.0002482385898474604, + 0.00045174386468715966, + 0.000490359088871628, + 0.0003910822852049023, + 0.001966833369806409, + 0.0007381021860055625, + 0.0011284874053671956, + 0.0002854544436559081, + 0.0006030005170032382, + 0.001687762443907559, + 0.0007548905559815466, + 0.0005599777214229107, + 0.0007528227288275957, + 0.0005102210561744869, + 0.0003791057097259909, + 0.0011518133105710149, + 0.0009723805706016719, + 0.00048360947403125465, + 0.0001663294096942991, + 0.00015586064546369016, + 0.00042490719351917505, + 0.0009113576961681247, + 0.0008905955473892391, + 0.0008297122549265623, + 0.0006384018925018609, + 0.0006737565854564309, + 0.0006215742905624211, + 0.00040664119296707213, + 0.000768364523537457, + 0.0009767693700268865, + 0.003173335688188672, + 0.0007590964087285101, + 0.0005342193762771785, + 0.0008727642707526684, + 0.0008634499972686172, + 0.00028478342574089766, + 0.0009055290138348937, + 0.0003967172233387828, + 0.0006361014675348997, + 0.0008400965598411858, + 0.000640920945443213, + 0.00018471747171133757, + 0.0007557278731837869, + 0.002429370069876313, + 0.0004084179818164557, + 0.0010379033628851175, + 0.0005447184084914625, + 0.0005065665463916957, + 0.0006548734963871539, + 0.00029585976153612137, + 0.0011924741556867957, + 0.0005116827669553459, + 0.017684947699308395, + 0.0013397781876847148, + 0.0005473991041071713, + 0.0005755973979830742, + 0.0008506292360834777, + 0.0005203686887398362, + 0.0004897243343293667, + 0.001290131825953722, + 0.002257241401821375, + 0.0007021107012405992, + 0.0005917736561968923, + 0.001284355646930635, + 0.0004400366742629558, + 0.00033880415139719844, + 0.0006660501239821315, + 0.0002739697811193764, + 0.0012263801181688905, + 0.0034628277644515038, + 0.0007947250851429999, + 0.004897534381598234, + 0.0004756851412821561, + 0.0011831138981506228, + 0.0005515989614650607, + 0.0008313640719279647, + 0.002382136881351471, + 0.0007333850953727961, + 0.0017490541795268655, + 0.001942841219715774 + ], + "r2": [ + -0.20588767528533936, + 0.8091425895690918, + -0.3445405960083008, + 0.9371975660324097, + 0.576246440410614, + 0.327006459236145, + 0.8424339294433594, + 0.686861515045166, + 0.3900236487388611, + -0.07255733013153076, + -0.43648481369018555, + 0.7702895402908325, + 0.20353597402572632, + -0.6973190307617188, + 0.3837941884994507, + 0.35406041145324707, + 0.3447573184967041, + -0.285636305809021, + 0.28360259532928467, + 0.6373168230056763, + -0.23032891750335693, + 0.15709614753723145, + 0.4600808620452881, + 0.03777611255645752, + 0.3655083179473877, + -0.11896657943725586, + 0.004921436309814453, + 0.066456139087677, + 0.39461636543273926, + 0.2339853048324585, + 0.5403043031692505, + 0.26136481761932373, + 0.38174140453338623, + 0.04237109422683716, + -0.09929919242858887, + -0.030287504196166992, + -0.02270209789276123, + 0.38719815015792847, + 0.31494754552841187, + 0.1296667456626892, + 0.025548040866851807, + -0.07647824287414551, + 0.41560155153274536, + 0.3751434087753296, + -0.004425406455993652, + 0.46141213178634644, + 0.45584404468536377, + 0.0982060432434082, + -0.16907238960266113, + 0.24153822660446167, + 0.5187370777130127, + -0.0938793420791626, + 0.5934585332870483, + 0.6157222986221313, + -0.06867766380310059, + -0.09785819053649902, + 0.339092493057251, + 0.4039953351020813, + -0.8705884218215942, + 0.387775719165802, + 0.1864757537841797, + -0.05910146236419678, + -0.018353700637817383, + 0.38129574060440063, + 0.1581493616104126, + 0.21893751621246338, + -0.05766630172729492, + -0.5318254232406616, + 0.3465535044670105, + 0.6707594990730286, + 0.06754028797149658, + 0.36049193143844604, + -0.2268468141555786, + 0.5062537789344788, + 0.5376465320587158, + 0.46998947858810425, + 0.4930293560028076, + -0.10991322994232178, + 0.4656425714492798, + 0.5459157824516296 + ], + "pearson": [ + 0.19717691838741302, + 0.9005478620529175, + 0.18589536845684052, + 0.9713582396507263, + 0.8110814094543457, + 0.6321790218353271, + 0.9293340444564819, + 0.8794742822647095, + 0.6346621513366699, + 0.34905537962913513, + -0.07565273344516754, + 0.8831305503845215, + 0.46894341707229614, + 0.156036838889122, + 0.6782864928245544, + 0.6093508005142212, + 0.5932121276855469, + 0.2508874237537384, + 0.5724385380744934, + 0.8149585723876953, + 0.46347522735595703, + 0.40686893463134766, + 0.6942399740219116, + 0.40639811754226685, + 0.7095261812210083, + 0.14735180139541626, + 0.3916641175746918, + 0.3746804893016815, + 0.6311635375022888, + 0.5156680345535278, + 0.7642084360122681, + 0.5351200699806213, + 0.6206784844398499, + 0.2578606903553009, + 0.1589634120464325, + 0.4066499173641205, + 0.19453977048397064, + 0.6635164022445679, + 0.5870082974433899, + 0.40200677514076233, + 0.23928773403167725, + 0.4867628514766693, + 0.6623163223266602, + 0.6331384181976318, + 0.4705985188484192, + 0.7325534820556641, + 0.7224423885345459, + 0.3827328383922577, + 0.21962767839431763, + 0.5692456364631653, + 0.7368344664573669, + 0.3163824677467346, + 0.8582233190536499, + 0.8286048173904419, + 0.2719399333000183, + 0.21844376623630524, + 0.5940831899642944, + 0.6630827188491821, + -0.21636082231998444, + 0.6426402926445007, + 0.4891781508922577, + 0.15080882608890533, + 0.2077670395374298, + 0.6181712746620178, + 0.480783611536026, + 0.482818603515625, + 0.17707064747810364, + 0.13104155659675598, + 0.6131221055984497, + 0.8468379974365234, + 0.35086092352867126, + 0.6239715814590454, + 0.1772345006465912, + 0.7369061708450317, + 0.7587236762046814, + 0.7093169093132019, + 0.7192721962928772, + 0.1854822188615799, + 0.7462291717529297, + 0.7881137132644653 + ], + "spearman": [ + 0.16866232216558052, + 0.856259152064788, + 0.22083861720965428, + 0.9546685051671262, + 0.7573857033464259, + 0.5751195612798903, + 0.902758556189639, + 0.8068046932011732, + 0.5413297098324275, + 0.31967923991981, + -0.016325303581325893, + 0.7949219002304752, + 0.45365457991364494, + 0.10292660573165142, + 0.6085034691258673, + 0.572748459687115, + 0.4457567389391847, + 0.26000556200139047, + 0.4046692551673138, + 0.6957858134464533, + 0.4398131254532814, + 0.13745187136296783, + 0.5842313045578261, + 0.30703617112418136, + 0.6573812888453222, + 0.1830912412728103, + 0.4546289486572372, + 0.28733177383294345, + 0.3747017691754423, + 0.3342555259392273, + 0.6910866332716582, + 0.34639829059957267, + 0.5731220572805142, + 0.15341616535404132, + 0.09341607685401922, + 0.36502501025625256, + 0.3456356004089001, + 0.4976741579185394, + 0.5572501303125326, + 0.3742400405600101, + -0.01096654624163656, + 0.4633375968343992, + 0.5491285822821456, + 0.5388305352076338, + 0.3343839190959798, + 0.5907347271836818, + 0.6101929675482418, + 0.32538313084578274, + 0.3081267005316751, + 0.5286397366599341, + 0.7291144900520404, + 0.19349538487384618, + 0.806189780047445, + 0.7510000952500238, + 0.31477875819468953, + 0.30267640716910177, + 0.5066870031717509, + 0.6020635595158899, + 0.0072096445551147565, + 0.5804600051150013, + 0.45200419850104956, + -0.0581443060360765, + 0.1855657468914367, + 0.5872103428025857, + 0.36594201148550287, + 0.44313890978472736, + 0.2568235092058773, + 0.08817904304476075, + 0.5390712342678086, + 0.8022655570663892, + 0.2806923461730865, + 0.5099824544956136, + 0.26928108817125246, + 0.7135502698875674, + 0.72951135987784, + 0.6812116073029018, + 0.6475992403998101, + 0.23473293168323292, + 0.6682944665736166, + 0.7848332877083218 + ], + "de_corr": [ + 0.3437618911266327, + 0.9735803008079529, + 0.11851970106363297, + 0.9928795695304871, + 0.9646739959716797, + 0.7963689565658569, + 0.9744515419006348, + 0.9806885123252869, + 0.7953635454177856, + 0.7012885212898254, + -0.3769508898258209, + 0.9811130166053772, + 0.5302207469940186, + -0.09953459352254868, + 0.9306344985961914, + 0.8591590523719788, + 0.7517970204353333, + 0.41018977761268616, + 0.7984139323234558, + 0.9667317867279053, + 0.7666957974433899, + 0.7438593506813049, + 0.9024503231048584, + 0.6619342565536499, + 0.8587373495101929, + 0.15117232501506805, + 0.12058426439762115, + 0.5688217878341675, + 0.8098616003990173, + 0.9100050330162048, + 0.9491345286369324, + 0.349025696516037, + 0.6825718283653259, + 0.43110179901123047, + -0.0034315120428800583, + 0.47583284974098206, + -0.369070827960968, + 0.9580983519554138, + 0.7099018692970276, + 0.7644200921058655, + 0.556671679019928, + 0.46315041184425354, + 0.9295991659164429, + 0.6248260140419006, + 0.8282192945480347, + 0.8415664434432983, + 0.9216052889823914, + 0.7120542526245117, + -0.5733141899108887, + 0.6905868649482727, + 0.713492751121521, + 0.4630085527896881, + 0.971298098564148, + 0.9192976951599121, + 0.054802581667900085, + 0.34102240204811096, + 0.7345413565635681, + 0.6364703178405762, + -0.5327271819114685, + 0.6061196327209473, + 0.01857166551053524, + 0.6311883330345154, + 0.40586143732070923, + 0.7135980725288391, + 0.7964053153991699, + 0.48196524381637573, + 0.02169019542634487, + 0.3784397542476654, + 0.7018744945526123, + 0.9654350280761719, + 0.5320989489555359, + 0.8798284530639648, + -0.043338190764188766, + 0.8886123299598694, + 0.866779088973999, + 0.9336277842521667, + 0.7747882604598999, + 0.2532675266265869, + 0.8504218459129333, + 0.7755652070045471 + ], + "mmd": [ + 0.2486573975698857, + 0.2710277069556468, + 0.24552850736524656, + 0.23868025983924268, + 0.33752896363288654, + 0.27630172339390824, + 0.27415056999638787, + 0.26716188049666745, + 0.261020037727818, + 0.2983943416531113, + 0.2708856740968154, + 0.2525625262736487, + 0.2921107687730986, + 0.2543344579695398, + 0.27137790907134596, + 0.2832064715279001, + 0.24857353135205595, + 0.2677570375808469, + 0.251386565137227, + 0.255074484500757, + 0.25012523646858464, + 0.2838898752065474, + 0.27201560390247503, + 0.2643692519549542, + 0.26504004673250936, + 0.2580900474252169, + 0.27280485587921466, + 0.26742934994803524, + 0.2597320899846097, + 0.26748492506085597, + 0.32227496126917543, + 0.24756374893517386, + 0.24087238237869468, + 0.2656759275160617, + 0.27077884571964494, + 0.2701214691674093, + 0.2622629246347177, + 0.24934182697509555, + 0.26264188227197993, + 0.2542210100167057, + 0.2815331202388366, + 0.24572992378629677, + 0.24665471392645788, + 0.29613049438647376, + 0.26419795824735093, + 0.22178242924550806, + 0.28883150873095487, + 0.26803943128968255, + 0.26201112916002023, + 0.25201859291087125, + 0.2832208640315248, + 0.2753893118741385, + 0.4265925808647473, + 0.3022646568119438, + 0.23909319254796235, + 0.24406205572145112, + 0.26185608970864105, + 0.24209915234021384, + 0.24780493127353942, + 0.3086340423927729, + 0.3254766342386216, + 0.27275301880046077, + 0.28243584778202646, + 0.28559129630454083, + 0.27023509521220446, + 0.27705856950283436, + 0.25016212934037807, + 0.2594566804443694, + 0.2721865422237648, + 0.32121593108486024, + 0.2714122581506796, + 0.3414189047959242, + 0.2633288450487624, + 0.29592255757872554, + 0.273466356000443, + 0.28814024295612883, + 0.3485198446866875, + 0.26831421577503745, + 0.3040850077803584, + 0.2885906774363758 + ], + "wasserstein": [ + 0.48828093287841, + 0.4471198151141752, + 0.4488112350434935, + 0.50217189398357, + 0.5968836226571762, + 0.5383439851391579, + 0.5802563629660061, + 0.5138755673862574, + 0.4806724820360704, + 0.449442950080591, + 0.46386784829037336, + 0.443125574672048, + 0.4957840913057273, + 0.492041819255636, + 0.46284955008658535, + 0.5415078696263617, + 0.46885357769862884, + 0.46119896218494216, + 0.4763700879166423, + 0.45517617479894346, + 0.45476123073566216, + 0.4739683057689913, + 0.47010894887135246, + 0.5227515170930609, + 0.5110464598170341, + 0.4843184130176394, + 0.5013767257407193, + 0.48044168267906634, + 0.4786165624571058, + 0.5313039844151587, + 0.5832563848014156, + 0.4093175025878733, + 0.4324551702817327, + 0.49671988551910057, + 0.5317294381110366, + 0.48104641311793045, + 0.5300306400171976, + 0.5043342192582557, + 0.5103185026936035, + 0.4392708565040615, + 0.5044502369017986, + 0.4840349287743596, + 0.5067842154099531, + 0.5940063248227425, + 0.47088037833207763, + 0.4580035247183092, + 0.4824495011996967, + 0.45678578676200954, + 0.4294297252127879, + 0.5013487833039562, + 0.5360978464884559, + 0.45248844986545406, + 0.9335867683151108, + 0.6581890182119657, + 0.4262248212325798, + 0.4681121735769381, + 0.4841912763029046, + 0.4714053614449054, + 0.47699760091961485, + 0.5198353897085543, + 0.6060066003428463, + 0.48718213226354423, + 0.47549552916597615, + 0.517845859220808, + 0.5036784842260127, + 0.44522289053903547, + 0.4750928965017518, + 0.48417069580406585, + 0.475192466120183, + 0.6037189499523952, + 0.4976485877033992, + 0.632561209430874, + 0.4771842439048742, + 0.532349431476453, + 0.5000863321007786, + 0.5354414300182885, + 0.5828147265372408, + 0.46497855540429534, + 0.539259707836504, + 0.5947644540882084 + ] + }, + "n_perts": 80 + }, + "MeanControl": { + "mse": 0.0022905822843313217, + "r2": -0.1224516473710537, + "pearson": 0.02966647447901778, + "spearman": 0.01992715045528407, + "de_corr": 0.04512151768431068, + "mmd": 0.03774984783844901, + "wasserstein": 0.27412720530288726, + "_per_pert": { + "mse": [ + 0.0003135530569124967, + 0.0023869420401751995, + 0.00045869467430748045, + 0.006298632826656103, + 0.004806647077202797, + 0.0014369032578542829, + 0.008517427369952202, + 0.000946979271247983, + 0.0012333106715232134, + 0.0019141276134178042, + 0.0005964463925920427, + 0.002873178105801344, + 0.0009904090547934175, + 0.0004019741900265217, + 0.0005515951779671013, + 0.0021324006374925375, + 0.0015554764540866017, + 0.00033043944858945906, + 0.0003248422872275114, + 0.0005623248289339244, + 0.0003560090553946793, + 0.0012256100308150053, + 0.0018651122227311134, + 0.0010445443913340569, + 0.0011137890396639705, + 0.0006109675741754472, + 0.0007604432757943869, + 0.0003866456972900778, + 0.0011687217047438025, + 0.001281142234802246, + 0.007101475726813078, + 0.001073954626917839, + 0.0009493547840975225, + 0.000846831826493144, + 0.0010421575279906392, + 0.00040319678373634815, + 0.0012662794906646013, + 0.000635273871012032, + 0.0009133966523222625, + 0.0010269922204315662, + 0.0007213311037048697, + 0.0002824447292368859, + 0.0012273142347112298, + 0.0037563221994787455, + 0.00044136185897514224, + 0.0020338541362434626, + 0.0013244912261143327, + 0.0006703351973555982, + 0.0005624775658361614, + 0.0004683740553446114, + 0.0026588367763906717, + 0.0004892455763183534, + 0.044183410704135895, + 0.003423706628382206, + 0.0005639897426590323, + 0.0005662624607793987, + 0.0016199243254959583, + 0.0007872314308770001, + 0.00031049130484461784, + 0.0022608372382819653, + 0.003007568884640932, + 0.000815203005913645, + 0.0006296854116953909, + 0.0015612709103152156, + 0.0006085234927013516, + 0.0005212764372117817, + 0.0006985863437876105, + 0.00026034287293441594, + 0.0019447391387075186, + 0.011613603681325912, + 0.0010304647730663419, + 0.00879842322319746, + 0.0004947177367284894, + 0.002385683823376894, + 0.0010427909437566996, + 0.002058811020106077, + 0.004687466658651829, + 0.0006942204781807959, + 0.003578119445592165, + 0.004758638795465231 + ], + "r2": [ + -0.5231711864471436, + -0.008460044860839844, + -0.2577183246612549, + -0.011473894119262695, + -0.035590410232543945, + -0.31015288829803467, + -0.18925392627716064, + -0.038819193840026855, + -0.24757802486419678, + -0.21641027927398682, + -0.13498079776763916, + -0.17861664295196533, + -0.047823309898376465, + -0.3372211456298828, + 0.10342627763748169, + -0.19585561752319336, + -0.04816412925720215, + 0.12155359983444214, + -0.399127721786499, + -0.30851292610168457, + -0.030832886695861816, + -0.13355207443237305, + -0.13071489334106445, + -0.2113661766052246, + -0.10696685314178467, + -0.014687299728393555, + -0.2173938751220703, + 0.11236071586608887, + 0.07918071746826172, + -0.004713773727416992, + -0.02873373031616211, + -0.045006513595581055, + -0.09869992733001709, + 0.07082521915435791, + -0.32682037353515625, + -0.4586822986602783, + -0.43013250827789307, + 0.01870405673980713, + 0.016313016414642334, + -0.06395578384399414, + -0.09670698642730713, + -0.646003007888794, + 0.050927579402923584, + 0.03383910655975342, + -0.08544468879699707, + -0.05540573596954346, + -0.3231234550476074, + -0.19333624839782715, + -0.0041283369064331055, + -0.20071721076965332, + -0.07306301593780518, + -0.0459134578704834, + -0.015688061714172363, + 0.01800614595413208, + -0.1010674238204956, + -0.08005321025848389, + -0.2586214542388916, + 0.09834378957748413, + -0.18597650527954102, + -0.07286691665649414, + -0.0839470624923706, + -0.22969591617584229, + -0.08359432220458984, + 0.24789905548095703, + -0.16418945789337158, + -0.20172536373138428, + -0.10933279991149902, + -0.45563483238220215, + -0.03620636463165283, + -0.10420417785644531, + -0.20905566215515137, + -0.14887666702270508, + -0.27593398094177246, + 0.004387915134429932, + 0.12592661380767822, + -0.31253111362457275, + 0.002404928207397461, + -0.05064129829406738, + -0.09315896034240723, + -0.1121973991394043 + ], + "pearson": [ + -0.21644580364227295, + 0.1042175143957138, + 0.13106493651866913, + 0.09760955721139908, + 0.007358158938586712, + -0.14829552173614502, + -0.3576577305793762, + 0.06429620832204819, + -0.15006688237190247, + -0.2385982722043991, + 0.092145636677742, + 0.02407045103609562, + 0.10666121542453766, + -0.04918693006038666, + 0.4433865547180176, + -0.13500572741031647, + -0.035410404205322266, + 0.38158494234085083, + -0.03492853417992592, + -0.2034790813922882, + 0.25720396637916565, + -0.11880083382129669, + -0.14922724664211273, + -0.21420618891716003, + 0.2990036904811859, + 0.253147691488266, + -0.03679570555686951, + 0.34075281023979187, + 0.30351197719573975, + 0.15207503736019135, + -0.01308598741889, + 0.06589823216199875, + -0.11117678135633469, + 0.2880307137966156, + -0.15887011587619781, + -0.2537006437778473, + -0.09232877939939499, + 0.2578761577606201, + 0.18827873468399048, + 0.03849196061491966, + -0.04920819401741028, + -0.09934255480766296, + 0.2885802090167999, + 0.18790365755558014, + 0.10903426259756088, + 0.12201783806085587, + -0.20839226245880127, + -0.1071806401014328, + 0.14567029476165771, + 0.1483369618654251, + -0.20555271208286285, + 0.14310581982135773, + 0.054714735597372055, + 0.3195492923259735, + 0.02492167241871357, + 0.02195236086845398, + -0.3668579161167145, + 0.34766048192977905, + 0.027576616033911705, + 0.10230936855077744, + -0.06371402740478516, + -0.16928501427173615, + 0.0031384737230837345, + 0.6421991586685181, + 0.02607305720448494, + -0.085452601313591, + -0.04037441685795784, + -0.06688378006219864, + -0.02383958362042904, + -0.43549561500549316, + -0.3370629549026489, + -0.12744681537151337, + -0.08400639146566391, + 0.2959127724170685, + 0.501526951789856, + -0.1336817741394043, + 0.16721303761005402, + 0.07728126645088196, + 0.006217338144779205, + 0.034800607711076736 + ], + "spearman": [ + -0.2851684687921172, + 0.09777964994491248, + 0.10182346795586697, + -0.004514770128692532, + 0.007072513768128442, + -0.08049392912348229, + -0.2362290695572674, + -0.028150691537672884, + -0.06101977225494307, + -0.2570101257525314, + 0.010396636099159026, + -0.07972070593017648, + 0.08710347080853151, + 0.049684201921050476, + 0.28505328226332055, + -0.13992541248135312, + -0.18330527632631907, + 0.24775457893864472, + 0.00496530124132531, + -0.09684809421202355, + 0.04394628048657012, + -0.17707416226854056, + -0.16246580861645213, + -0.19438462109615526, + 0.280123900030975, + 0.2282509980627495, + -0.10327789831947458, + 0.29164452241113054, + 0.18001067550266886, + 0.14672337668084418, + -0.05144382986095746, + -0.002147961536990384, + -0.08923866580966644, + 0.2658778174694544, + -0.1242450100612525, + -0.3370721147680287, + -0.008982972745743186, + 0.29263551115887776, + 0.2388225672056418, + 0.007832385458096364, + -0.12252070263017564, + 0.043791513447878364, + 0.013028610257152564, + 0.033908927477231864, + 0.10715351828837957, + 0.03793403898350974, + -0.18216283854070964, + -0.1352343848085962, + 0.14042754660688664, + 0.06335993933998484, + -0.17310380084086413, + 0.21100021225005305, + 0.049847199461799864, + 0.22253199463299864, + 0.058301228575307146, + 0.01756774789193697, + -0.24678717769679442, + 0.22874111118527782, + 0.2148393072098268, + 0.09783249495812374, + -0.04594179848544962, + -0.17413281303320327, + 0.08679603219900804, + 0.6014869083717271, + -0.12409024152256037, + 0.02761858740464685, + 0.11626595756648939, + 0.057246731311682834, + 0.09639752409938102, + -0.3985915866478966, + -0.3250261262565316, + -0.09425696806424201, + -0.1043312878719462, + 0.18919543579885895, + 0.5221075530268883, + -0.1304242024049597, + 0.18193299948324987, + 0.15696853674213418, + 0.11846008011502, + -0.00874754768688692 + ], + "de_corr": [ + -0.03622117266058922, + 0.20828865468502045, + -0.03621973097324371, + 0.45591089129447937, + 0.4728381335735321, + -0.41699478030204773, + -0.8033862709999084, + 0.4083835184574127, + -0.1466226726770401, + -0.608994722366333, + 0.5214008688926697, + -0.38360753655433655, + 0.20717477798461914, + -0.48103851079940796, + 0.6656442284584045, + -0.4428849518299103, + 0.5646765232086182, + 0.7292507886886597, + 0.09516989439725876, + -0.4726733863353729, + 0.6706956624984741, + 0.17503537237644196, + -0.16561850905418396, + -0.3443540930747986, + 0.7135056853294373, + 0.5684618949890137, + 0.172411248087883, + 0.30194059014320374, + 0.603882908821106, + -0.053448859602212906, + 0.2979581654071808, + 0.2559824287891388, + -0.26689234375953674, + 0.593704104423523, + -0.08733691275119781, + -0.12478853017091751, + -0.368988037109375, + 0.4716603755950928, + -0.1069912388920784, + 0.3471878468990326, + -0.09807274490594864, + -0.43367597460746765, + 0.6653913855552673, + 0.47266554832458496, + -0.01958230510354042, + 0.4013681709766388, + -0.21912091970443726, + -0.06988514959812164, + 0.2546762526035309, + 0.40011826157569885, + -0.7268434166908264, + 0.052366092801094055, + 0.13671278953552246, + 0.6884567141532898, + -0.10637510567903519, + -0.0608709454536438, + -0.44986432790756226, + 0.5261383056640625, + -0.12742769718170166, + 0.32255059480667114, + -0.21346139907836914, + -0.5204205513000488, + -0.40037015080451965, + 0.821496844291687, + 0.5258604884147644, + -0.39938032627105713, + -0.1941952258348465, + -0.43138375878334045, + -0.2626570463180542, + -0.7695042490959167, + -0.28996437788009644, + -0.3552553951740265, + -0.11542253196239471, + 0.600836992263794, + 0.6032857298851013, + -0.555926501750946, + -0.05847649276256561, + 0.12887315452098846, + -0.23754405975341797, + -0.029497560113668442 + ], + "mmd": [ + 0.010865961643706346, + 0.05054846452657813, + 0.01422797814673038, + 0.10684476369911045, + 0.06917327283658214, + 0.026565552992255714, + 0.1069428741974402, + 0.021347239900252002, + 0.027468319718501943, + 0.04392867467747663, + 0.018260191544575033, + 0.055195328636012064, + 0.024776832674219706, + 0.011371268103524512, + 0.013937829151025705, + 0.035460934167209524, + 0.035684153688181786, + 0.011888209474225309, + 0.009565067855383247, + 0.01513977092718477, + 0.012460190426496931, + 0.029818676972314218, + 0.039178074747819314, + 0.0204067942819125, + 0.023270166007833493, + 0.01701131694328495, + 0.02102455344241161, + 0.011638730985153223, + 0.029122536655155518, + 0.024780960453011813, + 0.11173807672932856, + 0.029495218385469424, + 0.024968154548761112, + 0.019917608768503348, + 0.024258711804634903, + 0.012360462845132258, + 0.027001708173204464, + 0.012124944835591123, + 0.01921798243528927, + 0.02931381434802216, + 0.01883548384246203, + 0.006639578873115504, + 0.023925340879753176, + 0.06282245587600355, + 0.014253139492391043, + 0.0381177992549262, + 0.029588154887629226, + 0.02085982407091136, + 0.020911521970735758, + 0.011122182566530081, + 0.052369751099481054, + 0.018377781101224122, + 0.3791615915445561, + 0.03536897931265082, + 0.01943580935877487, + 0.01608984434064953, + 0.034334627734487144, + 0.01666877323278415, + 0.009441982629030399, + 0.04589784364178906, + 0.05729250853390444, + 0.019322761084301754, + 0.021597821783511884, + 0.031073456240289565, + 0.012851529856980903, + 0.01618545703905938, + 0.01660552234340429, + 0.008210754476712823, + 0.04492233415454716, + 0.1587593155414726, + 0.026532058872810738, + 0.13117023029348718, + 0.014141944029925857, + 0.04816317541262982, + 0.02320142785712387, + 0.03607618039615024, + 0.07814315304558717, + 0.02191472779144843, + 0.06182874721860088, + 0.06947088304458082 + ], + "wasserstein": [ + 0.15711171303839613, + 0.3121499221192411, + 0.17834646191063253, + 0.3889794030141152, + 0.4102136933142445, + 0.23258250451573392, + 0.48766399335374466, + 0.21206108783173336, + 0.2626443660908406, + 0.3649056829461474, + 0.2485004868279747, + 0.3102982327789773, + 0.2816768179894778, + 0.16998563599856428, + 0.18993128185053795, + 0.31785639364189217, + 0.30405023458837976, + 0.19787415617765966, + 0.15405863233116132, + 0.19925227321365782, + 0.18098311048437432, + 0.24838140286217503, + 0.26952438674066015, + 0.2694212150322934, + 0.22393375308200117, + 0.21199874823244158, + 0.18880515483121, + 0.17103852416134602, + 0.2488377125084003, + 0.20439962780602636, + 0.48780347502186816, + 0.3025793674939964, + 0.2679925082943819, + 0.22353186595265417, + 0.22300177007790367, + 0.15814201542508396, + 0.2491484659055471, + 0.1720302669462278, + 0.2350301884036371, + 0.28089894245273217, + 0.21967908491481883, + 0.13948037854267153, + 0.20925351407938275, + 0.36213109971952895, + 0.18721203221572075, + 0.2606676806958279, + 0.24396448976596488, + 0.21059705973543763, + 0.2543436695268431, + 0.18145908484259288, + 0.3559556020310302, + 0.236737610208738, + 1.1055039334327073, + 0.32654301529633684, + 0.23982663877360738, + 0.20916558689978565, + 0.3231552914763818, + 0.16753448686240827, + 0.15194700862215424, + 0.3340856849186472, + 0.31651805195827065, + 0.21330888458450342, + 0.2578858669687515, + 0.27157019947639593, + 0.16833683641633898, + 0.2096001910844524, + 0.18138038526018666, + 0.14453342621857984, + 0.30795626668050263, + 0.5632831971215269, + 0.274359450170404, + 0.5746989971560678, + 0.20618389784808908, + 0.3210553154302969, + 0.23376340494909034, + 0.3099721782077547, + 0.4286447286497619, + 0.23731414076373514, + 0.43200462392154715, + 0.36093795355606567 + ] + }, + "n_perts": 80 + }, + "AvgPerturbationEffect": { + "mse": 0.0021959878715279045, + "r2": -0.5084860488772392, + "pearson": 0.21063763109559658, + "spearman": 0.17510878409035896, + "de_corr": 0.26164687089039945, + "mmd": 0.038276886467220064, + "wasserstein": 0.27949161117615606, + "_per_pert": { + "mse": [ + 0.000595998193603009, + 0.001810793881304562, + 0.0012213538866490126, + 0.005138915032148361, + 0.003084033029153943, + 0.0008304017828777432, + 0.005854329094290733, + 0.0007161229732446373, + 0.0015947161009535193, + 0.002487269463017583, + 0.0012451116926968098, + 0.0027573446277529, + 0.001322481781244278, + 0.0007396502769552171, + 0.0010641798144206405, + 0.0018826508894562721, + 0.001766084460541606, + 0.000934893439989537, + 0.0006674505420960486, + 0.0010260300477966666, + 0.0007958547212183475, + 0.0018833928043022752, + 0.002257050946354866, + 0.0011318533215671778, + 0.0010129103902727365, + 0.0009903139434754848, + 0.0011103046126663685, + 0.0007768788491375744, + 0.00170531973708421, + 0.0013174935011193156, + 0.00567611213773489, + 0.0019212891347706318, + 0.0012213800800964236, + 0.0012505111517384648, + 0.0010773635003715754, + 0.0006973157869651914, + 0.0013662608107551932, + 0.0005226401262916625, + 0.0010204825084656477, + 0.0018087675562128425, + 0.0007406367221847177, + 0.00047477419138886034, + 0.0012034514220431447, + 0.002795871812850237, + 0.000772701867390424, + 0.0027437505777925253, + 0.0013553125318139791, + 0.000983877805992961, + 0.0011673307744786143, + 0.0008264149655587971, + 0.0019357521086931229, + 0.0009961564792320132, + 0.039364658296108246, + 0.001976211555302143, + 0.0012652270961552858, + 0.0011407651472836733, + 0.0014164162566885352, + 0.0013028610264882445, + 0.0005143110756762326, + 0.001666258554905653, + 0.002467394806444645, + 0.0008912424673326313, + 0.0008407327113673091, + 0.0011855866760015488, + 0.0004853073332924396, + 0.0011527278693392873, + 0.001238119089975953, + 0.0007213957724161446, + 0.0018406407907605171, + 0.008790230378508568, + 0.0012271879240870476, + 0.007748774252831936, + 0.0008493165951222181, + 0.0018586268415674567, + 0.0009497959399595857, + 0.001483013853430748, + 0.0034490623511373997, + 0.0011269007809460163, + 0.003076439956203103, + 0.003370486432686448 + ], + "r2": [ + -1.895226001739502, + 0.2349570393562317, + -2.3488926887512207, + 0.17476075887680054, + 0.33554619550704956, + 0.2428486943244934, + 0.18258380889892578, + 0.2144259810447693, + -0.6131644248962402, + -0.5806366205215454, + -1.3693292140960693, + -0.13110017776489258, + -0.39914631843566895, + -1.460545539855957, + -0.7297389507293701, + -0.05579519271850586, + -0.19008326530456543, + -1.485337495803833, + -1.8747761249542236, + -1.3875410556793213, + -1.3044164180755615, + -0.7419278621673584, + -0.36832594871520996, + -0.31261909008026123, + -0.006706118583679199, + -0.6447007656097412, + -0.7774872779846191, + -0.7835142612457275, + -0.34359729290008545, + -0.03322172164916992, + 0.1777471899986267, + -0.8695012331008911, + -0.41351842880249023, + -0.372106671333313, + -0.3716423511505127, + -1.5227437019348145, + -0.5430512428283691, + 0.19268733263015747, + -0.0990133285522461, + -0.8738687038421631, + -0.12605905532836914, + -1.7668414115905762, + 0.06938052177429199, + 0.2808758020401001, + -0.9003115892410278, + -0.42378461360931396, + -0.35391318798065186, + -0.7515075206756592, + -1.08390474319458, + -1.1185860633850098, + 0.21876221895217896, + -1.1295912265777588, + 0.09508538246154785, + 0.43317925930023193, + -1.470080852508545, + -1.1758232116699219, + -0.10050320625305176, + -0.49223291873931885, + -0.9645018577575684, + 0.2092871069908142, + 0.11073505878448486, + -0.34439802169799805, + -0.4467751979827881, + 0.4288750886917114, + 0.07154005765914917, + -1.657442331314087, + -0.9660934209823608, + -3.033482551574707, + 0.019259989261627197, + 0.1642378568649292, + -0.4398730993270874, + -0.011815905570983887, + -1.1904852390289307, + 0.2243434190750122, + 0.20387542247772217, + 0.0545504093170166, + 0.2659643292427063, + -0.7054643630981445, + 0.06011015176773071, + 0.21224409341812134 + ], + "pearson": [ + 0.05672271177172661, + 0.4848932921886444, + 0.021013827994465828, + 0.4578694999217987, + 0.7130343914031982, + 0.5598609447479248, + 0.6942851543426514, + 0.5278257727622986, + -0.1066230833530426, + -0.038593653589487076, + 0.06997134536504745, + 0.2001589685678482, + 0.08039654046297073, + 0.14058801531791687, + 0.2669000029563904, + 0.24505901336669922, + 0.06796550750732422, + 0.0077692242339253426, + -0.07569713145494461, + -0.07698711007833481, + -0.002193011110648513, + -0.19049108028411865, + -0.05070196092128754, + 0.19075532257556915, + 0.3888321816921234, + 0.15095636248588562, + 0.131508007645607, + 0.19731159508228302, + -0.0777541771531105, + 0.32632315158843994, + 0.48728781938552856, + -0.14392368495464325, + 0.066754050552845, + 0.15657955408096313, + 0.1489914208650589, + 0.07345448434352875, + 0.044137146323919296, + 0.5609685778617859, + 0.40791982412338257, + -0.34436506032943726, + 0.36143824458122253, + 0.30871590971946716, + 0.4843851923942566, + 0.571428656578064, + 0.2000991851091385, + -0.0726737380027771, + 0.24507570266723633, + 0.052542172372341156, + -0.19413554668426514, + 0.07941228896379471, + 0.47271955013275146, + -0.08597005903720856, + 0.5830994844436646, + 0.7834280729293823, + -0.3917469382286072, + -0.16404463350772858, + 0.21568366885185242, + 0.08267325162887573, + 0.2513197064399719, + 0.49226945638656616, + 0.41893795132637024, + 0.22092637419700623, + 0.12559816241264343, + 0.6606823205947876, + 0.4686332643032074, + -0.1058088019490242, + -0.07133535295724869, + -0.16025127470493317, + 0.2797839641571045, + 0.6064351797103882, + 0.07631248980760574, + 0.26569613814353943, + 0.03630329295992851, + 0.5031899809837341, + 0.4850894510746002, + 0.42045989632606506, + 0.6024590730667114, + 0.022980021312832832, + 0.3208281993865967, + 0.5796067714691162 + ], + "spearman": [ + -0.013624797406199351, + 0.39509459077364767, + 0.05718138229534558, + 0.32829721307430326, + 0.616069828017457, + 0.47584961746240434, + 0.6839575709893927, + 0.410455904113976, + -0.006615661653915414, + -0.14278376219594055, + -0.020775630693907674, + 0.09478523969630993, + 0.06180683895170974, + 0.14846781411695353, + 0.1363634335908584, + 0.22846803011700753, + 0.09746879136719784, + 0.032382912595728154, + -0.05274027868506966, + -0.06326471631617908, + -0.03450006562501641, + -0.2403752475938119, + 0.014076647019161755, + 0.105370707882191, + 0.350446556111639, + 0.12437294659323664, + 0.041035831274346256, + 0.12966301891575474, + -0.11412552603138151, + 0.3379006474751619, + 0.3550448957612239, + -0.12216171804042951, + 0.10175316143829036, + 0.18929525182381296, + 0.18473918668479666, + 0.034236854559213645, + 0.2079045175540936, + 0.4821338645334662, + 0.3709759137439784, + -0.3611260892815223, + 0.286240180060045, + 0.30144287586071894, + 0.30904439176109794, + 0.46959588339897085, + 0.13912914778228694, + -0.14985411296352824, + 0.1934878143719536, + 0.07519247629811907, + -0.15743100185775044, + 0.07331899532974884, + 0.3960248311547171, + -0.09774134143533535, + 0.4694889483722371, + 0.6630154252538564, + -0.4064742566185641, + -0.1538770284692571, + 0.1998487609621902, + 0.0839622704905676, + 0.24212110253027563, + 0.4216612144153036, + 0.3399957894989474, + 0.15747556536889132, + 0.0364885096221274, + 0.5969580892395223, + 0.48057802664450666, + -0.06006062851515713, + 0.03156994689248672, + -0.06875929418982354, + 0.23882092170523042, + 0.45261309515327375, + 0.06643995910998977, + 0.22765737741434433, + 0.05775439671025709, + 0.40116729279182317, + 0.446841812210453, + 0.2959867099966775, + 0.5266668296667074, + 0.00994293548573387, + 0.29848927312231827, + 0.4903738675934669 + ], + "de_corr": [ + 0.6795547008514404, + 0.8173455595970154, + -0.19555197656154633, + 0.9086191058158875, + 0.9501972794532776, + 0.8444578647613525, + 0.6988091468811035, + 0.9087811708450317, + -0.6423086524009705, + -0.023008078336715698, + 0.6255181431770325, + 0.8040119409561157, + 0.12884576618671417, + -0.23136714100837708, + 0.5087198615074158, + -0.3248017430305481, + 0.4812922775745392, + -0.17621910572052002, + -0.41062310338020325, + -0.4986657202243805, + 0.037861838936805725, + -0.11219538748264313, + -0.20493629574775696, + 0.3347386121749878, + 0.6805623769760132, + 0.6117284893989563, + -0.056225672364234924, + 0.2569712996482849, + -0.002006791066378355, + 0.1194739118218422, + 0.7243301868438721, + 0.12780329585075378, + 0.0013009961694478989, + 0.014316813088953495, + 0.19244393706321716, + -0.2595793604850769, + -0.8286257386207581, + 0.8535451292991638, + 0.49598196148872375, + -0.056878361850976944, + -0.3241399824619293, + -0.013941076584160328, + 0.7050033211708069, + 0.6747863292694092, + 0.26943838596343994, + 0.11114496737718582, + 0.5434620976448059, + -0.03880246356129646, + 0.1550026834011078, + 0.34202873706817627, + 0.7708920836448669, + -0.15790432691574097, + 0.8224325776100159, + 0.8878723382949829, + -0.20048503577709198, + 0.011172322556376457, + 0.10067470371723175, + 0.29093605279922485, + 0.15547069907188416, + 0.5268821120262146, + -0.1996365636587143, + 0.44005754590034485, + 0.33619633316993713, + 0.8349352478981018, + 0.6264446377754211, + -0.2421439290046692, + -0.1860623061656952, + -0.11250001937150955, + 0.1581939309835434, + 0.8919805884361267, + 0.05610169842839241, + 0.29707416892051697, + 0.004912140779197216, + 0.8494806289672852, + 0.7245584726333618, + 0.6719977259635925, + 0.6575203537940979, + -0.07045873254537582, + 0.10349959135055542, + 0.6734530925750732 + ], + "mmd": [ + 0.015848980509615584, + 0.03975651672006264, + 0.023451659439274852, + 0.08970295232100933, + 0.0600596462476124, + 0.0206708232351156, + 0.08231619222319231, + 0.016316701674266354, + 0.03358604048958691, + 0.051770778811150664, + 0.0251241935747738, + 0.0519014745792411, + 0.028752291926166462, + 0.018104878321753004, + 0.021389159954941395, + 0.0356783369410677, + 0.036004181419953096, + 0.019559225062100394, + 0.014769385671107194, + 0.021501838571127196, + 0.016584049911963716, + 0.03976300854296222, + 0.04255633454514174, + 0.028253028442517758, + 0.023092694200297248, + 0.022436494234185034, + 0.022816475677379766, + 0.015586581287422185, + 0.03833514610588151, + 0.028200666915333072, + 0.09906996727720285, + 0.039393060004922265, + 0.02814589974928583, + 0.02527741091336111, + 0.027705428108879993, + 0.015452256565999911, + 0.027861983103570243, + 0.013215062711478542, + 0.01970766367275778, + 0.038671325070677054, + 0.019174020411628412, + 0.013845413025966424, + 0.021508593900489315, + 0.05709566906643715, + 0.017155895401377408, + 0.047626453169032024, + 0.030548991772054168, + 0.02727020661326951, + 0.029955297234213196, + 0.01787439527564727, + 0.041867104469072935, + 0.027590587090185803, + 0.35545822016586603, + 0.0290998770720311, + 0.027002240642055697, + 0.02344235667657435, + 0.03331925790316259, + 0.022525359888761254, + 0.013466125592154099, + 0.038137048261521644, + 0.05380634440108889, + 0.02532807867785114, + 0.025099592150722994, + 0.02599351414506701, + 0.014693159140602985, + 0.02351655249710305, + 0.022711605036701443, + 0.015569284922028581, + 0.04301531062823161, + 0.13115303359020025, + 0.03170057250558933, + 0.11584399296023862, + 0.018939874680683588, + 0.040413689342772896, + 0.022576100697451085, + 0.03226845042281279, + 0.0670847022311134, + 0.0288033738924095, + 0.05663740438591558, + 0.054643372703182425 + ], + "wasserstein": [ + 0.20194809898910374, + 0.27967290411329754, + 0.23789198578064333, + 0.35951457671341525, + 0.35370647725710486, + 0.19227499603821577, + 0.4062059040663462, + 0.17997236153882182, + 0.27806128536847075, + 0.41447135637637367, + 0.30263676742548323, + 0.31342080018961277, + 0.3045955522736588, + 0.1991131574522735, + 0.23248530758887453, + 0.3006288408883304, + 0.31911392293720753, + 0.23981822647065734, + 0.19654651842496082, + 0.23775711919699777, + 0.20839019852931207, + 0.26021123887004355, + 0.2904216461001441, + 0.2731785690565208, + 0.2222559349230698, + 0.2524683894605095, + 0.23414155857309057, + 0.20012712761537757, + 0.2766421173191392, + 0.2209759988653262, + 0.4245999657063124, + 0.33424080690849556, + 0.27913392445937074, + 0.2373863712829229, + 0.210657421851175, + 0.19491732808880566, + 0.2547876123520195, + 0.16623587377926768, + 0.24558639269240382, + 0.31083862534567414, + 0.22845042412610286, + 0.17011461050794907, + 0.22094418440179844, + 0.3574624610049989, + 0.20313442235037468, + 0.28681812906913534, + 0.25943440250494, + 0.22571705864129754, + 0.28274628042427796, + 0.1980740795821249, + 0.32266626971802426, + 0.26676125664525574, + 1.020193736203262, + 0.26990130281031, + 0.2826073057875583, + 0.2699029921253273, + 0.3088781771368151, + 0.17748467421377595, + 0.17985269714438903, + 0.31660295933810906, + 0.3240724722056379, + 0.23446872991732895, + 0.26037512294477755, + 0.23788943205005658, + 0.161699299943524, + 0.2538516222943231, + 0.2069481647949157, + 0.19432904266817283, + 0.30265179195057884, + 0.4983278004571953, + 0.28388016111092557, + 0.533420291763944, + 0.2250069545297918, + 0.28338448794154175, + 0.22156076244444636, + 0.2779816415024388, + 0.40054682440789696, + 0.250129390590885, + 0.4065195911992578, + 0.30750462477019075 + ] + }, + "n_perts": 80 + }, + "Additive": { + "mse": 0.00034133204599129385, + "r2": 0.5863893613219261, + "pearson": 0.8470813885331154, + "spearman": 0.8051852191744894, + "de_corr": 0.9229583293199539, + "mmd": 0.017067014377718525, + "wasserstein": 0.1988901412503695, + "_per_pert": { + "mse": [ + 0.00024666020181030035, + 0.0002835692139342427, + 0.0002478808455634862, + 0.0002530887140892446, + 0.0008113104850053787, + 0.00031414139084517956, + 0.0001556011411594227, + 0.00011543891014298424, + 0.00018039382121060044, + 0.0005247522494755685, + 0.000285469664959237, + 0.00031463391496799886, + 0.0002934891090262681, + 0.0001771973620634526, + 0.0001831721019698307, + 0.0003111836267635226, + 0.00034181663068011403, + 0.0002738374751061201, + 0.00020665943156927824, + 0.00018150197865907103, + 0.000694804941304028, + 0.0003870493092108518, + 0.00042090078932233155, + 0.0005266591324470937, + 0.0002580670698080212, + 0.0002574709360487759, + 0.00020162059809081256, + 0.00024779874365776777, + 0.00015485667972825468, + 0.00047555603669025004, + 0.0004128919681534171, + 0.00033290308783762157, + 0.00024150818353518844, + 0.0003065782075282186, + 0.0005201814346946776, + 0.00018012113287113607, + 0.00023756151495035738, + 0.00021429650951176882, + 0.00037766972673125565, + 0.0005346401012502611, + 0.00046765425940975547, + 0.0001525706029497087, + 0.00023469510779250413, + 0.00045847598812542856, + 0.00021582734188996255, + 0.0004265247262082994, + 0.0004218204121571034, + 0.00026694839471019804, + 0.0003895766567438841, + 0.00019685173174366355, + 0.0003251571033615619, + 0.0004144203558098525, + 0.0006686582928523421, + 0.0001693940139375627, + 0.00026813967269845307, + 0.00025309206102974713, + 0.0006148631800897419, + 0.0001430534030077979, + 0.0003080190217588097, + 0.0005426203715614974, + 0.0007035515154711902, + 0.000473168445751071, + 0.0003781112318392843, + 0.0006829269113950431, + 0.00024176001898013055, + 0.0003351293853484094, + 0.000161615593242459, + 0.00021044899767730385, + 0.0003129686519969255, + 0.0003514640557114035, + 0.0004350198432803154, + 0.0005762679502367973, + 0.00031753379153087735, + 0.00042344362009316683, + 0.0002516146923881024, + 0.00014651250967290252, + 0.0007338344003073871, + 0.00031683166162110865, + 0.00047412680578418076, + 0.00015693652676418424 + ], + "r2": [ + -0.1982201337814331, + 0.8801947236061096, + 0.3203226923942566, + 0.9593574404716492, + 0.8252034783363342, + 0.7135692834854126, + 0.9782740473747253, + 0.8733656406402588, + 0.8175193071365356, + 0.6665247678756714, + 0.45677846670150757, + 0.8709328770637512, + 0.6894972324371338, + 0.41052907705307007, + 0.7022684812545776, + 0.8254874348640442, + 0.7696654796600342, + 0.27202552556991577, + 0.10989713668823242, + 0.5776503086090088, + -1.011824369430542, + 0.6420226097106934, + 0.7448309659957886, + 0.3892292380332947, + 0.7435135841369629, + 0.5723954439163208, + 0.6772255301475525, + 0.43111783266067505, + 0.8779906034469604, + 0.6270532608032227, + 0.9401876926422119, + 0.6760702133178711, + 0.7204996347427368, + 0.6636111736297607, + 0.33773231506347656, + 0.34835904836654663, + 0.7316986322402954, + 0.6689801216125488, + 0.5932667255401611, + 0.44611722230911255, + 0.28898149728775024, + 0.11086416244506836, + 0.8185121417045593, + 0.8820757269859314, + 0.4692140221595764, + 0.7786681652069092, + 0.5786152482032776, + 0.5247761011123657, + 0.304532527923584, + 0.4953535795211792, + 0.8687719106674194, + 0.11404889822006226, + 0.9846288561820984, + 0.9514141082763672, + 0.47651559114456177, + 0.517268180847168, + 0.5222740173339844, + 0.8361536860466003, + -0.17653298377990723, + 0.742502748966217, + 0.7464355230331421, + 0.2862473726272583, + 0.3493272662162781, + 0.6710180044174194, + 0.5374797582626343, + 0.2274090051651001, + 0.7433596849441528, + -0.1766669750213623, + 0.8332423567771912, + 0.9665833115577698, + 0.48958641290664673, + 0.9247523546218872, + 0.1810438632965088, + 0.8232852220535278, + 0.7890951037406921, + 0.9065954685211182, + 0.843824028968811, + 0.5205032825469971, + 0.855148434638977, + 0.9633204936981201 + ], + "pearson": [ + 0.6622630953788757, + 0.9482947587966919, + 0.7638598680496216, + 0.9803251028060913, + 0.9218734502792358, + 0.8484968543052673, + 0.9895062446594238, + 0.9400838613510132, + 0.952268123626709, + 0.8506721258163452, + 0.7825549244880676, + 0.9471822381019592, + 0.862853467464447, + 0.791928231716156, + 0.8924075961112976, + 0.9179850220680237, + 0.9185068011283875, + 0.7556659579277039, + 0.599132776260376, + 0.7923889756202698, + 0.7520980834960938, + 0.8279823064804077, + 0.8709278702735901, + 0.7238227725028992, + 0.8980642557144165, + 0.8512991666793823, + 0.8481652140617371, + 0.8245766162872314, + 0.946467399597168, + 0.927772581577301, + 0.9711289405822754, + 0.934574544429779, + 0.8855801224708557, + 0.8632384538650513, + 0.6815758943557739, + 0.7021403908729553, + 0.8993551731109619, + 0.9136989116668701, + 0.861723005771637, + 0.823276937007904, + 0.6200095415115356, + 0.615280032157898, + 0.9335994124412537, + 0.9592396020889282, + 0.8497616648674011, + 0.9243088364601135, + 0.8281819820404053, + 0.7785792946815491, + 0.7988719940185547, + 0.8236978054046631, + 0.9329332709312439, + 0.7214899063110352, + 0.9955779910087585, + 0.9781326055526733, + 0.7653619647026062, + 0.7866364121437073, + 0.7645065188407898, + 0.9549956321716309, + 0.7164408564567566, + 0.9000207185745239, + 0.8839771747589111, + 0.7555385828018188, + 0.7390056252479553, + 0.8930636644363403, + 0.790571928024292, + 0.6510560512542725, + 0.8938482403755188, + 0.7096249461174011, + 0.9432356953620911, + 0.9882639646530151, + 0.7792956829071045, + 0.9726700186729431, + 0.536105215549469, + 0.9133987426757812, + 0.9362773895263672, + 0.9594414234161377, + 0.9417586922645569, + 0.7996209859848022, + 0.9248581528663635, + 0.9815547466278076 + ], + "spearman": [ + 0.5572901668225416, + 0.9144513426128354, + 0.71560855990214, + 0.9614981358745339, + 0.8917783314445829, + 0.8364303576075892, + 0.9839011150942417, + 0.880348268587067, + 0.9081236870309217, + 0.7932723088655544, + 0.6856410784102696, + 0.881486195871549, + 0.7742467560616889, + 0.7080133665033417, + 0.8519049944762485, + 0.9030076237519059, + 0.8813226448306613, + 0.7292154758038689, + 0.48943340185835044, + 0.6810150667537667, + 0.7162727825681956, + 0.7467115776778944, + 0.7734659653664913, + 0.7344892566223141, + 0.8778268774567194, + 0.8247502221875554, + 0.813524007881002, + 0.757526047881512, + 0.904790315697579, + 0.8888554637138659, + 0.9591585597896399, + 0.8730197302549325, + 0.8225327446331862, + 0.8297972869493218, + 0.6309029312257327, + 0.7045173666293417, + 0.8163245070811267, + 0.8789276827319206, + 0.794712188178047, + 0.787976251994063, + 0.46576784044196007, + 0.6591360432840108, + 0.8731878937969735, + 0.9424590091147523, + 0.7745397781349445, + 0.8747203451800861, + 0.8010643822660956, + 0.7602120460530114, + 0.6688201417050353, + 0.7908854307213578, + 0.9165395999786025, + 0.6489359327339832, + 0.9933628448407111, + 0.9634811393702848, + 0.6505098741274685, + 0.734998935749734, + 0.7517218234304558, + 0.9383035925758982, + 0.7301091880272971, + 0.8622200020550004, + 0.8301324155331039, + 0.682158446045421, + 0.6762637900659475, + 0.8475465708866426, + 0.7424297771074443, + 0.6224288876072219, + 0.836573888143472, + 0.6762609010652252, + 0.928051132012783, + 0.9836085074021267, + 0.7376251784062946, + 0.9515502653875664, + 0.6368560759528401, + 0.9029208097302023, + 0.9305236636309159, + 0.9417113609278402, + 0.9174386518596629, + 0.7721571325392831, + 0.8697788954447239, + 0.963754695938674 + ], + "de_corr": [ + 0.8233381509780884, + 0.9944189190864563, + 0.9492918252944946, + 0.9924170970916748, + 0.9667571187019348, + 0.9821245074272156, + 0.9963752031326294, + 0.9912564754486084, + 0.990851640701294, + 0.9379763007164001, + 0.8643018007278442, + 0.9939460754394531, + 0.9750210046768188, + 0.9391907453536987, + 0.9721632599830627, + 0.9803709387779236, + 0.887528657913208, + 0.8982088565826416, + 0.863335132598877, + 0.9438390731811523, + 0.913702130317688, + 0.974104106426239, + 0.9827861189842224, + 0.9112609624862671, + 0.9421620965003967, + 0.9721039533615112, + 0.9763815999031067, + 0.8823307156562805, + 0.981546938419342, + 0.9908109307289124, + 0.9816513061523438, + 0.9523004293441772, + 0.9762136936187744, + 0.8590486645698547, + 0.8789148330688477, + 0.7450559139251709, + 0.9780750274658203, + 0.9861814379692078, + 0.9468588829040527, + 0.9632684588432312, + 0.5080381035804749, + 0.38759344816207886, + 0.9810343384742737, + 0.9889212250709534, + 0.9669134616851807, + 0.9823195338249207, + 0.9283587336540222, + 0.7961477041244507, + 0.9765648245811462, + 0.9416943788528442, + 0.9919747710227966, + 0.8515219688415527, + 0.9992368221282959, + 0.9888402223587036, + 0.9446985125541687, + 0.9691312313079834, + 0.913862407207489, + 0.9544129967689514, + 0.8326265811920166, + 0.9443677663803101, + 0.7987475395202637, + 0.9190412759780884, + 0.9169807434082031, + 0.9866119623184204, + 0.9515770673751831, + 0.8060968518257141, + 0.9840064644813538, + 0.6230958700180054, + 0.9705268144607544, + 0.9957144260406494, + 0.9058505296707153, + 0.9926018714904785, + 0.6038965582847595, + 0.9730700850486755, + 0.9609211087226868, + 0.9932594299316406, + 0.9768669605255127, + 0.9145826697349548, + 0.9825996160507202, + 0.9948884844779968 + ], + "mmd": [ + 0.00861471026633287, + 0.015850958356578793, + 0.011168985170651236, + 0.012544080870102392, + 0.0403930227622622, + 0.014412037060329341, + 0.032646464798473884, + 0.006817389730260359, + 0.011241588480763953, + 0.02288264690680708, + 0.013795565817492084, + 0.015637990025116677, + 0.01586777019693053, + 0.008487530414388877, + 0.009985637072597364, + 0.02079829898480512, + 0.016078367830316243, + 0.014881184787551516, + 0.008292433259912024, + 0.010521919668421198, + 0.01915835885523387, + 0.014316713949633186, + 0.015621474676267733, + 0.018905761724313908, + 0.01470466894442135, + 0.013490490476648964, + 0.012594070475060115, + 0.009493048544520821, + 0.009926495473791541, + 0.016216025154012104, + 0.03748812800407797, + 0.01893581162382041, + 0.015191622378489589, + 0.010961317444988183, + 0.014562321510761422, + 0.00930285204774417, + 0.009570683629771759, + 0.01023162070263084, + 0.015944526053711883, + 0.022193966931672415, + 0.014124008674479493, + 0.00701235402884226, + 0.012097545179472924, + 0.03871183003279621, + 0.01036184856893585, + 0.01245642738497621, + 0.015778047502285952, + 0.015158481004360014, + 0.018789483934928075, + 0.009228166205351385, + 0.016934060405057183, + 0.017072986036436255, + 0.06629092512637202, + 0.03267826015478492, + 0.017604368955630778, + 0.011226106519970802, + 0.021845680969722392, + 0.005884412076189727, + 0.010575343915000923, + 0.02263752534523411, + 0.03880127410795908, + 0.014647104512315878, + 0.01577091581636736, + 0.021068085453773366, + 0.008908639946343166, + 0.014454257903358525, + 0.007708235540586861, + 0.007473751895319736, + 0.01697578779367237, + 0.03428144653769183, + 0.014242229095073222, + 0.023488148548916454, + 0.012774065042546301, + 0.017762771775204023, + 0.013327470834526012, + 0.015175104212474366, + 0.04110852609799254, + 0.016159519156740076, + 0.021996010214857864, + 0.013041402649269562 + ], + "wasserstein": [ + 0.15939027859748545, + 0.19767920056306879, + 0.16590432787470544, + 0.14541088616972087, + 0.30994768122481653, + 0.16518187871623816, + 0.20374154633488473, + 0.1272430576123276, + 0.16872819439257622, + 0.2868264855221191, + 0.22772281790030646, + 0.20960894426437776, + 0.24333675508432667, + 0.13350842177704125, + 0.16480392223938295, + 0.18906234350013307, + 0.21928059385522577, + 0.20929833607964102, + 0.13683644949441562, + 0.14818583986059422, + 0.22041934293620485, + 0.20129405947723794, + 0.17542537927499233, + 0.24329484918268016, + 0.1852426565183384, + 0.19322789751495761, + 0.16814322417083385, + 0.15889253730653194, + 0.16075345896635604, + 0.17026306802581712, + 0.23098168369504746, + 0.26484794061873584, + 0.2265387849697547, + 0.16687411010387346, + 0.19446466347578073, + 0.15178468529325925, + 0.14693544365141262, + 0.14783502029494097, + 0.1993589541319545, + 0.25912960735020446, + 0.19833153133818787, + 0.12915607592176745, + 0.1728180238139721, + 0.2793682596848274, + 0.17410488076615965, + 0.14335478845406693, + 0.21584624344964046, + 0.1951607052046777, + 0.2443581005087557, + 0.15270128685665038, + 0.20953142023713395, + 0.23470642394089128, + 0.326788704714795, + 0.21144523660205852, + 0.2157754898610451, + 0.18230257642021708, + 0.2640444521945328, + 0.11821254579768857, + 0.15550862039693983, + 0.2526096272298554, + 0.2501063160152313, + 0.19629777781579041, + 0.22986575849822194, + 0.21530863366929542, + 0.14011143895704886, + 0.20543652880828298, + 0.1446386059880904, + 0.13232971709253533, + 0.2243876066356031, + 0.22476518723686506, + 0.20957066181908865, + 0.27874416532393037, + 0.18719258165782984, + 0.22538776322129792, + 0.17463301533101577, + 0.17481109379300055, + 0.2950615632748853, + 0.2181881649437035, + 0.277562369456816, + 0.1532820290748621 + ] + }, + "n_perts": 80 + }, + "LinearResponse": { + "mse": 0.000659852591707022, + "r2": 0.4656556911766529, + "pearson": 0.730800811573863, + "spearman": 0.6756585862986254, + "de_corr": 0.8571043156087399, + "mmd": 0.02022720643205938, + "wasserstein": 0.21406553779618842, + "_per_pert": { + "mse": [ + 0.00020937240333296359, + 0.0004932541050948203, + 0.0003899661242030561, + 0.0014147398760542274, + 0.001287768711335957, + 0.00041565109859220684, + 0.0014939585234969854, + 0.000268590752966702, + 0.0003747372538782656, + 0.0007916538743302226, + 0.00040012525278143585, + 0.0008189975051209331, + 0.00038987546577118337, + 0.00029816414462402463, + 0.00029928734875284135, + 0.000703389523550868, + 0.00047704987809993327, + 0.00033865144359879196, + 0.0002781887596938759, + 0.00034494081046432257, + 0.0003724437556229532, + 0.0007346042548306286, + 0.0008385448600165546, + 0.00044596067164093256, + 0.00034873097320087254, + 0.0003482428437564522, + 0.00034695034264586866, + 0.0002632352989166975, + 0.00047769767115823925, + 0.0004404167702887207, + 0.001599853509105742, + 0.0004965020925737917, + 0.0003969034878537059, + 0.0004101562954019755, + 0.00046535232104361057, + 0.0002611697418615222, + 0.0004488748381845653, + 0.00021978456061333418, + 0.0002992495137732476, + 0.0005049233441241086, + 0.00035850543645210564, + 0.00021168473176658154, + 0.0004309353535063565, + 0.0006563083152286708, + 0.00022589215950574726, + 0.0007480778731405735, + 0.0006190811982378364, + 0.0004096655757166445, + 0.0003187180554959923, + 0.0003480443556327373, + 0.0007330236257985234, + 0.0003631313447840512, + 0.00859390664845705, + 0.00058763287961483, + 0.0004668178444262594, + 0.0003905901394318789, + 0.0005637231515720487, + 0.0003368728212080896, + 0.00024254870368167758, + 0.0006677373894490302, + 0.0009321564575657248, + 0.0003563944483175874, + 0.0002756761678028852, + 0.0004772970569320023, + 0.0002285419759573415, + 0.0004888479597866535, + 0.0004073601448908448, + 0.00026913287001661956, + 0.00046591495629400015, + 0.0020607549231499434, + 0.0004243199073243886, + 0.0022687087766826153, + 0.0004149610176682472, + 0.0008123211446218193, + 0.000319385202601552, + 0.0004266832838766277, + 0.0011387651320546865, + 0.0004230969352647662, + 0.0011370248394086957, + 0.000980006530880928 + ], + "r2": [ + -0.017084360122680664, + 0.7916048765182495, + -0.06926822662353516, + 0.7728122472763062, + 0.7225507497787476, + 0.6210137605667114, + 0.791404664516449, + 0.7053608894348145, + 0.6209276914596558, + 0.49691128730773926, + 0.23859959840774536, + 0.6640361547470093, + 0.5875234603881836, + 0.00811678171157837, + 0.5135325193405151, + 0.6055374145507812, + 0.6785379648208618, + 0.09972292184829712, + -0.198186993598938, + 0.19733315706253052, + -0.07841992378234863, + 0.320573091506958, + 0.4916364550590515, + 0.4828159213066101, + 0.6534050703048706, + 0.4216427803039551, + 0.4445669651031494, + 0.39567941427230835, + 0.6236287355422974, + 0.6546106338500977, + 0.7682421207427979, + 0.5168810486793518, + 0.5406587719917297, + 0.549961507320404, + 0.4075379967689514, + 0.055142104625701904, + 0.4930419921875, + 0.6605027914047241, + 0.6777218580245972, + 0.47690337896347046, + 0.45493072271347046, + -0.23363542556762695, + 0.6667611002922058, + 0.8311914205551147, + 0.4444615840911865, + 0.6118080615997314, + 0.38155806064605713, + 0.2707098126411438, + 0.4310283064842224, + 0.10775822401046753, + 0.704163670539856, + 0.22369486093521118, + 0.8024433255195618, + 0.8314540386199951, + 0.08863961696624756, + 0.255013108253479, + 0.5620079636573792, + 0.6141624450683594, + 0.07354241609573364, + 0.6831292510032654, + 0.6640448570251465, + 0.4623955488204956, + 0.5256025791168213, + 0.7700748443603516, + 0.5627676248550415, + -0.12696611881256104, + 0.3531249761581421, + -0.5047813653945923, + 0.7517486810684204, + 0.80406653881073, + 0.5021408200263977, + 0.7037575840950012, + -0.07023227214813232, + 0.6609958410263062, + 0.7322894930839539, + 0.7279813289642334, + 0.7576459646224976, + 0.35968029499053955, + 0.6526250839233398, + 0.7709511518478394 + ], + "pearson": [ + 0.5138034820556641, + 0.9208900332450867, + 0.5013998746871948, + 0.9460477232933044, + 0.926625669002533, + 0.8083833456039429, + 0.9797981977462769, + 0.8406901955604553, + 0.8030244708061218, + 0.7594410181045532, + 0.5788348913192749, + 0.8809172511100769, + 0.783146858215332, + 0.5420008301734924, + 0.7582314014434814, + 0.8008154034614563, + 0.8451504707336426, + 0.52269446849823, + 0.32313430309295654, + 0.5242363810539246, + 0.5991553068161011, + 0.5695352554321289, + 0.7175536155700684, + 0.6957017183303833, + 0.835363507270813, + 0.6816643476486206, + 0.7034434080123901, + 0.68125981092453, + 0.822049081325531, + 0.818467915058136, + 0.9482342004776001, + 0.7249173521995544, + 0.7373356223106384, + 0.7453157901763916, + 0.6464552283287048, + 0.4829002022743225, + 0.7577155232429504, + 0.8357207775115967, + 0.8252015113830566, + 0.6968631148338318, + 0.6770010590553284, + 0.5008049607276917, + 0.8326525092124939, + 0.9460806250572205, + 0.7197527885437012, + 0.8320439457893372, + 0.6891418099403381, + 0.5726849436759949, + 0.6613622307777405, + 0.5601784586906433, + 0.9012441039085388, + 0.5496571063995361, + 0.9937958121299744, + 0.9640278816223145, + 0.38293731212615967, + 0.5351413488388062, + 0.781030535697937, + 0.7837299108505249, + 0.6179688572883606, + 0.8483543395996094, + 0.8479949831962585, + 0.7031363844871521, + 0.7292076349258423, + 0.8818854093551636, + 0.7641108632087708, + 0.3457423448562622, + 0.6158385872840881, + 0.33373793959617615, + 0.889988899230957, + 0.9754258394241333, + 0.7133121490478516, + 0.929914116859436, + 0.3323092460632324, + 0.854766845703125, + 0.8625905513763428, + 0.8858919739723206, + 0.9278745055198669, + 0.6152197122573853, + 0.8667010068893433, + 0.9547058343887329 + ], + "spearman": [ + 0.3551002697750674, + 0.8752332028083006, + 0.5062283405570852, + 0.8925145106286274, + 0.8935718128929533, + 0.792958044239511, + 0.9731857742964434, + 0.7327176466794116, + 0.7422755515688879, + 0.6109911867477967, + 0.43073375968343997, + 0.7688560307140077, + 0.7029256652314162, + 0.4826749146687286, + 0.6749563747390938, + 0.8064968691242171, + 0.8116340809085203, + 0.5175743508935877, + 0.23827147306786825, + 0.4450527467631866, + 0.49655129363782335, + 0.45929787532446886, + 0.5966313426578357, + 0.6582983410745852, + 0.7928880422220106, + 0.6633843788460947, + 0.659426111856528, + 0.5687452961863241, + 0.6816125094031273, + 0.7674553303638325, + 0.9287862956965739, + 0.6377351749337936, + 0.6939198999799749, + 0.6853765623441406, + 0.6186321046580261, + 0.46948320487080125, + 0.6935735423933855, + 0.7820704655176163, + 0.7724341676085418, + 0.6184201336050333, + 0.529664755916189, + 0.5271334572833642, + 0.6894847213711803, + 0.9189160667290166, + 0.6217916094479023, + 0.7147584136896035, + 0.6316998124249531, + 0.5671401007850251, + 0.5090112132528033, + 0.5451680292920073, + 0.8761449536147928, + 0.4790990652747663, + 0.9920517575129393, + 0.9333241823310457, + 0.23214752653688162, + 0.46381490295372574, + 0.7526867226716807, + 0.716000984000246, + 0.6361008450252112, + 0.8073573938393483, + 0.7841329885332472, + 0.6250874942718735, + 0.6288450222112555, + 0.8351432867858217, + 0.7514602113650528, + 0.3678116404529101, + 0.5879786339946586, + 0.3434638128659532, + 0.8715287388821846, + 0.9652897033224257, + 0.6492152688038171, + 0.8916548664137166, + 0.37266453030588176, + 0.8250075832518958, + 0.8364779631194909, + 0.8372769263192315, + 0.9025774971443742, + 0.609701084425271, + 0.8071121422780355, + 0.9180863160215789 + ], + "de_corr": [ + 0.8400782346725464, + 0.9914580583572388, + 0.7413449287414551, + 0.9885497689247131, + 0.9731115698814392, + 0.9681177139282227, + 0.9870736598968506, + 0.9782699346542358, + 0.9791398048400879, + 0.9248965978622437, + 0.8754847645759583, + 0.9832534193992615, + 0.9064744710922241, + 0.8152456879615784, + 0.8907590508460999, + 0.9364259839057922, + 0.8897709250450134, + 0.7848345041275024, + 0.629766583442688, + 0.7820711731910706, + 0.8329855799674988, + 0.9202947020530701, + 0.9082382917404175, + 0.877228319644928, + 0.910660982131958, + 0.9573796391487122, + 0.8389540314674377, + 0.8317044377326965, + 0.944405734539032, + 0.9732319116592407, + 0.969050407409668, + 0.8457163572311401, + 0.9492238163948059, + 0.7779922485351562, + 0.7274694442749023, + 0.3676888048648834, + 0.82864910364151, + 0.9785706400871277, + 0.9349413514137268, + 0.9145183563232422, + 0.3266739547252655, + 0.1731509268283844, + 0.9493516683578491, + 0.9833021759986877, + 0.9222703576087952, + 0.9594581127166748, + 0.895564615726471, + 0.5758483409881592, + 0.9586327075958252, + 0.830090343952179, + 0.9847747087478638, + 0.8112952709197998, + 0.9977201223373413, + 0.9873412847518921, + 0.8129101991653442, + 0.8175467848777771, + 0.9162713289260864, + 0.8978613615036011, + 0.6783237457275391, + 0.9210202693939209, + 0.7652575969696045, + 0.8814684748649597, + 0.9248030781745911, + 0.9733914136886597, + 0.9390395879745483, + 0.49930518865585327, + 0.8518674969673157, + 0.41619256138801575, + 0.9679468274116516, + 0.9933686256408691, + 0.826414942741394, + 0.9752758145332336, + 0.35669320821762085, + 0.9769209027290344, + 0.9304039478302002, + 0.9810045957565308, + 0.9767888784408569, + 0.812245786190033, + 0.9756495356559753, + 0.9898675084114075 + ], + "mmd": [ + 0.00875419337673533, + 0.018901383864335486, + 0.011492099679396461, + 0.034464755788055745, + 0.041252565591309964, + 0.015188948665860513, + 0.04092113227976457, + 0.009152290670727536, + 0.014138724751968423, + 0.025848919709151263, + 0.014114231640869668, + 0.02274778585586934, + 0.016006847598730167, + 0.010728510545168213, + 0.010492992400500833, + 0.021958718787864484, + 0.01709879240528589, + 0.012592556644813513, + 0.008898535585350476, + 0.011471211308052509, + 0.012048292873706012, + 0.020088327242251625, + 0.020931044349346895, + 0.01776937962533276, + 0.014443102530596996, + 0.013375681014654028, + 0.012466334987934258, + 0.00818482788462327, + 0.016487445906067433, + 0.01573483508075424, + 0.049067608034404686, + 0.019120704453495008, + 0.016121854025002547, + 0.01209726707389236, + 0.016334373211004727, + 0.009390969488661471, + 0.01310053031946301, + 0.009434179023507072, + 0.01190225287162594, + 0.01926518244389086, + 0.01246917364323441, + 0.008709600298924758, + 0.01273073633088484, + 0.0363909765786834, + 0.009345628042090826, + 0.016699832667679493, + 0.01883854254764117, + 0.01741336400648963, + 0.016553545155983618, + 0.010839614658618824, + 0.023189381774259443, + 0.01639120708181141, + 0.1648764141565483, + 0.025404693363214603, + 0.016818986203552377, + 0.012208498967399617, + 0.020789653225791738, + 0.007904257896994782, + 0.009393915135102349, + 0.023880213252756355, + 0.03781358644071742, + 0.014734571712326283, + 0.014809083975840798, + 0.016719989281322123, + 0.009558056292536699, + 0.014532486107808662, + 0.010393870967796315, + 0.008205675031552673, + 0.01991491718804339, + 0.055293260392988874, + 0.016373491195037282, + 0.044176559316343345, + 0.0130623337711675, + 0.024023948458640643, + 0.013406675524519462, + 0.017832394796838846, + 0.04166550727997642, + 0.017570888327378076, + 0.03017051770014667, + 0.02348107622608131 + ], + "wasserstein": [ + 0.16064244934151706, + 0.20969929496721842, + 0.1731346195509874, + 0.21880669232161, + 0.3094637265628525, + 0.1663653983523223, + 0.25464071912319103, + 0.14231142083316442, + 0.19208573771346443, + 0.314121608648234, + 0.2405281543280589, + 0.2369466772209709, + 0.2502913147598597, + 0.15233191521118586, + 0.16960409749717975, + 0.21558755398439605, + 0.24135060791749918, + 0.20061193291072096, + 0.14989138933159513, + 0.17293271327237086, + 0.19211025463297143, + 0.21032085300316933, + 0.20790991197317002, + 0.23805397636642006, + 0.1893038485272561, + 0.19644391100896244, + 0.18100710963494357, + 0.1550818475776541, + 0.1975661911630335, + 0.17121191063510122, + 0.2795621061320265, + 0.27233217391565756, + 0.2336330206326581, + 0.1798729950948195, + 0.18665169204248686, + 0.15269859552670673, + 0.17746935036681927, + 0.14637146647001256, + 0.18283540746499793, + 0.2540156017667799, + 0.19211621848079935, + 0.14103494202863076, + 0.18368944337134205, + 0.27871893308602513, + 0.16607950922116757, + 0.17484437472935643, + 0.21872270460880874, + 0.19974842660947728, + 0.2298973741690009, + 0.16721705980257842, + 0.24406950531426216, + 0.22084741084055554, + 0.552503316538283, + 0.2139562806006162, + 0.22621678350845273, + 0.2052749709714062, + 0.2654241137837255, + 0.12300516015585018, + 0.15545339586408413, + 0.2672395563666953, + 0.26057486089304194, + 0.18769100590394483, + 0.2296301640479723, + 0.2103510200482025, + 0.13620993482011254, + 0.212030225573646, + 0.16161872858679963, + 0.14578356324227826, + 0.23659983680345423, + 0.3081912224849957, + 0.2205342132479629, + 0.35863110220031125, + 0.19367598930505026, + 0.2446940687218101, + 0.1766466440769793, + 0.20555768347079578, + 0.3093929980596797, + 0.2117477122679677, + 0.31810966897648063, + 0.195712653128428 + ] + }, + "n_perts": 80 + }, + "NearestPerturbationCentroid": { + "mse": 0.00034133204599129385, + "r2": 0.5863893613219261, + "pearson": 0.8470813885331154, + "spearman": 0.8051852191744894, + "de_corr": 0.9229583293199539, + "mmd": 0.017067014377718525, + "wasserstein": 0.1988901412503695, + "_per_pert": { + "mse": [ + 0.00024666020181030035, + 0.0002835692139342427, + 0.0002478808455634862, + 0.0002530887140892446, + 0.0008113104850053787, + 0.00031414139084517956, + 0.0001556011411594227, + 0.00011543891014298424, + 0.00018039382121060044, + 0.0005247522494755685, + 0.000285469664959237, + 0.00031463391496799886, + 0.0002934891090262681, + 0.0001771973620634526, + 0.0001831721019698307, + 0.0003111836267635226, + 0.00034181663068011403, + 0.0002738374751061201, + 0.00020665943156927824, + 0.00018150197865907103, + 0.000694804941304028, + 0.0003870493092108518, + 0.00042090078932233155, + 0.0005266591324470937, + 0.0002580670698080212, + 0.0002574709360487759, + 0.00020162059809081256, + 0.00024779874365776777, + 0.00015485667972825468, + 0.00047555603669025004, + 0.0004128919681534171, + 0.00033290308783762157, + 0.00024150818353518844, + 0.0003065782075282186, + 0.0005201814346946776, + 0.00018012113287113607, + 0.00023756151495035738, + 0.00021429650951176882, + 0.00037766972673125565, + 0.0005346401012502611, + 0.00046765425940975547, + 0.0001525706029497087, + 0.00023469510779250413, + 0.00045847598812542856, + 0.00021582734188996255, + 0.0004265247262082994, + 0.0004218204121571034, + 0.00026694839471019804, + 0.0003895766567438841, + 0.00019685173174366355, + 0.0003251571033615619, + 0.0004144203558098525, + 0.0006686582928523421, + 0.0001693940139375627, + 0.00026813967269845307, + 0.00025309206102974713, + 0.0006148631800897419, + 0.0001430534030077979, + 0.0003080190217588097, + 0.0005426203715614974, + 0.0007035515154711902, + 0.000473168445751071, + 0.0003781112318392843, + 0.0006829269113950431, + 0.00024176001898013055, + 0.0003351293853484094, + 0.000161615593242459, + 0.00021044899767730385, + 0.0003129686519969255, + 0.0003514640557114035, + 0.0004350198432803154, + 0.0005762679502367973, + 0.00031753379153087735, + 0.00042344362009316683, + 0.0002516146923881024, + 0.00014651250967290252, + 0.0007338344003073871, + 0.00031683166162110865, + 0.00047412680578418076, + 0.00015693652676418424 + ], + "r2": [ + -0.1982201337814331, + 0.8801947236061096, + 0.3203226923942566, + 0.9593574404716492, + 0.8252034783363342, + 0.7135692834854126, + 0.9782740473747253, + 0.8733656406402588, + 0.8175193071365356, + 0.6665247678756714, + 0.45677846670150757, + 0.8709328770637512, + 0.6894972324371338, + 0.41052907705307007, + 0.7022684812545776, + 0.8254874348640442, + 0.7696654796600342, + 0.27202552556991577, + 0.10989713668823242, + 0.5776503086090088, + -1.011824369430542, + 0.6420226097106934, + 0.7448309659957886, + 0.3892292380332947, + 0.7435135841369629, + 0.5723954439163208, + 0.6772255301475525, + 0.43111783266067505, + 0.8779906034469604, + 0.6270532608032227, + 0.9401876926422119, + 0.6760702133178711, + 0.7204996347427368, + 0.6636111736297607, + 0.33773231506347656, + 0.34835904836654663, + 0.7316986322402954, + 0.6689801216125488, + 0.5932667255401611, + 0.44611722230911255, + 0.28898149728775024, + 0.11086416244506836, + 0.8185121417045593, + 0.8820757269859314, + 0.4692140221595764, + 0.7786681652069092, + 0.5786152482032776, + 0.5247761011123657, + 0.304532527923584, + 0.4953535795211792, + 0.8687719106674194, + 0.11404889822006226, + 0.9846288561820984, + 0.9514141082763672, + 0.47651559114456177, + 0.517268180847168, + 0.5222740173339844, + 0.8361536860466003, + -0.17653298377990723, + 0.742502748966217, + 0.7464355230331421, + 0.2862473726272583, + 0.3493272662162781, + 0.6710180044174194, + 0.5374797582626343, + 0.2274090051651001, + 0.7433596849441528, + -0.1766669750213623, + 0.8332423567771912, + 0.9665833115577698, + 0.48958641290664673, + 0.9247523546218872, + 0.1810438632965088, + 0.8232852220535278, + 0.7890951037406921, + 0.9065954685211182, + 0.843824028968811, + 0.5205032825469971, + 0.855148434638977, + 0.9633204936981201 + ], + "pearson": [ + 0.6622630953788757, + 0.9482947587966919, + 0.7638598680496216, + 0.9803251028060913, + 0.9218734502792358, + 0.8484968543052673, + 0.9895062446594238, + 0.9400838613510132, + 0.952268123626709, + 0.8506721258163452, + 0.7825549244880676, + 0.9471822381019592, + 0.862853467464447, + 0.791928231716156, + 0.8924075961112976, + 0.9179850220680237, + 0.9185068011283875, + 0.7556659579277039, + 0.599132776260376, + 0.7923889756202698, + 0.7520980834960938, + 0.8279823064804077, + 0.8709278702735901, + 0.7238227725028992, + 0.8980642557144165, + 0.8512991666793823, + 0.8481652140617371, + 0.8245766162872314, + 0.946467399597168, + 0.927772581577301, + 0.9711289405822754, + 0.934574544429779, + 0.8855801224708557, + 0.8632384538650513, + 0.6815758943557739, + 0.7021403908729553, + 0.8993551731109619, + 0.9136989116668701, + 0.861723005771637, + 0.823276937007904, + 0.6200095415115356, + 0.615280032157898, + 0.9335994124412537, + 0.9592396020889282, + 0.8497616648674011, + 0.9243088364601135, + 0.8281819820404053, + 0.7785792946815491, + 0.7988719940185547, + 0.8236978054046631, + 0.9329332709312439, + 0.7214899063110352, + 0.9955779910087585, + 0.9781326055526733, + 0.7653619647026062, + 0.7866364121437073, + 0.7645065188407898, + 0.9549956321716309, + 0.7164408564567566, + 0.9000207185745239, + 0.8839771747589111, + 0.7555385828018188, + 0.7390056252479553, + 0.8930636644363403, + 0.790571928024292, + 0.6510560512542725, + 0.8938482403755188, + 0.7096249461174011, + 0.9432356953620911, + 0.9882639646530151, + 0.7792956829071045, + 0.9726700186729431, + 0.536105215549469, + 0.9133987426757812, + 0.9362773895263672, + 0.9594414234161377, + 0.9417586922645569, + 0.7996209859848022, + 0.9248581528663635, + 0.9815547466278076 + ], + "spearman": [ + 0.5572901668225416, + 0.9144513426128354, + 0.71560855990214, + 0.9614981358745339, + 0.8917783314445829, + 0.8364303576075892, + 0.9839011150942417, + 0.880348268587067, + 0.9081236870309217, + 0.7932723088655544, + 0.6856410784102696, + 0.881486195871549, + 0.7742467560616889, + 0.7080133665033417, + 0.8519049944762485, + 0.9030076237519059, + 0.8813226448306613, + 0.7292154758038689, + 0.48943340185835044, + 0.6810150667537667, + 0.7162727825681956, + 0.7467115776778944, + 0.7734659653664913, + 0.7344892566223141, + 0.8778268774567194, + 0.8247502221875554, + 0.813524007881002, + 0.757526047881512, + 0.904790315697579, + 0.8888554637138659, + 0.9591585597896399, + 0.8730197302549325, + 0.8225327446331862, + 0.8297972869493218, + 0.6309029312257327, + 0.7045173666293417, + 0.8163245070811267, + 0.8789276827319206, + 0.794712188178047, + 0.787976251994063, + 0.46576784044196007, + 0.6591360432840108, + 0.8731878937969735, + 0.9424590091147523, + 0.7745397781349445, + 0.8747203451800861, + 0.8010643822660956, + 0.7602120460530114, + 0.6688201417050353, + 0.7908854307213578, + 0.9165395999786025, + 0.6489359327339832, + 0.9933628448407111, + 0.9634811393702848, + 0.6505098741274685, + 0.734998935749734, + 0.7517218234304558, + 0.9383035925758982, + 0.7301091880272971, + 0.8622200020550004, + 0.8301324155331039, + 0.682158446045421, + 0.6762637900659475, + 0.8475465708866426, + 0.7424297771074443, + 0.6224288876072219, + 0.836573888143472, + 0.6762609010652252, + 0.928051132012783, + 0.9836085074021267, + 0.7376251784062946, + 0.9515502653875664, + 0.6368560759528401, + 0.9029208097302023, + 0.9305236636309159, + 0.9417113609278402, + 0.9174386518596629, + 0.7721571325392831, + 0.8697788954447239, + 0.963754695938674 + ], + "de_corr": [ + 0.8233381509780884, + 0.9944189190864563, + 0.9492918252944946, + 0.9924170970916748, + 0.9667571187019348, + 0.9821245074272156, + 0.9963752031326294, + 0.9912564754486084, + 0.990851640701294, + 0.9379763007164001, + 0.8643018007278442, + 0.9939460754394531, + 0.9750210046768188, + 0.9391907453536987, + 0.9721632599830627, + 0.9803709387779236, + 0.887528657913208, + 0.8982088565826416, + 0.863335132598877, + 0.9438390731811523, + 0.913702130317688, + 0.974104106426239, + 0.9827861189842224, + 0.9112609624862671, + 0.9421620965003967, + 0.9721039533615112, + 0.9763815999031067, + 0.8823307156562805, + 0.981546938419342, + 0.9908109307289124, + 0.9816513061523438, + 0.9523004293441772, + 0.9762136936187744, + 0.8590486645698547, + 0.8789148330688477, + 0.7450559139251709, + 0.9780750274658203, + 0.9861814379692078, + 0.9468588829040527, + 0.9632684588432312, + 0.5080381035804749, + 0.38759344816207886, + 0.9810343384742737, + 0.9889212250709534, + 0.9669134616851807, + 0.9823195338249207, + 0.9283587336540222, + 0.7961477041244507, + 0.9765648245811462, + 0.9416943788528442, + 0.9919747710227966, + 0.8515219688415527, + 0.9992368221282959, + 0.9888402223587036, + 0.9446985125541687, + 0.9691312313079834, + 0.913862407207489, + 0.9544129967689514, + 0.8326265811920166, + 0.9443677663803101, + 0.7987475395202637, + 0.9190412759780884, + 0.9169807434082031, + 0.9866119623184204, + 0.9515770673751831, + 0.8060968518257141, + 0.9840064644813538, + 0.6230958700180054, + 0.9705268144607544, + 0.9957144260406494, + 0.9058505296707153, + 0.9926018714904785, + 0.6038965582847595, + 0.9730700850486755, + 0.9609211087226868, + 0.9932594299316406, + 0.9768669605255127, + 0.9145826697349548, + 0.9825996160507202, + 0.9948884844779968 + ], + "mmd": [ + 0.00861471026633287, + 0.015850958356578793, + 0.011168985170651236, + 0.012544080870102392, + 0.0403930227622622, + 0.014412037060329341, + 0.032646464798473884, + 0.006817389730260359, + 0.011241588480763953, + 0.02288264690680708, + 0.013795565817492084, + 0.015637990025116677, + 0.01586777019693053, + 0.008487530414388877, + 0.009985637072597364, + 0.02079829898480512, + 0.016078367830316243, + 0.014881184787551516, + 0.008292433259912024, + 0.010521919668421198, + 0.01915835885523387, + 0.014316713949633186, + 0.015621474676267733, + 0.018905761724313908, + 0.01470466894442135, + 0.013490490476648964, + 0.012594070475060115, + 0.009493048544520821, + 0.009926495473791541, + 0.016216025154012104, + 0.03748812800407797, + 0.01893581162382041, + 0.015191622378489589, + 0.010961317444988183, + 0.014562321510761422, + 0.00930285204774417, + 0.009570683629771759, + 0.01023162070263084, + 0.015944526053711883, + 0.022193966931672415, + 0.014124008674479493, + 0.00701235402884226, + 0.012097545179472924, + 0.03871183003279621, + 0.01036184856893585, + 0.01245642738497621, + 0.015778047502285952, + 0.015158481004360014, + 0.018789483934928075, + 0.009228166205351385, + 0.016934060405057183, + 0.017072986036436255, + 0.06629092512637202, + 0.03267826015478492, + 0.017604368955630778, + 0.011226106519970802, + 0.021845680969722392, + 0.005884412076189727, + 0.010575343915000923, + 0.02263752534523411, + 0.03880127410795908, + 0.014647104512315878, + 0.01577091581636736, + 0.021068085453773366, + 0.008908639946343166, + 0.014454257903358525, + 0.007708235540586861, + 0.007473751895319736, + 0.01697578779367237, + 0.03428144653769183, + 0.014242229095073222, + 0.023488148548916454, + 0.012774065042546301, + 0.017762771775204023, + 0.013327470834526012, + 0.015175104212474366, + 0.04110852609799254, + 0.016159519156740076, + 0.021996010214857864, + 0.013041402649269562 + ], + "wasserstein": [ + 0.15939027859748545, + 0.19767920056306879, + 0.16590432787470544, + 0.14541088616972087, + 0.30994768122481653, + 0.16518187871623816, + 0.20374154633488473, + 0.1272430576123276, + 0.16872819439257622, + 0.2868264855221191, + 0.22772281790030646, + 0.20960894426437776, + 0.24333675508432667, + 0.13350842177704125, + 0.16480392223938295, + 0.18906234350013307, + 0.21928059385522577, + 0.20929833607964102, + 0.13683644949441562, + 0.14818583986059422, + 0.22041934293620485, + 0.20129405947723794, + 0.17542537927499233, + 0.24329484918268016, + 0.1852426565183384, + 0.19322789751495761, + 0.16814322417083385, + 0.15889253730653194, + 0.16075345896635604, + 0.17026306802581712, + 0.23098168369504746, + 0.26484794061873584, + 0.2265387849697547, + 0.16687411010387346, + 0.19446466347578073, + 0.15178468529325925, + 0.14693544365141262, + 0.14783502029494097, + 0.1993589541319545, + 0.25912960735020446, + 0.19833153133818787, + 0.12915607592176745, + 0.1728180238139721, + 0.2793682596848274, + 0.17410488076615965, + 0.14335478845406693, + 0.21584624344964046, + 0.1951607052046777, + 0.2443581005087557, + 0.15270128685665038, + 0.20953142023713395, + 0.23470642394089128, + 0.326788704714795, + 0.21144523660205852, + 0.2157754898610451, + 0.18230257642021708, + 0.2640444521945328, + 0.11821254579768857, + 0.15550862039693983, + 0.2526096272298554, + 0.2501063160152313, + 0.19629777781579041, + 0.22986575849822194, + 0.21530863366929542, + 0.14011143895704886, + 0.20543652880828298, + 0.1446386059880904, + 0.13232971709253533, + 0.2243876066356031, + 0.22476518723686506, + 0.20957066181908865, + 0.27874416532393037, + 0.18719258165782984, + 0.22538776322129792, + 0.17463301533101577, + 0.17481109379300055, + 0.2950615632748853, + 0.2181881649437035, + 0.277562369456816, + 0.1532820290748621 + ] + }, + "n_perts": 80 + }, + "EndpointMLP": { + "mse": 0.0003406599750633177, + "r2": 0.5871598266065121, + "pearson": 0.8471646890044212, + "spearman": 0.8052958114709275, + "de_corr": 0.9229515738785267, + "mmd": 0.017055392492700782, + "wasserstein": 0.19882559091841162, + "_per_pert": { + "mse": [ + 0.0002451569016557187, + 0.0002829641161952168, + 0.00024796664365567267, + 0.0002532804210204631, + 0.0008103779400698841, + 0.00031305133597925305, + 0.00015439529670402408, + 0.00011577622353797778, + 0.00017975654918700457, + 0.0005224926862865686, + 0.0002835539635270834, + 0.0003153712605126202, + 0.0002925387234427035, + 0.00017692046822048724, + 0.00018229828856419772, + 0.00031084546935744584, + 0.0003421083965804428, + 0.00027344899717718363, + 0.00020640023285523057, + 0.00018145015928894281, + 0.0006929700030013919, + 0.0003858937125187367, + 0.00042003183625638485, + 0.0005249707610346377, + 0.00025629528681747615, + 0.00025687640300020576, + 0.00020141601271461695, + 0.00024762540124356747, + 0.00015393098874483258, + 0.00047484523383900523, + 0.000412393274018541, + 0.0003312125336378813, + 0.00024105295597109944, + 0.0003058855654671788, + 0.0005189260118640959, + 0.0001801397738745436, + 0.00023799380869604647, + 0.00021429128537420183, + 0.00037747068563476205, + 0.0005325409583747387, + 0.0004665703163482249, + 0.00015260525105986744, + 0.00023390456044580787, + 0.00045639611198566854, + 0.00021505850600078702, + 0.0004254328378010541, + 0.0004213270731270313, + 0.00026760270702652633, + 0.00038825388764962554, + 0.0001974678598344326, + 0.0003239883226342499, + 0.00041435231105424464, + 0.0006666473345831037, + 0.0001693216327112168, + 0.0002679058234207332, + 0.0002528163022361696, + 0.0006145455408841372, + 0.00014397045015357435, + 0.0003069027152378112, + 0.0005418625078164041, + 0.0007006080122664571, + 0.0004735655675176531, + 0.0003761783882509917, + 0.0006832035724073648, + 0.00024111160018946975, + 0.00033473531948402524, + 0.00016173413314390928, + 0.00021033933444414288, + 0.0003120499604847282, + 0.00034852439421229064, + 0.00043340359115973115, + 0.0005776165635325015, + 0.00031696123187430203, + 0.0004206656594760716, + 0.0002525485760997981, + 0.00014610865036956966, + 0.0007339142612181604, + 0.0003167137037962675, + 0.0004724411992356181, + 0.00015652566798962653 + ], + "r2": [ + -0.19091784954071045, + 0.8804503679275513, + 0.3200874328613281, + 0.9593266248703003, + 0.8254044055938721, + 0.7145631909370422, + 0.9784424304962158, + 0.8729954957962036, + 0.8181639313697815, + 0.6679607629776001, + 0.4604237675666809, + 0.8706304430961609, + 0.6905027031898499, + 0.41145020723342896, + 0.7036888599395752, + 0.8256770372390747, + 0.7694688439369202, + 0.2730581760406494, + 0.11101353168487549, + 0.5777708292007446, + -1.0065109729766846, + 0.6430914402008057, + 0.745357871055603, + 0.39118731021881104, + 0.745274543762207, + 0.5733829140663147, + 0.6775529384613037, + 0.4315156936645508, + 0.8787199258804321, + 0.6276107430458069, + 0.9402598738670349, + 0.6777153015136719, + 0.7210264205932617, + 0.6643712520599365, + 0.33933067321777344, + 0.34829163551330566, + 0.731210470199585, + 0.6689881086349487, + 0.5934811234474182, + 0.4482918381690979, + 0.29062962532043457, + 0.11066210269927979, + 0.819123387336731, + 0.8826106786727905, + 0.4711049199104309, + 0.779234766960144, + 0.5791079998016357, + 0.523611307144165, + 0.30689382553100586, + 0.4937741160392761, + 0.8692436218261719, + 0.11419421434402466, + 0.9846751093864441, + 0.951434850692749, + 0.47697222232818604, + 0.517794132232666, + 0.5225207805633545, + 0.8351033926010132, + -0.17226898670196533, + 0.742862343788147, + 0.7474963665008545, + 0.28564828634262085, + 0.352653443813324, + 0.6708847284317017, + 0.5387202501296997, + 0.2283174991607666, + 0.7431714534759521, + -0.17605376243591309, + 0.8337318897247314, + 0.9668627977371216, + 0.4914829134941101, + 0.9245762228965759, + 0.18252050876617432, + 0.8244445323944092, + 0.7883123755455017, + 0.9068529605865479, + 0.84380704164505, + 0.5206819176673889, + 0.8556634187698364, + 0.963416576385498 + ], + "pearson": [ + 0.6628715395927429, + 0.9481897354125977, + 0.7639820575714111, + 0.9802577495574951, + 0.922028660774231, + 0.8490656614303589, + 0.9895695447921753, + 0.9396947026252747, + 0.9522976875305176, + 0.8510151505470276, + 0.7834555506706238, + 0.9469543695449829, + 0.8631741404533386, + 0.791841447353363, + 0.8927629590034485, + 0.9180622696876526, + 0.9182173609733582, + 0.755874752998352, + 0.5993704199790955, + 0.7923566699028015, + 0.751682460308075, + 0.8283665180206299, + 0.8710699677467346, + 0.724395215511322, + 0.8984379172325134, + 0.851213812828064, + 0.8481504917144775, + 0.8243175745010376, + 0.946657657623291, + 0.9277622699737549, + 0.9711458086967468, + 0.9348112940788269, + 0.8857949376106262, + 0.8631748557090759, + 0.6818594336509705, + 0.7017205357551575, + 0.8990825414657593, + 0.9132640957832336, + 0.8616408109664917, + 0.8232053518295288, + 0.6206115484237671, + 0.6156358122825623, + 0.933754026889801, + 0.9593169689178467, + 0.8496901988983154, + 0.9243282675743103, + 0.8283736109733582, + 0.7776526212692261, + 0.79877769947052, + 0.8237152099609375, + 0.933186948299408, + 0.7211769819259644, + 0.9955801367759705, + 0.9781747460365295, + 0.7655885815620422, + 0.7866921424865723, + 0.7645494341850281, + 0.9546882510185242, + 0.7166984677314758, + 0.9001299738883972, + 0.8844075798988342, + 0.75528484582901, + 0.7392101287841797, + 0.8930581212043762, + 0.7911680340766907, + 0.6508437991142273, + 0.8938544392585754, + 0.7092456817626953, + 0.9431127309799194, + 0.9883258938789368, + 0.7798099517822266, + 0.9725818634033203, + 0.5368213057518005, + 0.9140937328338623, + 0.9362403750419617, + 0.9595828056335449, + 0.9418078660964966, + 0.799859881401062, + 0.9251397848129272, + 0.9816086888313293 + ], + "spearman": [ + 0.5584368316092078, + 0.9142329620582406, + 0.7160659460164865, + 0.9613920738480185, + 0.8920277875069469, + 0.8368721257180314, + 0.9840161418730415, + 0.8792760728190182, + 0.9080878370219592, + 0.7935631953907987, + 0.6866569716642429, + 0.8810776767694192, + 0.7750106102526525, + 0.7075371613842903, + 0.8527601296900325, + 0.9030069907517477, + 0.8810898267724567, + 0.7296224304056076, + 0.489944307986077, + 0.6812548628137157, + 0.7159651594912898, + 0.7473076793269198, + 0.7737006614251652, + 0.7350175552543887, + 0.8783641745910437, + 0.824760867690217, + 0.8134063058515764, + 0.7569003347250837, + 0.9052531528132883, + 0.8890234967558741, + 0.9590625597656398, + 0.873687363921841, + 0.822863596715899, + 0.829755415938854, + 0.6314253048563262, + 0.7044369261092314, + 0.8160689320732588, + 0.8782855460713865, + 0.794879859719965, + 0.7878864829716208, + 0.4662944815736204, + 0.6600351795087949, + 0.8736703329175832, + 0.9425338696334674, + 0.774704100176025, + 0.8747426696856673, + 0.8011428157857038, + 0.7589627182406795, + 0.6683679860919965, + 0.7908602877150719, + 0.916877267313146, + 0.6481777065444266, + 0.9933780248445062, + 0.963381272345318, + 0.6508267282066821, + 0.7351547812886954, + 0.7518014464503616, + 0.9378173029543257, + 0.7300297570074392, + 0.8621180995295248, + 0.8303979980994995, + 0.6816626979156745, + 0.6768231942057985, + 0.8474011998502999, + 0.7433034223258556, + 0.6222626890656723, + 0.8365585626396407, + 0.675619208404802, + 0.9280245265061317, + 0.9836635919158979, + 0.7380711180177795, + 0.9513830393457597, + 0.6376320771471313, + 0.9036664914166228, + 0.9306986521746631, + 0.9419280734820182, + 0.9174206458551615, + 0.7721668615417153, + 0.8702459085614772, + 0.9638748129687033 + ], + "de_corr": [ + 0.8236453533172607, + 0.9944010972976685, + 0.9493169784545898, + 0.9923644065856934, + 0.9667429327964783, + 0.9822631478309631, + 0.9963963627815247, + 0.9912626147270203, + 0.9909237623214722, + 0.9380022287368774, + 0.8647262454032898, + 0.9939004182815552, + 0.9751684069633484, + 0.9392239451408386, + 0.9720357060432434, + 0.9804102182388306, + 0.8873279094696045, + 0.8980666399002075, + 0.8632408976554871, + 0.9438132643699646, + 0.9133414626121521, + 0.974026620388031, + 0.9827566146850586, + 0.9113296866416931, + 0.9422133564949036, + 0.972223162651062, + 0.9763030409812927, + 0.8821622133255005, + 0.9815057516098022, + 0.9908207058906555, + 0.9816794991493225, + 0.9523823857307434, + 0.9763120412826538, + 0.8586507439613342, + 0.8793909549713135, + 0.7441322207450867, + 0.9780510067939758, + 0.9859546422958374, + 0.9468039274215698, + 0.9632728099822998, + 0.5075690150260925, + 0.3869283199310303, + 0.9809654355049133, + 0.9890062212944031, + 0.9668359756469727, + 0.9823259115219116, + 0.9285085797309875, + 0.7959281802177429, + 0.9764617085456848, + 0.9414841532707214, + 0.9919409155845642, + 0.8515597581863403, + 0.9992383122444153, + 0.988921582698822, + 0.9447911381721497, + 0.9689820408821106, + 0.9137360453605652, + 0.9540570974349976, + 0.8328443169593811, + 0.9444842338562012, + 0.7999925017356873, + 0.9188260436058044, + 0.9168280363082886, + 0.9864639639854431, + 0.9518573880195618, + 0.8060646653175354, + 0.9839274287223816, + 0.6232530474662781, + 0.9704629182815552, + 0.9957499504089355, + 0.9061139225959778, + 0.9925869107246399, + 0.6038426160812378, + 0.9734526872634888, + 0.9609492421150208, + 0.9932650327682495, + 0.9768959879875183, + 0.9148581027984619, + 0.9826741814613342, + 0.9949469566345215 + ], + "mmd": [ + 0.008592572111500552, + 0.01583778380696166, + 0.011166188059412052, + 0.012565370504396212, + 0.040354258290568046, + 0.014392359022403323, + 0.0326176286547305, + 0.006818267295846336, + 0.011232094989096497, + 0.022839813851338597, + 0.013769709215589399, + 0.01564794098226585, + 0.015854934371383078, + 0.008479852755235884, + 0.00997117513264234, + 0.020793888150104523, + 0.01608683553919621, + 0.014866556705694478, + 0.008287362074898441, + 0.010521602245037287, + 0.019132536329887095, + 0.01429591229622773, + 0.015600712198181155, + 0.01889091837561374, + 0.014673131396402805, + 0.013470507012409483, + 0.012585225530219568, + 0.009482579446779327, + 0.009913108412193838, + 0.016199254909137206, + 0.03746839625766896, + 0.018907725790868968, + 0.015181339469893196, + 0.01094541585680553, + 0.014546885470368864, + 0.009298483083221232, + 0.009578040758489448, + 0.01022366667831176, + 0.015938331657148752, + 0.022162149321894642, + 0.014097144483408486, + 0.007020470084501018, + 0.01208382166337818, + 0.03868865631682972, + 0.010346415562236144, + 0.012442952912122274, + 0.015768403163165856, + 0.015167501859085664, + 0.018760761950598392, + 0.00923762883908541, + 0.0169180046266727, + 0.01707205147992641, + 0.06628409646662226, + 0.03268278307529948, + 0.017593058294775643, + 0.011228943414138493, + 0.021844837109567194, + 0.005898460814273587, + 0.010557304067710493, + 0.02260788938568048, + 0.03876828088517714, + 0.014646965930447786, + 0.015743741906847464, + 0.021077680318573266, + 0.008892937008624702, + 0.014444899679759593, + 0.007709419470510515, + 0.007464978882816609, + 0.0169670933233933, + 0.03424505240493503, + 0.01421850549783854, + 0.023488415645581906, + 0.012758988671291105, + 0.017722458882563963, + 0.013343791734074184, + 0.015163128840328, + 0.041113880531238434, + 0.016162182246443613, + 0.021973714531461463, + 0.013033583445053454 + ], + "wasserstein": [ + 0.15927880699330216, + 0.19755147321324665, + 0.16583885378333058, + 0.1454632929170672, + 0.30974732187004117, + 0.16512614522713853, + 0.20365949122843527, + 0.12725214878254051, + 0.16862660454297485, + 0.286649932340666, + 0.22755546682804367, + 0.20968361447617312, + 0.2433444749649729, + 0.13350880414915287, + 0.16472592020859922, + 0.1891404863394678, + 0.2193880689867119, + 0.20923723819609985, + 0.13680359849676843, + 0.1480977277839393, + 0.22013730147449795, + 0.20120322472575766, + 0.1752495435269127, + 0.24315923454494764, + 0.18508663134909656, + 0.19325272786118744, + 0.16819908925733654, + 0.158873750811319, + 0.1607213555387905, + 0.17030760661192848, + 0.23097020061405857, + 0.26477848576462554, + 0.22648715750550774, + 0.16678573155509852, + 0.19433864550119395, + 0.15169737073666564, + 0.1469980540410782, + 0.14791047779128724, + 0.19926189560921106, + 0.25893719703157886, + 0.1982072025794242, + 0.12916563664647113, + 0.1727768204496147, + 0.27917370524150603, + 0.1739228260948291, + 0.14318717936749517, + 0.21568415387584072, + 0.19524417372230246, + 0.24421098705212302, + 0.1526884422135434, + 0.20935953609213256, + 0.23456020112727238, + 0.32675985104954924, + 0.2115601158638255, + 0.21567680995526572, + 0.18228307007982028, + 0.2638983114455887, + 0.11832243087259264, + 0.15539130997582545, + 0.2525637135937998, + 0.25003620723528713, + 0.1962838102320079, + 0.2297744006617648, + 0.2153004541427901, + 0.14009001468275098, + 0.2053067531609029, + 0.14470361316491437, + 0.1323018533724196, + 0.2244121580603714, + 0.2246111410431569, + 0.2093653204861599, + 0.27860443947722635, + 0.18709217928127458, + 0.22514618392944782, + 0.17475729445989546, + 0.17465363343148682, + 0.2950911047051416, + 0.21820104077647653, + 0.2773898310121564, + 0.15325420967772463 + ] + }, + "n_perts": 80 + }, + "Random": { + "mse": 0.004570414199406514, + "r2": -3.855829080939293, + "pearson": 0.04383660850580782, + "spearman": 0.03590088170271198, + "de_corr": 0.045302987471222876, + "mmd": 0.07370423095827724, + "wasserstein": 0.3674525313352839, + "_per_pert": { + "mse": [ + 0.006647543981671333, + 0.005887101404368877, + 0.008411294780671597, + 0.007480265107005835, + 0.004157226532697678, + 0.0013040736084803939, + 0.009739981032907963, + 0.0021522727329283953, + 0.004157081712037325, + 0.0047319079749286175, + 0.002457860391587019, + 0.007022047881036997, + 0.0049921320751309395, + 0.0009228716953657568, + 0.0010590632446110249, + 0.011609440669417381, + 0.005191118456423283, + 0.0007321974844671786, + 0.000921351951546967, + 0.007144860457628965, + 0.0022514229640364647, + 0.001953399507328868, + 0.005162539426237345, + 0.003987716045230627, + 0.001288212020881474, + 0.005121036432683468, + 0.0017310412367805839, + 0.0028983743395656347, + 0.006063160020858049, + 0.007058603223413229, + 0.008321576751768589, + 0.005358923692256212, + 0.0078752301633358, + 0.0030519168358296156, + 0.002866151509806514, + 0.004637082107365131, + 0.001677048858255148, + 0.0010902169160544872, + 0.005489823874086142, + 0.0030639739707112312, + 0.001556138857267797, + 0.0011088679311797023, + 0.003925140015780926, + 0.005548982415348291, + 0.000844137801323086, + 0.0044260635040700436, + 0.002326725050806999, + 0.0008745187660679221, + 0.001267372746951878, + 0.0012252229498699307, + 0.008209612220525742, + 0.0010155851487070322, + 0.04107152670621872, + 0.0035108549054712057, + 0.001100932597182691, + 0.0015187911922112107, + 0.0033025427255779505, + 0.00554728414863348, + 0.0011073772329837084, + 0.0025435336865484715, + 0.0046948217786848545, + 0.0025769853964447975, + 0.008849868550896645, + 0.003339171875268221, + 0.002806962002068758, + 0.004540368914604187, + 0.0016704525332897902, + 0.007994311861693859, + 0.0035793003626167774, + 0.007685457356274128, + 0.0011718485038727522, + 0.009129468351602554, + 0.0013741027796640992, + 0.004828084725886583, + 0.009334889240562916, + 0.005703842267394066, + 0.004972162190824747, + 0.0022107684053480625, + 0.005789923947304487, + 0.0036799912340939045 + ], + "r2": [ + -31.292282104492188, + -1.487243413925171, + -22.06336212158203, + -0.20122790336608887, + 0.10432708263397217, + -0.18904006481170654, + -0.35995399951934814, + -1.3610045909881592, + -3.2051730155944824, + -2.0070836544036865, + -3.677074432373047, + -1.8805396556854248, + -4.281527996063232, + -2.070056915283203, + -0.7214221954345703, + -5.510603427886963, + -2.4980556964874268, + -0.9464870691299438, + -2.9683547019958496, + -15.625875473022461, + -5.51904821395874, + -0.8066760301589966, + -2.129763603210449, + -3.6245851516723633, + -0.28032171726226807, + -7.504952430725098, + -1.7712247371673584, + -5.653923034667969, + -3.7770776748657227, + -4.535589218139648, + -0.20548009872436523, + -4.214475631713867, + -8.114102363586426, + -2.348674774169922, + -2.649033784866333, + -15.7760009765625, + -0.8940542936325073, + -0.6840386390686035, + -4.91229248046875, + -2.174252510070801, + -1.365943193435669, + -5.462149143218994, + -2.035280227661133, + -0.4272499084472656, + -1.0759944915771484, + -1.296769142150879, + -1.3243227005004883, + -0.5568256378173828, + -1.2624986171722412, + -2.1409645080566406, + -2.3132643699645996, + -1.171126365661621, + 0.05584794282913208, + -0.00699007511138916, + -1.1493315696716309, + -1.8968462944030762, + -1.5659539699554443, + -5.353586196899414, + -3.229823112487793, + -0.20701873302459717, + -0.6920437812805176, + -2.887263298034668, + -14.22929859161377, + -0.6085575819015503, + -4.370105266571045, + -9.467144966125488, + -1.6526250839233398, + -43.697975158691406, + -0.907142162322998, + 0.26927828788757324, + -0.3749428987503052, + -0.19210350513458252, + -2.54396915435791, + -1.0148940086364746, + -6.824558734893799, + -2.636308193206787, + -0.05818450450897217, + -2.3458025455474854, + -0.7688922882080078, + 0.13990598917007446 + ], + "pearson": [ + -0.20589673519134521, + -0.04912915453314781, + -0.17215953767299652, + -0.11280805617570877, + 0.42215317487716675, + 0.5063994526863098, + -0.33061185479164124, + 0.3224865198135376, + -0.3463473320007324, + 0.11055134236812592, + -0.23863480985164642, + -0.11885271221399307, + 0.041105687618255615, + -0.1102423220872879, + 0.12255921959877014, + -0.1631145477294922, + -0.03508380427956581, + 0.27608683705329895, + -0.32377132773399353, + 0.08569736033678055, + -0.18390727043151855, + 0.06238039210438728, + -0.2056450992822647, + 0.2615281045436859, + 0.2500225007534027, + 0.12598486244678497, + -0.17105147242546082, + 0.3489246666431427, + 0.19657115638256073, + -0.04498099908232689, + 0.2612016201019287, + -0.13688945770263672, + -0.1727815419435501, + 0.10675923526287079, + -0.18124715983867645, + 0.0052176592871546745, + -0.20436039566993713, + 0.12995079159736633, + -0.009443649090826511, + -0.21220959722995758, + -0.10272880643606186, + 0.030981453135609627, + -0.027137352153658867, + 0.23472602665424347, + 0.18289971351623535, + 0.05331848934292793, + 0.09874836355447769, + 0.03985300660133362, + 0.08344502002000809, + 0.022404294461011887, + 0.1364443451166153, + 0.10810792446136475, + 0.26106971502304077, + 0.3301801383495331, + -0.13146694004535675, + 0.09178832173347473, + 0.017772506922483444, + 0.08196956664323807, + -0.19513081014156342, + 0.31531012058258057, + 0.12687456607818604, + 0.11849027872085571, + 0.03124854899942875, + 0.3377266526222229, + 0.02338779903948307, + 0.22752687335014343, + -0.254217267036438, + -0.06700445711612701, + -0.2774626612663269, + 0.6161667108535767, + 0.2671074867248535, + 0.19863183796405792, + 0.259273886680603, + -0.1474999189376831, + 0.14214354753494263, + -0.13336877524852753, + 0.10000039637088776, + -0.4508238732814789, + 0.4063456952571869, + 0.44341450929641724 + ], + "spearman": [ + -0.21253047213261803, + -0.01759778539944635, + -0.0781530180382545, + -0.07772661143165287, + 0.3868809787202447, + 0.4566812956703239, + -0.2889251297312824, + 0.1845822851455713, + -0.28782625795656447, + 0.0017553469388367346, + -0.20445409511352375, + -0.18334434983608747, + 0.12710052627513155, + -0.13884052221013055, + 0.2468096057024014, + -0.1367490296872574, + -0.1698644059661015, + 0.3063287385821846, + -0.3021076685269171, + -0.017076032769008192, + -0.10980901095225273, + 0.032857482714370674, + -0.08031403557850889, + 0.22095811623952902, + 0.30548675387168844, + 0.007770274942568735, + -0.04132654033163508, + 0.2974423268605817, + 0.08928245082061269, + 0.04125663131415783, + 0.26493235073308763, + -0.06513219728304932, + -0.13663724815931205, + 0.16512205728051432, + -0.26483096420774105, + 0.015502838375709595, + -0.09044484311121079, + 0.13732390333097585, + 0.05127470931867733, + -0.06010028102507025, + -0.13247654861913716, + 0.14303943575985892, + -0.19051629712907428, + 0.2054320428580107, + 0.09992810801950004, + -0.0656446124111531, + -0.0236095004023751, + 0.08853106463276615, + 0.06630357907589476, + -0.0830765547691387, + -0.012992633503030614, + 0.09053796613449153, + 0.08663203865800966, + 0.24924802890547526, + -0.16183876045969012, + 0.11508868527217131, + -0.10854949513737379, + 0.04786546796636699, + -0.03136841134210284, + 0.2114087133521783, + 0.1753995708498927, + 0.10886941821735455, + -0.02816417954104488, + 0.33614658903664724, + 0.005719633429908357, + 0.2590778072694518, + -0.2392735408183852, + -0.009911238977809744, + -0.28008195852048956, + 0.5635117873779468, + 0.21797335549333885, + 0.26516846129211535, + 0.1953490156605098, + -0.11998694549673637, + 0.10780298695074673, + -0.1050269647567412, + 0.02056176364044091, + -0.33938055084513774, + 0.3812376173094043, + 0.38757741839435456 + ], + "de_corr": [ + -0.06991409510374069, + -0.2783658504486084, + -0.3373911380767822, + -0.3189454674720764, + 0.4513323903083801, + 0.7092305421829224, + -0.5768485069274902, + 0.6220636367797852, + -0.5247228741645813, + 0.4144252836704254, + -0.23817110061645508, + 0.1804094761610031, + -0.1269863247871399, + 0.2127237319946289, + -0.20395003259181976, + -0.7087032198905945, + 0.48737937211990356, + 0.3027157783508301, + -0.5586394667625427, + 0.2280016392469406, + -0.7551730275154114, + 0.15953680872917175, + -0.5616952180862427, + 0.16191959381103516, + 0.6431093811988831, + 0.6383596658706665, + -0.26369717717170715, + 0.40792593359947205, + 0.5201324224472046, + -0.5504587292671204, + 0.24455992877483368, + -0.16370420157909393, + -0.614254891872406, + -0.41632574796676636, + -0.40115535259246826, + -0.05870254710316658, + -0.6217166185379028, + 0.23225374519824982, + -0.14819709956645966, + -0.5091150999069214, + -0.08810608088970184, + -0.1275816112756729, + 0.027489017695188522, + 0.18707288801670074, + 0.6027622222900391, + 0.24422971904277802, + 0.4340241551399231, + 0.02436957135796547, + 0.2650553286075592, + -0.06008575111627579, + 0.3321959376335144, + 0.15375573933124542, + 0.7904263734817505, + 0.5716975331306458, + -0.024184811860322952, + -0.0783996656537056, + -0.07074395567178726, + 0.26354655623435974, + -0.5646301507949829, + 0.811008632183075, + 0.347667932510376, + 0.23200711607933044, + 0.2966429591178894, + 0.6245260834693909, + -0.05450102686882019, + 0.3211802542209625, + -0.42361268401145935, + -0.10520084202289581, + -0.3438662886619568, + 0.8839427828788757, + 0.5226710438728333, + -0.03694850951433182, + 0.40940845012664795, + -0.5932114124298096, + 0.35271307826042175, + -0.3986108899116516, + 0.27721744775772095, + -0.6475280523300171, + 0.2503073811531067, + 0.40628698468208313 + ], + "mmd": [ + 0.10881842346077797, + 0.102339340740363, + 0.13405531846923302, + 0.12192501676589262, + 0.07354260240879584, + 0.02637084528510747, + 0.120354418769275, + 0.036495671075919, + 0.07554846257213188, + 0.08624695698458973, + 0.04494634050034274, + 0.11378038022390446, + 0.08132332478909543, + 0.01973266917360994, + 0.021095758420324073, + 0.17078312288189113, + 0.09054851254529439, + 0.019601514363143724, + 0.019888020684785013, + 0.12084628028343081, + 0.04218839801774554, + 0.04157454461789922, + 0.08750366629929629, + 0.07724215425069836, + 0.02514454260830279, + 0.08514205048409595, + 0.040244797966954016, + 0.051908424987485935, + 0.10934466756902372, + 0.11074376752738768, + 0.13193383819407645, + 0.09183694884241433, + 0.13121738819650675, + 0.05409520595580619, + 0.04758813064736922, + 0.07982830469739055, + 0.03421892302240159, + 0.019064100944049533, + 0.0843936788223465, + 0.0617975932833954, + 0.032901124144746885, + 0.020923399279994204, + 0.06200135542664886, + 0.09055223181124972, + 0.02121553772206486, + 0.07443114035066323, + 0.04636970755895109, + 0.025592385577160037, + 0.031917372983860215, + 0.02099980737511753, + 0.13472706998879758, + 0.0272073474288671, + 0.3648494592626542, + 0.045378783232280284, + 0.02868901835008064, + 0.03284345369347508, + 0.06364797771803332, + 0.08540593953537101, + 0.022330767143274532, + 0.05037102195401755, + 0.07774381825661936, + 0.05437357680648236, + 0.15137290957449345, + 0.06022918122255527, + 0.0489827351455101, + 0.07549853003428775, + 0.029261182796350593, + 0.13231636435521, + 0.06968062607598757, + 0.11890606720820729, + 0.028172128944229602, + 0.13057088412816065, + 0.02618862254427179, + 0.08385692682430612, + 0.153090413713449, + 0.08989867635139526, + 0.08110114873888963, + 0.04780083172942262, + 0.09809613321987243, + 0.06158871112261721 + ], + "wasserstein": [ + 0.4542324245734609, + 0.3410277469102676, + 0.4896113544975815, + 0.4266147689466922, + 0.4152864105381011, + 0.2244260000238162, + 0.5077183816443375, + 0.23692938506256087, + 0.37271264549401484, + 0.4780569573924292, + 0.35111673760002576, + 0.4734299890449927, + 0.39752850580159055, + 0.20592982841064233, + 0.24402767562651106, + 0.5841402399832065, + 0.4250507532447047, + 0.23908281698243294, + 0.1879361464033955, + 0.474783671966303, + 0.3245319817626008, + 0.2745011751985593, + 0.3861980852173738, + 0.3837521644131286, + 0.22862086033283685, + 0.4119694673556088, + 0.2556511320263827, + 0.265746265066652, + 0.42773738640556014, + 0.41108588922282463, + 0.49249222937085707, + 0.4276763089623828, + 0.48233955402354495, + 0.33311410218083737, + 0.3248953455992931, + 0.3725501153099598, + 0.2798730171589734, + 0.18654529693816127, + 0.4512839454540909, + 0.37382769519794673, + 0.26720431336417666, + 0.22715781287938713, + 0.3637392165091287, + 0.4959008859396732, + 0.2250653792531341, + 0.3351927207807685, + 0.30678319219354483, + 0.22473691975923235, + 0.31726713217939906, + 0.21215472204195673, + 0.49144224358085664, + 0.27832470544832455, + 1.03102774378163, + 0.35029322008109937, + 0.28837285829801645, + 0.2498915303340465, + 0.38490119271098217, + 0.3980972432522007, + 0.19725443419603836, + 0.3455933373880469, + 0.3875441247894322, + 0.303391909753405, + 0.5341081057179089, + 0.3634664009764177, + 0.2803803015940536, + 0.4168902478886767, + 0.25426737266518745, + 0.406167668063315, + 0.3696430535843871, + 0.458185810987369, + 0.27068228340522915, + 0.5952451335993421, + 0.23871126940254797, + 0.43748856254258817, + 0.5294993684393158, + 0.40972632418899146, + 0.41974041394335543, + 0.30010457643994, + 0.4985993062931714, + 0.30792501123179306 + ] + }, + "n_perts": 80 + } + } +} \ No newline at end of file diff --git a/experiments/results/replogle_k562_forward_gene.json b/experiments/results/replogle_k562_forward_gene.json new file mode 100644 index 0000000000000000000000000000000000000000..d8a78bf691661e600035864961be479f465ae17a --- /dev/null +++ b/experiments/results/replogle_k562_forward_gene.json @@ -0,0 +1,4693 @@ +{ + "split": "gene", + "models": { + "PIVOT": { + "mse": 0.00352818784340343, + "r2": -0.26236898452043533, + "pearson": 0.2231441641575657, + "spearman": 0.183619250538228, + "de_corr": 0.2637165174470283, + "mmd": 0.29977669859486583, + "wasserstein": 0.5616400440952315, + "_per_pert": { + "mse": [ + 0.0004183893615845591, + 0.006474472116678953, + 0.0021709862630814314, + 0.0006202924996614456, + 0.0009933121036738157, + 0.0019236196530982852, + 0.0004073556629009545, + 0.0006712346221320331, + 0.0006953433039598167, + 0.04155607521533966, + 0.00046265037963166833, + 0.0015052490634843707, + 0.000834540231153369, + 0.000529815093614161, + 0.0018011053325608373, + 0.0006414663512259722, + 0.014269738458096981, + 0.0009946534410119057, + 0.001527434098534286, + 0.0006342097767628729, + 0.004901292733848095, + 0.0008148839115165174, + 0.0005767702823504806, + 0.002867396455258131, + 0.00048795592738315463, + 0.003248463151976466, + 0.005023881793022156, + 0.0027576470747590065, + 0.0018078493885695934, + 0.0007328210049308836, + 0.0014346985844895244, + 0.008179746568202972, + 0.000413593283155933, + 0.0010515680769458413, + 0.0005035088397562504, + 0.02628696896135807, + 0.000950124638620764, + 0.0005435735220089555, + 0.002285960130393505, + 0.0019694166257977486, + 0.0005853448528796434, + 0.013186083175241947, + 0.007174076046794653, + 0.001900749746710062, + 0.003274191403761506, + 0.001431853510439396, + 0.0010490906424820423, + 0.0039128754287958145, + 0.0015086133498698473, + 0.0010007931850850582, + 0.0009671663865447044, + 0.0017109570326283574, + 0.0005051845801062882, + 0.0010319313732907176, + 0.0037170208524912596, + 0.0037441106978803873, + 0.0008641124004498124, + 0.011496526189148426, + 0.0005824683466926217, + 0.0009100291645154357, + 0.0012185433879494667, + 0.0004599977401085198, + 0.00038920564111322165, + 0.0012212364235892892, + 0.00043449123040772974, + 0.0008360805222764611, + 0.012305818498134613, + 0.0005984247545711696, + 0.002723759040236473, + 0.0010364840272814035, + 0.014385532587766647, + 0.0008350104326382279, + 0.01552374567836523, + 0.0018040752038359642, + 0.0005669324309565127, + 0.0015236969338729978, + 0.0019257930107414722, + 0.005395359825342894, + 0.007736751809716225, + 0.004810845945030451 + ], + "r2": [ + -1.0324418544769287, + 0.09599572420120239, + -0.46292805671691895, + -0.7960715293884277, + -0.15195178985595703, + -0.5155925750732422, + -0.4737282991409302, + -0.03684389591217041, + -0.7100632190704346, + 0.044709086418151855, + -0.7671747207641602, + 0.27488601207733154, + -0.32522332668304443, + -0.28429150581359863, + -0.2757863998413086, + -1.419585943222046, + 0.05404055118560791, + 0.29645633697509766, + 0.22873777151107788, + -0.8365613222122192, + 0.0906299352645874, + -1.3037564754486084, + -1.0992012023925781, + 0.02845001220703125, + -0.23344552516937256, + 0.07899123430252075, + 0.07019364833831787, + 0.15899884700775146, + 0.012782096862792969, + -0.10454106330871582, + -0.17030417919158936, + 0.04218137264251709, + -2.0922248363494873, + -0.09042787551879883, + 0.04335963726043701, + 0.028685808181762695, + -0.7753041982650757, + -0.6994646787643433, + 0.01287919282913208, + 0.2813159227371216, + -1.2588398456573486, + -0.10245001316070557, + 0.08401632308959961, + 0.17237091064453125, + 0.13634920120239258, + 0.128001868724823, + -1.145512342453003, + 0.17450487613677979, + 0.22091853618621826, + 0.2385130524635315, + 0.24240869283676147, + 0.09446454048156738, + -0.30651843547821045, + 0.3070296049118042, + -0.1477450132369995, + -0.02627706527709961, + -0.0325319766998291, + -0.022925496101379395, + -0.053505659103393555, + 0.28819555044174194, + -0.2946740388870239, + -2.5369017124176025, + 0.1741756796836853, + -0.20651960372924805, + -1.0263948440551758, + -0.08526194095611572, + 0.030261099338531494, + -1.32771635055542, + -0.1271423101425171, + -0.13943397998809814, + 0.07004070281982422, + -0.532806396484375, + 0.105319082736969, + 0.05822038650512695, + -1.2686951160430908, + -0.3265988826751709, + 0.08249664306640625, + 0.09807652235031128, + 0.09886741638183594, + -0.011677145957946777 + ], + "pearson": [ + 0.16414739191532135, + 0.4989616274833679, + -0.1675606071949005, + 0.003922448027879, + 0.1354316622018814, + -0.3585653305053711, + 0.32153239846229553, + 0.261723130941391, + 0.010550487786531448, + 0.3558017313480377, + 0.055892206728458405, + 0.5579474568367004, + 0.1291331797838211, + 0.3601835370063782, + -0.1302056610584259, + -0.10786620527505875, + 0.3345814347267151, + 0.5656357407569885, + 0.5699748396873474, + -0.18960949778556824, + 0.46970510482788086, + -0.13831304013729095, + -0.11814968287944794, + 0.3234124183654785, + 0.26916369795799255, + 0.2887971103191376, + 0.27842316031455994, + 0.4875914752483368, + 0.2589651644229889, + 0.22267106175422668, + 0.10029973834753036, + 0.22005757689476013, + -0.09251335263252258, + 0.20090582966804504, + 0.3527199327945709, + 0.436358243227005, + -0.020324796438217163, + 0.24640128016471863, + 0.19398383796215057, + 0.6015207767486572, + -0.1979515552520752, + 0.3586641848087311, + 0.4839131534099579, + 0.43712034821510315, + 0.4048866033554077, + 0.3776278495788574, + -0.2505618929862976, + 0.5619581341743469, + 0.5449535250663757, + 0.5935354232788086, + 0.5395822525024414, + 0.4047049880027771, + 0.25122806429862976, + 0.6491894721984863, + -0.046344030648469925, + 0.06821976602077484, + 0.20278547704219818, + 0.11460965871810913, + 0.3494391143321991, + 0.7098778486251831, + -0.173472061753273, + -0.007686627563089132, + 0.48192858695983887, + 0.1110503077507019, + -0.08441612869501114, + 0.2892584204673767, + 0.32772988080978394, + -0.12613089382648468, + 0.18163706362247467, + 0.16075372695922852, + 0.3511885702610016, + 0.0692998468875885, + 0.4989047646522522, + 0.3027234673500061, + -0.05527155101299286, + -0.20598319172859192, + 0.2976270318031311, + 0.34562772512435913, + 0.3778499662876129, + 0.19818833470344543 + ], + "spearman": [ + 0.19500354825088706, + 0.4529395552348888, + -0.04354929488732372, + 0.08892857973214494, + -0.06984462846115712, + -0.354507650509853, + 0.3071099792774948, + 0.10545516836379208, + 0.0049012557253139315, + 0.18652063063015764, + 0.0053519038379759585, + 0.5242329515582378, + 0.19764810891202725, + 0.2594848593712148, + -0.0753141248285312, + -0.06389538847384713, + 0.2837583169395792, + 0.39353628988407247, + 0.4839611964902991, + -0.19213664853416212, + 0.366361368090342, + -0.1364307356076839, + -0.08511558777889694, + 0.28329732482433123, + 0.11664485616121402, + 0.21454300463575118, + 0.2820125550031387, + 0.40526782281695567, + 0.1836386844096711, + 0.17775256743814186, + 0.21892713073178266, + 0.08742895885723972, + -0.07033255908313978, + 0.2073253748313437, + 0.2820072285018071, + 0.3367572571893143, + 0.05966614641653661, + 0.2515906213976553, + 0.03553832388458097, + 0.5647535656883914, + -0.037791844947961235, + 0.3221265085316271, + 0.47062548365637086, + 0.3780540015135004, + 0.30596903699225925, + 0.30671069417767355, + -0.2938075994518998, + 0.5741244055311013, + 0.531724545380533, + 0.5279104694776173, + 0.4484631536157883, + 0.195911583977896, + 0.19416827611488213, + 0.5423528255882064, + -0.04238940609735152, + 0.03553223088305772, + 0.2622323380580845, + -0.05703288225822056, + 0.2928208992052248, + 0.5965598811399703, + -0.06984657096164273, + 0.048349095087273766, + 0.3073825203456301, + 0.013076892269223067, + 0.012179892044973011, + 0.29090738622684653, + 0.2804367551091888, + -0.0653264488316122, + 0.007022913255728315, + 0.054850986212746554, + 0.3058415669603917, + 0.09056008517398133, + 0.4758455869613968, + 0.336596532149133, + -0.07161950540487634, + -0.1601723020430755, + 0.25189674897418723, + 0.26429585257396315, + 0.24542148735537184, + 0.04635745158936289 + ], + "de_corr": [ + 0.5172263979911804, + 0.6299213171005249, + -0.35112079977989197, + -0.3620549440383911, + 0.586102306842804, + -0.6215607523918152, + 0.27611038088798523, + 0.4157406985759735, + -0.036084115505218506, + 0.6332446336746216, + -0.20287896692752838, + 0.626244843006134, + 0.08699708431959152, + 0.5968127846717834, + -0.07892541587352753, + -0.44097986817359924, + 0.25776076316833496, + 0.7474575638771057, + 0.8176789283752441, + -0.00892728567123413, + 0.6782980561256409, + -0.19397002458572388, + -0.05449830740690231, + 0.3541225790977478, + 0.70524662733078, + 0.09701266884803772, + -0.03643452003598213, + 0.514058530330658, + 0.17007248103618622, + 0.1708168238401413, + -0.43114370107650757, + 0.44162142276763916, + -0.10717208683490753, + 0.11801306903362274, + 0.31071287393569946, + 0.5152987241744995, + -0.05768194794654846, + 0.3590278625488281, + 0.3173027038574219, + 0.7634254693984985, + -0.37272799015045166, + 0.5877619981765747, + 0.6803731322288513, + 0.5156942009925842, + 0.5294910669326782, + 0.6640853881835938, + -0.5807622075080872, + 0.5576549172401428, + 0.5960761904716492, + 0.72491854429245, + 0.8085029125213623, + 0.7927160263061523, + -0.015301848761737347, + 0.9109399318695068, + -0.3996136486530304, + -0.10053883492946625, + 0.0633142739534378, + 0.347936749458313, + 0.6271618604660034, + 0.8943688273429871, + -0.3035609722137451, + 0.07784861326217651, + 0.8186551928520203, + -0.0653189942240715, + -0.21100136637687683, + 0.1857006549835205, + 0.5893077254295349, + -0.1655719131231308, + 0.3887280523777008, + 0.3897906243801117, + 0.04508250579237938, + 0.29942387342453003, + 0.4987005591392517, + 0.1734967827796936, + -0.2324758768081665, + -0.1111326813697815, + 0.44189462065696716, + 0.5295796990394592, + 0.5145438313484192, + 0.6786841154098511 + ], + "mmd": [ + 0.2637751565563232, + 0.3327816178431864, + 0.28487319580146053, + 0.2564313652877962, + 0.304060684573064, + 0.30350342728223934, + 0.26508931002836367, + 0.2729289502474749, + 0.26439543823856626, + 0.5605459742549644, + 0.25671149246312175, + 0.29379970222127594, + 0.2804467748478304, + 0.25902966296832663, + 0.2943604554837832, + 0.26711000566714493, + 0.3748210737958869, + 0.3060696945286516, + 0.313490295731519, + 0.2673918193417575, + 0.3221545923971919, + 0.251399921121628, + 0.26907292300615526, + 0.29879568168755155, + 0.2709868817094523, + 0.3215575202590478, + 0.34128452885850136, + 0.30683729523492287, + 0.3065857639371635, + 0.2580797384959871, + 0.2720843824920236, + 0.34655619748097444, + 0.24194306653647135, + 0.27265723257060814, + 0.2847676689986959, + 0.4371051522149597, + 0.25993227985076905, + 0.2541569245196217, + 0.29462263862853655, + 0.3219330491900718, + 0.25341900289867125, + 0.36046582755357015, + 0.355984427726858, + 0.30571135597913723, + 0.3137095176072179, + 0.2653423910277867, + 0.273312125815276, + 0.33834286627456467, + 0.28706939011231125, + 0.27223350558495063, + 0.2511807539293367, + 0.2949774162597506, + 0.2665108043624975, + 0.27966145229678396, + 0.3191885082773468, + 0.312969863994229, + 0.26923170554015885, + 0.3623046381000432, + 0.26683529296328734, + 0.2873808348038053, + 0.2716406431484787, + 0.25675791027015316, + 0.2924891016371197, + 0.29341268671909904, + 0.2542671940429796, + 0.2813896100810972, + 0.4277591142548317, + 0.24830454632125554, + 0.31151755395685654, + 0.27675291878026087, + 0.37538374793824214, + 0.2513335537814929, + 0.41052755712310607, + 0.29502420475779845, + 0.256546751257841, + 0.29489687666205777, + 0.3010744153217735, + 0.35558028718154877, + 0.3165416809671322, + 0.320974315925489 + ], + "wasserstein": [ + 0.504971348653227, + 0.6915582339515605, + 0.5236566024836322, + 0.4620975811783563, + 0.5520300926863395, + 0.5318937116695857, + 0.47716443621068416, + 0.5326099511069252, + 0.46963005567327976, + 1.2144304710253824, + 0.4858217069810306, + 0.5310436970682288, + 0.5050074880802223, + 0.4759531535943184, + 0.48584571393793474, + 0.4975596544464278, + 0.836683424570048, + 0.5835653856106403, + 0.6881183258346425, + 0.4887959806954183, + 0.5953587559174789, + 0.46531574544208243, + 0.4808050737153314, + 0.5465136460051212, + 0.5039070904236772, + 0.6030071464688648, + 0.5323700014552653, + 0.5223591668819354, + 0.6228476145586396, + 0.4346819317638373, + 0.5077667405635412, + 0.7709346481412567, + 0.44910030697475234, + 0.4778339562654449, + 0.5129201134274733, + 1.0416273800583655, + 0.43298661985796477, + 0.4706968879459956, + 0.49135007007457054, + 0.5536442021159496, + 0.46715107868321654, + 0.8204094870690518, + 0.6675939346065207, + 0.47480456366809326, + 0.563530999914838, + 0.49262303567396465, + 0.5091830865295424, + 0.6303367952312375, + 0.5374674629668412, + 0.4863399808728275, + 0.47716180617341514, + 0.5967257647987994, + 0.4988841996695186, + 0.5218639767930601, + 0.5303603171613455, + 0.5419473266435604, + 0.47285136714396103, + 0.7635607740109869, + 0.4512973029981016, + 0.5300901258524773, + 0.4807950452853621, + 0.49168125794752404, + 0.5293536855542618, + 0.5303333816774678, + 0.48417018596633155, + 0.5222816896524769, + 0.7072950488424047, + 0.4452566660910638, + 0.6274641450557323, + 0.5381004545889828, + 0.8636244718913106, + 0.41095721380618144, + 0.8587718692336938, + 0.5298586686730209, + 0.4956691748376463, + 0.41501201222527856, + 0.4710748789552381, + 0.6192738478119133, + 0.7033800570324976, + 0.6182022725133383 + ] + }, + "n_perts": 80 + }, + "MeanControl": { + "mse": 0.0037895372535786008, + "r2": -0.14595426246523857, + "pearson": 0.004628284195496235, + "spearman": -0.0007950709341632556, + "de_corr": 0.04540404016152024, + "mmd": 0.05126164886924247, + "wasserstein": 0.315581765298599, + "_per_pert": { + "mse": [ + 0.0003135530569124967, + 0.007565529551357031, + 0.00159974314738065, + 0.00041434390004724264, + 0.001057824119925499, + 0.0012700867373496294, + 0.00038360542384907603, + 0.0007954876637086272, + 0.00036734045716002584, + 0.043578583747148514, + 0.0003190361603628844, + 0.0018691866425797343, + 0.0007922602817416191, + 0.0005640173912979662, + 0.0019363417522981763, + 0.0003283632395323366, + 0.01513818558305502, + 0.0012615913292393088, + 0.002322000451385975, + 0.00032479382934980094, + 0.0060366191901266575, + 0.0003600410127546638, + 0.000425270787673071, + 0.003986774943768978, + 0.0005002517718821764, + 0.003681278321892023, + 0.005548305343836546, + 0.003138965228572488, + 0.0023685062769800425, + 0.0007131922757253051, + 0.0014301537303254008, + 0.008315999060869217, + 0.0001708415220491588, + 0.0010590070160105824, + 0.0007750060758553445, + 0.02823387086391449, + 0.0005691223777830601, + 0.00048193958355113864, + 0.002259922446683049, + 0.00288140750490129, + 0.0002663169871084392, + 0.01575201004743576, + 0.0074739293195307255, + 0.002405586652457714, + 0.003736414248123765, + 0.001574805355630815, + 0.0008918708772398531, + 0.0047572460025548935, + 0.00199701264500618, + 0.0012434691889211535, + 0.0013458342291414738, + 0.001973779173567891, + 0.0004220961418468505, + 0.0013149866135790944, + 0.0036813688930124044, + 0.003905249759554863, + 0.0010133606847375631, + 0.011102687567472458, + 0.0006861949223093688, + 0.0016585740959271789, + 0.0010814041597768664, + 0.00024069577921181917, + 0.0005773089942522347, + 0.0012526626233011484, + 0.00018732303578872234, + 0.0007441260968334973, + 0.013510736636817455, + 0.0004385379434097558, + 0.002753057749941945, + 0.0010888457763940096, + 0.01575833186507225, + 0.0006098570302128792, + 0.017467468976974487, + 0.0019267674069851637, + 0.00036751842708326876, + 0.0013341380981728435, + 0.0023392115253955126, + 0.006029635202139616, + 0.008585790172219276, + 0.004528419580310583 + ], + "r2": [ + -0.5231711864471436, + -0.05634415149688721, + -0.07799339294433594, + -0.19974267482757568, + -0.22676682472229004, + -0.0006834268569946289, + -0.38780510425567627, + -0.22877538204193115, + 0.0965965986251831, + -0.001784205436706543, + -0.21861493587493896, + 0.09956890344619751, + -0.2580840587615967, + -0.36719954013824463, + -0.37157928943634033, + -0.23857367038726807, + -0.003530144691467285, + 0.10764443874359131, + -0.17247068881988525, + 0.05945366621017456, + -0.12001478672027588, + -0.01787126064300537, + -0.5478066205978394, + -0.35082530975341797, + -0.26452624797821045, + -0.04372107982635498, + -0.026865124702453613, + 0.04270803928375244, + -0.2933773994445801, + -0.07495594024658203, + -0.1665971279144287, + 0.026226699352264404, + -0.2772942781448364, + -0.09814167022705078, + -0.4724709987640381, + -0.04325294494628906, + -0.06340301036834717, + -0.506767988204956, + 0.02412283420562744, + -0.051489949226379395, + -0.02771472930908203, + -0.31697940826416016, + 0.04573112726211548, + -0.047446489334106445, + 0.014426171779632568, + 0.04094433784484863, + -0.8239794969558716, + -0.003631114959716797, + -0.03130173683166504, + 0.05386495590209961, + -0.05420541763305664, + -0.044635891914367676, + -0.09163355827331543, + 0.11695027351379395, + -0.13673663139343262, + -0.07044601440429688, + -0.21086955070495605, + 0.01211702823638916, + -0.24111509323120117, + -0.2972996234893799, + -0.14896690845489502, + -0.8506988286972046, + -0.2249457836151123, + -0.23756706714630127, + 0.12635666131973267, + 0.03409820795059204, + -0.0646902322769165, + -0.7057983875274658, + -0.13926661014556885, + -0.19699668884277344, + -0.018704533576965332, + -0.1194983720779419, + -0.006703615188598633, + -0.0058286190032958984, + -0.4706993103027344, + -0.1615605354309082, + -0.11446774005889893, + -0.007953166961669922, + -2.372264862060547e-05, + 0.04771435260772705 + ], + "pearson": [ + -0.21644580364227295, + 0.23483537137508392, + -0.0005317168543115258, + 0.13985086977481842, + -0.19946721196174622, + 0.12864293158054352, + -0.17481984198093414, + -0.08385606855154037, + 0.3229949176311493, + 0.21456672251224518, + 0.2328033596277237, + 0.4145580232143402, + -0.13677188754081726, + -0.17974503338336945, + -0.39655187726020813, + 0.005322729703038931, + -0.006408405490219593, + 0.3476566970348358, + -0.1814814656972885, + 0.2700512111186981, + -0.02433258295059204, + 0.23205558955669403, + -0.16232909262180328, + -0.4905180335044861, + -0.15489687025547028, + 0.030346892774105072, + -0.01711989752948284, + 0.22520405054092407, + -0.49047741293907166, + 0.17707619071006775, + 0.10346878319978714, + 0.19181008636951447, + 0.005092858336865902, + 0.001998172141611576, + -0.3337039649486542, + 0.13154591619968414, + 0.13716234266757965, + -0.059707339853048325, + 0.1893775761127472, + -0.07294772565364838, + 0.19234991073608398, + -0.27877891063690186, + 0.5292931199073792, + 0.0016831941902637482, + 0.2390739917755127, + 0.20871996879577637, + -0.3954814076423645, + 0.03975219652056694, + -0.013415707275271416, + 0.297225683927536, + -0.10845108330249786, + 0.011838908307254314, + 0.01556172501295805, + 0.41493499279022217, + -0.34031444787979126, + -0.08314527571201324, + -0.2369609922170639, + 0.21057097613811493, + -0.2164701670408249, + 0.031848758459091187, + -0.026233380660414696, + -0.22145536541938782, + -0.16412152349948883, + -0.15401369333267212, + 0.39817267656326294, + 0.20422665774822235, + 0.044849853962659836, + -0.24804569780826569, + -0.2711746096611023, + -0.02838457189500332, + -0.05656258761882782, + 0.032593511044979095, + 0.06686683744192123, + 0.08814864605665207, + -0.2807958126068115, + -0.08646180480718613, + -0.18501870334148407, + 0.003255219431594014, + 0.06884294003248215, + 0.3114296495914459 + ], + "spearman": [ + -0.2851684687921172, + 0.18591052347763085, + 0.13271025017756255, + 0.05135837783959445, + -0.15025929906482477, + 0.21325318131329532, + -0.18941778535444634, + -0.13669835067458766, + 0.300010524002631, + 0.09210283402570851, + 0.2110023167505792, + 0.40189168497292116, + -0.20181448902930268, + -0.13205372101343024, + -0.29198597949649485, + 0.020944190236047558, + -0.06909687377421844, + 0.20337405284351323, + -0.10753343688335922, + 0.183818571954643, + 0.18253596063399016, + 0.1583769215942304, + -0.19224566706141677, + -0.43370692692673174, + -0.16069139367284843, + -0.0416239664059916, + -0.08766276391569097, + 0.14421377405344352, + -0.38826861706715426, + 0.16490751972687992, + 0.24116854979213745, + 0.11093013823253456, + 0.019369273842318457, + -0.1248129132032283, + -0.14487105771776443, + 0.1018206524551631, + 0.203658827914707, + -0.04712122928030731, + 0.18127947331986835, + -0.058616794154198536, + 0.22842015310503827, + -0.30412270303067573, + 0.4070255847563962, + -0.05600968100242024, + 0.17591752547938136, + 0.15667782466945615, + -0.31603919950979986, + 0.09133868883467221, + 0.04179049194762298, + 0.2560116460029115, + -0.08960167790041947, + -0.03707772326943081, + 0.035176136794034196, + 0.3319584829896208, + -0.19913080378270093, + 0.00488276372069093, + -0.1056317949079487, + 0.1540773460193365, + -0.3700087555021888, + -0.013817911954477987, + -0.07298156724539182, + -0.2183526860881715, + -0.14457450164362542, + -0.20079306819826703, + 0.22561346340336583, + 0.2607705396926349, + 0.0680014940003735, + -0.2806478936619734, + -0.27926900181725045, + -0.13235749058937263, + -0.0347957651989413, + 0.03108909628393249, + 0.16514513778628445, + 0.092081288020322, + -0.2940463365115841, + -0.04698630501419612, + -0.16146368386592094, + -0.02465691016422754, + 0.06946570286642571, + 0.26232855408213857 + ], + "de_corr": [ + -0.03622117266058922, + 0.38581591844558716, + -0.14239460229873657, + 0.25438085198402405, + -0.41583362221717834, + 0.09088260680437088, + -0.12077896296977997, + -0.1917145848274231, + 0.5123120546340942, + 0.4046861231327057, + 0.18827927112579346, + 0.5574362277984619, + 0.09375456720590591, + -0.2718583941459656, + -0.5104306936264038, + -0.5676153302192688, + 0.3156948983669281, + 0.322867214679718, + -0.5215802788734436, + 0.4345199167728424, + 0.06456367671489716, + 0.6212273836135864, + 0.19039873778820038, + -0.8067936301231384, + -0.3089989125728607, + -0.0720198005437851, + 0.1396462768316269, + 0.23179945349693298, + -0.4338334798812866, + 0.6532859802246094, + -0.39463475346565247, + 0.4321286976337433, + 0.7570123076438904, + 0.3519911468029022, + -0.4297088086605072, + 0.6994672417640686, + 0.39958176016807556, + 0.21339523792266846, + 0.2590979039669037, + 0.020650368183851242, + 0.3302801549434662, + -0.0758761391043663, + 0.7354885339736938, + 0.07079026848077774, + 0.4478318393230438, + 0.5474826693534851, + -0.5955069661140442, + 0.06928855180740356, + 0.09410976618528366, + 0.4581683576107025, + -0.20661939680576324, + 0.43922898173332214, + -0.21895869076251984, + 0.6155246496200562, + -0.3576083779335022, + -0.17602530121803284, + -0.7841647267341614, + 0.3739136755466461, + 0.14571139216423035, + 0.06457892805337906, + -0.567622721195221, + -0.17433546483516693, + -0.12049046158790588, + -0.18151460587978363, + 0.569500744342804, + -0.6042268872261047, + 0.15831904113292694, + 0.14160946011543274, + -0.5267399549484253, + 0.49778082966804504, + 0.20231327414512634, + 0.06598637253046036, + -0.3405483365058899, + -0.12429828196763992, + -0.2662769854068756, + -0.27096059918403625, + -0.6506903767585754, + -0.2900505065917969, + 0.0674976333975792, + 0.698974072933197 + ], + "mmd": [ + 0.010865961643706346, + 0.09526803016238794, + 0.03414542261651732, + 0.012655752878092397, + 0.020272161876809136, + 0.03169784512836238, + 0.012166964591528107, + 0.01378156822626575, + 0.013161506222683972, + 0.37595768685632247, + 0.00979984181546123, + 0.03530890636017725, + 0.019015213965092426, + 0.016309612110020577, + 0.040693729480614116, + 0.009759099563166451, + 0.14422062098187793, + 0.027405931798592853, + 0.04225285796992073, + 0.010066702584036746, + 0.08631084179813242, + 0.012579651170403272, + 0.012571158202939525, + 0.0544084974687441, + 0.012538237883627623, + 0.06043173430925475, + 0.08415542539479814, + 0.05715046483273867, + 0.04296383245996371, + 0.02260156939242275, + 0.0338301946091234, + 0.11078357005261896, + 0.0071098337255163235, + 0.02846107153033539, + 0.01655639894340888, + 0.2022278918590673, + 0.019905650584001622, + 0.013048109468331903, + 0.04738914453273513, + 0.04963575853962676, + 0.009787493390163271, + 0.12135835753612634, + 0.10259814448897608, + 0.044599706333020905, + 0.05809130101644877, + 0.037120262448286256, + 0.020010755580221473, + 0.08324532378402383, + 0.038567341303705316, + 0.02504657601456528, + 0.028043479383941916, + 0.023580868962572943, + 0.012261321474307363, + 0.027796521068628288, + 0.07056779294627435, + 0.07657451455867637, + 0.025348443521783004, + 0.13396834963126292, + 0.018971953290527366, + 0.02903435660072362, + 0.02464025590783192, + 0.008471001609619888, + 0.014732542644024682, + 0.02646491019592334, + 0.0070139229480018495, + 0.02027082889037135, + 0.19370670986072713, + 0.013233911056386383, + 0.029681144910746693, + 0.025743508355221656, + 0.1718426196244932, + 0.02127572315814641, + 0.20704057653164076, + 0.03706375762058778, + 0.010190299027388883, + 0.04344536307718494, + 0.050616749429386876, + 0.09801713847439131, + 0.09072148283962445, + 0.07072214645403596 + ], + "wasserstein": [ + 0.15711171303839613, + 0.46294023210619595, + 0.3097925711144535, + 0.17848178976516998, + 0.2558425643177493, + 0.2559152283247149, + 0.1523232535434542, + 0.17515080517140816, + 0.18794170353453712, + 1.0767282260840905, + 0.14973940369189223, + 0.279458939479929, + 0.19361002135671626, + 0.18688165814365643, + 0.3416138962697992, + 0.15231269407574455, + 0.6458083908505542, + 0.29904459258819494, + 0.3109280343243505, + 0.16514292369639894, + 0.46247828408234914, + 0.17453192282617175, + 0.1799635980070445, + 0.3773264153691337, + 0.1796078135437695, + 0.36506880286672433, + 0.402834494647329, + 0.32065433000087823, + 0.27373344862515764, + 0.2363612896551318, + 0.23622519191526703, + 0.5010627928032602, + 0.1411489296246058, + 0.28799037297564195, + 0.19722798016939144, + 0.8823939408087295, + 0.19776941429478243, + 0.18002998043785606, + 0.3522717425518259, + 0.36787555259517163, + 0.16654906649832257, + 0.590740631660127, + 0.45340379509496137, + 0.37648602514223944, + 0.36508584566005786, + 0.2806632280416268, + 0.22090181440084117, + 0.412335814245266, + 0.2943957928612708, + 0.23597065972378597, + 0.2635683387412148, + 0.27071601108734555, + 0.17228513715224256, + 0.22547700541340243, + 0.3958086142289623, + 0.44895616289241747, + 0.2428726501733131, + 0.5025236601456261, + 0.19312637648130992, + 0.3033928678882094, + 0.242617471118807, + 0.16100098873112714, + 0.1985966803497802, + 0.24211151260618344, + 0.1368346764976086, + 0.2179747543314429, + 0.5514291366005489, + 0.19428340909703454, + 0.332240721408113, + 0.22806193624733515, + 0.7469295981944093, + 0.24160652140088504, + 0.7204070569427354, + 0.30062080042906286, + 0.14440878485734845, + 0.32440906950855847, + 0.3228828228753159, + 0.43125815121369304, + 0.45646147939874554, + 0.3838212152690147 + ] + }, + "n_perts": 80 + }, + "AvgPerturbationEffect": { + "mse": 0.0034097483727236976, + "r2": -0.5520497620105743, + "pearson": 0.2455340233165771, + "spearman": 0.20520113064383114, + "de_corr": 0.3118328441982158, + "mmd": 0.04959944534207577, + "wasserstein": 0.3117223404292129, + "_per_pert": { + "mse": [ + 0.0005825996631756425, + 0.005143148358911276, + 0.002098729368299246, + 0.0008992542279884219, + 0.0010414323769509792, + 0.0019154194742441177, + 0.0005956629756838083, + 0.0007027163519524038, + 0.0006015186081640422, + 0.038948047906160355, + 0.0007910301210358739, + 0.001374799874611199, + 0.0012303432449698448, + 0.0008642044849693775, + 0.002258309628814459, + 0.0008396322373300791, + 0.013300895690917969, + 0.0010313605889678001, + 0.0012407514732331038, + 0.0009599042823538184, + 0.004448774736374617, + 0.0008785222889855504, + 0.0009774609934538603, + 0.003598140785470605, + 0.0005044957506470382, + 0.0025347189512103796, + 0.004212670028209686, + 0.0023060282692313194, + 0.0014529528561979532, + 0.0010840122122317553, + 0.0011912997579202056, + 0.007326717022806406, + 0.0008223414188250899, + 0.0015441392315551639, + 0.0008783814264461398, + 0.024095291271805763, + 0.0010421122424304485, + 0.0009365165024064481, + 0.002395705319941044, + 0.002136200899258256, + 0.000885691144503653, + 0.01343798078596592, + 0.006077340804040432, + 0.0020331263076514006, + 0.0034025576896965504, + 0.0017901232931762934, + 0.0014572122599929571, + 0.0034621572121977806, + 0.0012116495054215193, + 0.0010567789431661367, + 0.0011296012671664357, + 0.0013417028822004795, + 0.0006667429697699845, + 0.0009453942766413093, + 0.0031355093233287334, + 0.003944715950638056, + 0.0013999188086017966, + 0.009762878529727459, + 0.0010465531377121806, + 0.000922837236430496, + 0.0014585708267986774, + 0.000788472534622997, + 0.00033990427618846297, + 0.0012442066799849272, + 0.00043771331547759473, + 0.0008133788360282779, + 0.01128991600126028, + 0.0009627271210774779, + 0.00234303530305624, + 0.0009936345741152763, + 0.014348622411489487, + 0.0012312277685850859, + 0.014440884813666344, + 0.001793248113244772, + 0.0009979413589462638, + 0.0019191720057278872, + 0.0023601490538567305, + 0.004366768058389425, + 0.00694182887673378, + 0.0038117526564747095 + ], + "r2": [ + -1.830138921737671, + 0.2818831205368042, + -0.41423749923706055, + -1.6038119792938232, + -0.20775723457336426, + -0.5091320276260376, + -1.154984951019287, + -0.08547317981719971, + -0.4793192148208618, + 0.10466253757476807, + -2.0214786529541016, + 0.33772653341293335, + -0.9537457227706909, + -1.0948638916015625, + -0.5996403694152832, + -2.167059898376465, + 0.1182662844657898, + 0.2704926133155823, + 0.37349504232406616, + -1.7797160148620605, + 0.17458879947662354, + -1.4836680889129639, + -2.557546615600586, + -0.21914541721343994, + -0.2752542495727539, + 0.2813529968261719, + 0.2203305959701538, + 0.29672926664352417, + 0.20658165216445923, + -0.6338726282119751, + 0.028239667415618896, + 0.14206808805465698, + -5.1482253074646, + -0.6012016534805298, + -0.668878436088562, + 0.10966914892196655, + -0.947182297706604, + -1.9279875755310059, + -0.03451097011566162, + 0.22045254707336426, + -2.4178738594055176, + -0.12351036071777344, + 0.22404712438583374, + 0.11473113298416138, + 0.10248935222625732, + -0.09018433094024658, + -1.9801685810089111, + 0.26959240436553955, + 0.3742772936820984, + 0.19591444730758667, + 0.11517179012298584, + 0.2898947596549988, + -0.72434401512146, + 0.36514174938201904, + 0.0318145751953125, + -0.08126378059387207, + -0.6727696657180786, + 0.13132917881011963, + -0.8928917646408081, + 0.27817726135253906, + -0.549697756767273, + -5.06252908706665, + 0.27878421545028687, + -0.22921323776245117, + -1.041421890258789, + -0.05579423904418945, + 0.1103176474571228, + -2.744757890701294, + 0.03040832281112671, + -0.09232854843139648, + 0.0724266767501831, + -1.2601323127746582, + 0.16772770881652832, + 0.06387245655059814, + -2.993464708328247, + -0.670917272567749, + -0.12444305419921875, + 0.2700226306915283, + 0.19145560264587402, + 0.19842296838760376 + ], + "pearson": [ + 0.061030272394418716, + 0.7549564242362976, + 0.05751093477010727, + -0.04369966685771942, + 0.2069295048713684, + -0.15974394977092743, + 0.10329172015190125, + 0.40790867805480957, + 0.3015619218349457, + 0.5682238340377808, + 0.2668112814426422, + 0.5916787981987, + -0.10949701815843582, + 0.1466212272644043, + -0.09198087453842163, + -0.1529911756515503, + 0.4653441905975342, + 0.5345771312713623, + 0.6695622205734253, + -0.20527151226997375, + 0.5552326440811157, + 0.03321439400315285, + -0.27204808592796326, + 0.06643685698509216, + 0.39208218455314636, + 0.5634338855743408, + 0.5327057242393494, + 0.5942004323005676, + 0.4918018877506256, + 0.14649564027786255, + 0.35414794087409973, + 0.4047507345676422, + -0.5029024481773376, + 0.020647374913096428, + 0.09025587886571884, + 0.678824782371521, + -0.0769813284277916, + 0.003349633887410164, + 0.24272526800632477, + 0.48075902462005615, + -0.2761193811893463, + 0.35070180892944336, + 0.5991236567497253, + 0.3593236207962036, + 0.33737534284591675, + 0.18851716816425323, + -0.20311783254146576, + 0.6176603436470032, + 0.671146810054779, + 0.5221258997917175, + 0.4010509252548218, + 0.6069756746292114, + 0.1967179924249649, + 0.6405487656593323, + 0.30995842814445496, + 0.06903589516878128, + -0.13440066576004028, + 0.47568202018737793, + -0.03781207278370857, + 0.6508035063743591, + -0.14448973536491394, + -0.33955514430999756, + 0.5968636870384216, + 0.286220520734787, + 0.2697177529335022, + 0.41081514954566956, + 0.4911011755466461, + -0.37710583209991455, + 0.3493264615535736, + 0.261532187461853, + 0.31147611141204834, + -0.11166010051965714, + 0.6022918224334717, + 0.34165671467781067, + -0.34675857424736023, + -0.2415717989206314, + 0.08801211416721344, + 0.6292260885238647, + 0.5155384540557861, + 0.532830536365509 + ], + "spearman": [ + -0.012090679522669881, + 0.6977098954274739, + 0.1450131167532792, + -0.04749983087495771, + 0.08937925334481334, + -0.10078236419559106, + -0.0030729682682420667, + 0.2994747283686821, + 0.24681941720485429, + 0.4445384461346115, + 0.2151244512811128, + 0.5230794167698543, + -0.09617779354444839, + 0.14578339844584962, + 0.016144364536091135, + -0.18157021539255383, + 0.41461215765303944, + 0.37162702040675505, + 0.6083111270777818, + -0.19910819127704785, + 0.5227272826818207, + -0.07909027827256955, + -0.2634291793572948, + 0.06228738957184738, + 0.2968642672160668, + 0.47850050662512666, + 0.4742138535534634, + 0.4903983655995914, + 0.39055762963940743, + 0.13457892664473164, + 0.38983801045950256, + 0.28748236637059155, + -0.4623955155988788, + 0.015348783837195958, + 0.23744550536137635, + 0.6466587926646982, + -0.04224213606053401, + 0.03156737515368156, + 0.2035127023781756, + 0.38654380713595177, + -0.14385154646288661, + 0.2781673565418391, + 0.5253280193320048, + 0.25493990623497653, + 0.27062060265515064, + 0.13380912045228008, + -0.2755979323994831, + 0.5598281464570365, + 0.6065233186308296, + 0.4819712854928213, + 0.33117250829312705, + 0.4559284994821249, + 0.1753191528297882, + 0.4896906249226562, + 0.29153785588446396, + 0.09279895119973779, + -0.019934050983512745, + 0.3380452145113036, + -0.10794779548694886, + 0.5706981301745325, + -0.13142376135594033, + -0.3113019523254881, + 0.49452539913134974, + 0.18348677937169483, + 0.22354803788700944, + 0.36348784137196033, + 0.4486911626727906, + -0.3045453656363414, + 0.200340044085011, + 0.1644751776187944, + 0.3022850810712702, + -0.0745589384176942, + 0.5974200368550092, + 0.2816811629202907, + -0.32133867233466806, + -0.17233515558378887, + 0.039975867993967, + 0.5392369033092258, + 0.37754505438626357, + 0.4271651747912937 + ], + "de_corr": [ + 0.6897227764129639, + 0.8864696621894836, + 0.12434157729148865, + -0.10353509336709976, + 0.464594304561615, + -0.34774771332740784, + -0.27771130204200745, + 0.5040099024772644, + 0.5618944764137268, + 0.8007484674453735, + 0.22932881116867065, + 0.7646660804748535, + -0.044790640473365784, + -0.12429240345954895, + 0.03533187881112099, + -0.3967788815498352, + 0.6527127623558044, + 0.5345252752304077, + 0.5684742331504822, + -0.008411237969994545, + 0.8292492628097534, + 0.397029310464859, + -0.03514830768108368, + -0.5421869158744812, + 0.6469439268112183, + 0.730720043182373, + 0.12504473328590393, + 0.3785933256149292, + 0.57857745885849, + 0.2224082499742508, + 0.05066896975040436, + 0.5418410301208496, + 0.18211892247200012, + -0.16815973818302155, + -0.35439443588256836, + 0.7907842397689819, + 0.008259550668299198, + 0.4223805367946625, + 0.29191523790359497, + 0.7320380210876465, + -0.47352707386016846, + 0.8487730026245117, + 0.8590567708015442, + 0.5939439535140991, + 0.44766291975975037, + 0.5950221419334412, + -0.364464670419693, + 0.6831140518188477, + 0.5967739820480347, + 0.7253232002258301, + 0.6219841241836548, + 0.8695692420005798, + 0.02088513784110546, + 0.8723554015159607, + 0.6031555533409119, + 0.1709887832403183, + -0.5777860879898071, + 0.7562110424041748, + 0.21485689282417297, + 0.6774112582206726, + -0.259482741355896, + -0.42325034737586975, + 0.7348798513412476, + 0.19625763595104218, + 0.37473225593566895, + 0.1757725030183792, + 0.8413135409355164, + -0.3673400282859802, + 0.6309633255004883, + 0.5124343037605286, + 0.2654319703578949, + -0.25711071491241455, + 0.6562187671661377, + 0.39367181062698364, + -0.7479125261306763, + -0.31291115283966064, + -0.07712652534246445, + 0.820684015750885, + 0.7898404002189636, + 0.9160152077674866 + ], + "mmd": [ + 0.015612012265772357, + 0.07244129978026048, + 0.037323246311130265, + 0.017966079909563892, + 0.026677115551748676, + 0.04194380412900667, + 0.016127729490181042, + 0.013699043463356708, + 0.015605133216123113, + 0.3531200415957221, + 0.01629205845509074, + 0.029364245061944017, + 0.02297392911477736, + 0.019984428308042146, + 0.0426202242977991, + 0.017350741273044368, + 0.13386211423933303, + 0.026131277704813938, + 0.03322666060616575, + 0.018543118232818734, + 0.06982541647391594, + 0.020134879227340474, + 0.020461732595834392, + 0.05236381049286176, + 0.015794230805896148, + 0.04831285063738111, + 0.07089491254470959, + 0.04683226925414441, + 0.03707041441722303, + 0.03172074122933666, + 0.03209863117312117, + 0.10394586763225966, + 0.014881325502262732, + 0.03123161148303566, + 0.021796626732261903, + 0.1856120953338638, + 0.02859580932279815, + 0.01863937809511451, + 0.049704348785424846, + 0.042040765393655755, + 0.01811251040826045, + 0.11454712947438028, + 0.08671286605099227, + 0.04086847865650145, + 0.05490606824338029, + 0.03667710650340994, + 0.02549084091629339, + 0.0667841550847531, + 0.029319744194779784, + 0.023698859926828852, + 0.025432609755765778, + 0.02375579149193896, + 0.01672538336871099, + 0.02296611570991347, + 0.06483577064107693, + 0.07479617702428154, + 0.03087995600074156, + 0.12237882385158527, + 0.023488062777768826, + 0.027452723249229538, + 0.030384945921135165, + 0.01793141495997852, + 0.019243247666952623, + 0.028563930804721305, + 0.013113530908661741, + 0.022313901038961137, + 0.16970258084218226, + 0.020046126399569597, + 0.03186086311614589, + 0.02666626056421295, + 0.15998976095182682, + 0.029796040472007235, + 0.18131855588601598, + 0.033866676238364035, + 0.01996912456637656, + 0.049657362535969196, + 0.050930449245672915, + 0.07621316988507532, + 0.0808905825671209, + 0.06284590932538292 + ], + "wasserstein": [ + 0.20109765374154162, + 0.38141535845814334, + 0.33192984244805374, + 0.21404119225360824, + 0.25326986483802755, + 0.2864221796191552, + 0.19187670042300778, + 0.16402139777856714, + 0.2073851044581886, + 0.9984526134097971, + 0.1923749183023051, + 0.24002639069753087, + 0.2138438086901091, + 0.18659729013797335, + 0.34276256547473516, + 0.19914072402508542, + 0.5767906592102415, + 0.283577669826469, + 0.25615216609917574, + 0.22012732562144105, + 0.4078474990948107, + 0.19877238605731073, + 0.2189543272460741, + 0.3659234304059749, + 0.19020897003871554, + 0.319304124225768, + 0.3712434263786506, + 0.3158606655532214, + 0.2525657980324252, + 0.25984407213166355, + 0.22488612010391518, + 0.4700802544401308, + 0.19176222361719758, + 0.3158310116728782, + 0.20851331624214553, + 0.816144425031109, + 0.25133037663114616, + 0.20657744362919403, + 0.3529377000062945, + 0.3384781344159699, + 0.2080608437210573, + 0.5503556950978512, + 0.4237569522938746, + 0.3647263618096864, + 0.33440008784853403, + 0.2713213874763763, + 0.26364353985622474, + 0.36472518748423277, + 0.2584174006145461, + 0.22072431701581718, + 0.24307713276754325, + 0.24127512673258075, + 0.2015307135233507, + 0.22736481283579402, + 0.3971641853486821, + 0.4624401129355364, + 0.2616543491169932, + 0.46842239847386763, + 0.22826554245247604, + 0.2603108535131246, + 0.24459127344521975, + 0.21543957854417894, + 0.1865079242631454, + 0.2260926098407223, + 0.15413924348716793, + 0.2368527085665848, + 0.5070019407138747, + 0.2288087538210125, + 0.3105734593152807, + 0.22351871910528132, + 0.7094728677688954, + 0.27027927904847215, + 0.6563384825094751, + 0.27708720342334286, + 0.19827973314315703, + 0.34677151547271734, + 0.3281853009414478, + 0.3773901473939231, + 0.40244759763011634, + 0.368000764517111 + ] + }, + "n_perts": 80 + }, + "Additive": { + "mse": 0.0034097483727236976, + "r2": -0.5520497620105743, + "pearson": 0.2455340233165771, + "spearman": 0.20520113064383114, + "de_corr": 0.3118328441982158, + "mmd": 0.04959944534207577, + "wasserstein": 0.3117223404292129, + "_per_pert": { + "mse": [ + 0.0005825996631756425, + 0.005143148358911276, + 0.002098729368299246, + 0.0008992542279884219, + 0.0010414323769509792, + 0.0019154194742441177, + 0.0005956629756838083, + 0.0007027163519524038, + 0.0006015186081640422, + 0.038948047906160355, + 0.0007910301210358739, + 0.001374799874611199, + 0.0012303432449698448, + 0.0008642044849693775, + 0.002258309628814459, + 0.0008396322373300791, + 0.013300895690917969, + 0.0010313605889678001, + 0.0012407514732331038, + 0.0009599042823538184, + 0.004448774736374617, + 0.0008785222889855504, + 0.0009774609934538603, + 0.003598140785470605, + 0.0005044957506470382, + 0.0025347189512103796, + 0.004212670028209686, + 0.0023060282692313194, + 0.0014529528561979532, + 0.0010840122122317553, + 0.0011912997579202056, + 0.007326717022806406, + 0.0008223414188250899, + 0.0015441392315551639, + 0.0008783814264461398, + 0.024095291271805763, + 0.0010421122424304485, + 0.0009365165024064481, + 0.002395705319941044, + 0.002136200899258256, + 0.000885691144503653, + 0.01343798078596592, + 0.006077340804040432, + 0.0020331263076514006, + 0.0034025576896965504, + 0.0017901232931762934, + 0.0014572122599929571, + 0.0034621572121977806, + 0.0012116495054215193, + 0.0010567789431661367, + 0.0011296012671664357, + 0.0013417028822004795, + 0.0006667429697699845, + 0.0009453942766413093, + 0.0031355093233287334, + 0.003944715950638056, + 0.0013999188086017966, + 0.009762878529727459, + 0.0010465531377121806, + 0.000922837236430496, + 0.0014585708267986774, + 0.000788472534622997, + 0.00033990427618846297, + 0.0012442066799849272, + 0.00043771331547759473, + 0.0008133788360282779, + 0.01128991600126028, + 0.0009627271210774779, + 0.00234303530305624, + 0.0009936345741152763, + 0.014348622411489487, + 0.0012312277685850859, + 0.014440884813666344, + 0.001793248113244772, + 0.0009979413589462638, + 0.0019191720057278872, + 0.0023601490538567305, + 0.004366768058389425, + 0.00694182887673378, + 0.0038117526564747095 + ], + "r2": [ + -1.830138921737671, + 0.2818831205368042, + -0.41423749923706055, + -1.6038119792938232, + -0.20775723457336426, + -0.5091320276260376, + -1.154984951019287, + -0.08547317981719971, + -0.4793192148208618, + 0.10466253757476807, + -2.0214786529541016, + 0.33772653341293335, + -0.9537457227706909, + -1.0948638916015625, + -0.5996403694152832, + -2.167059898376465, + 0.1182662844657898, + 0.2704926133155823, + 0.37349504232406616, + -1.7797160148620605, + 0.17458879947662354, + -1.4836680889129639, + -2.557546615600586, + -0.21914541721343994, + -0.2752542495727539, + 0.2813529968261719, + 0.2203305959701538, + 0.29672926664352417, + 0.20658165216445923, + -0.6338726282119751, + 0.028239667415618896, + 0.14206808805465698, + -5.1482253074646, + -0.6012016534805298, + -0.668878436088562, + 0.10966914892196655, + -0.947182297706604, + -1.9279875755310059, + -0.03451097011566162, + 0.22045254707336426, + -2.4178738594055176, + -0.12351036071777344, + 0.22404712438583374, + 0.11473113298416138, + 0.10248935222625732, + -0.09018433094024658, + -1.9801685810089111, + 0.26959240436553955, + 0.3742772936820984, + 0.19591444730758667, + 0.11517179012298584, + 0.2898947596549988, + -0.72434401512146, + 0.36514174938201904, + 0.0318145751953125, + -0.08126378059387207, + -0.6727696657180786, + 0.13132917881011963, + -0.8928917646408081, + 0.27817726135253906, + -0.549697756767273, + -5.06252908706665, + 0.27878421545028687, + -0.22921323776245117, + -1.041421890258789, + -0.05579423904418945, + 0.1103176474571228, + -2.744757890701294, + 0.03040832281112671, + -0.09232854843139648, + 0.0724266767501831, + -1.2601323127746582, + 0.16772770881652832, + 0.06387245655059814, + -2.993464708328247, + -0.670917272567749, + -0.12444305419921875, + 0.2700226306915283, + 0.19145560264587402, + 0.19842296838760376 + ], + "pearson": [ + 0.061030272394418716, + 0.7549564242362976, + 0.05751093477010727, + -0.04369966685771942, + 0.2069295048713684, + -0.15974394977092743, + 0.10329172015190125, + 0.40790867805480957, + 0.3015619218349457, + 0.5682238340377808, + 0.2668112814426422, + 0.5916787981987, + -0.10949701815843582, + 0.1466212272644043, + -0.09198087453842163, + -0.1529911756515503, + 0.4653441905975342, + 0.5345771312713623, + 0.6695622205734253, + -0.20527151226997375, + 0.5552326440811157, + 0.03321439400315285, + -0.27204808592796326, + 0.06643685698509216, + 0.39208218455314636, + 0.5634338855743408, + 0.5327057242393494, + 0.5942004323005676, + 0.4918018877506256, + 0.14649564027786255, + 0.35414794087409973, + 0.4047507345676422, + -0.5029024481773376, + 0.020647374913096428, + 0.09025587886571884, + 0.678824782371521, + -0.0769813284277916, + 0.003349633887410164, + 0.24272526800632477, + 0.48075902462005615, + -0.2761193811893463, + 0.35070180892944336, + 0.5991236567497253, + 0.3593236207962036, + 0.33737534284591675, + 0.18851716816425323, + -0.20311783254146576, + 0.6176603436470032, + 0.671146810054779, + 0.5221258997917175, + 0.4010509252548218, + 0.6069756746292114, + 0.1967179924249649, + 0.6405487656593323, + 0.30995842814445496, + 0.06903589516878128, + -0.13440066576004028, + 0.47568202018737793, + -0.03781207278370857, + 0.6508035063743591, + -0.14448973536491394, + -0.33955514430999756, + 0.5968636870384216, + 0.286220520734787, + 0.2697177529335022, + 0.41081514954566956, + 0.4911011755466461, + -0.37710583209991455, + 0.3493264615535736, + 0.261532187461853, + 0.31147611141204834, + -0.11166010051965714, + 0.6022918224334717, + 0.34165671467781067, + -0.34675857424736023, + -0.2415717989206314, + 0.08801211416721344, + 0.6292260885238647, + 0.5155384540557861, + 0.532830536365509 + ], + "spearman": [ + -0.012090679522669881, + 0.6977098954274739, + 0.1450131167532792, + -0.04749983087495771, + 0.08937925334481334, + -0.10078236419559106, + -0.0030729682682420667, + 0.2994747283686821, + 0.24681941720485429, + 0.4445384461346115, + 0.2151244512811128, + 0.5230794167698543, + -0.09617779354444839, + 0.14578339844584962, + 0.016144364536091135, + -0.18157021539255383, + 0.41461215765303944, + 0.37162702040675505, + 0.6083111270777818, + -0.19910819127704785, + 0.5227272826818207, + -0.07909027827256955, + -0.2634291793572948, + 0.06228738957184738, + 0.2968642672160668, + 0.47850050662512666, + 0.4742138535534634, + 0.4903983655995914, + 0.39055762963940743, + 0.13457892664473164, + 0.38983801045950256, + 0.28748236637059155, + -0.4623955155988788, + 0.015348783837195958, + 0.23744550536137635, + 0.6466587926646982, + -0.04224213606053401, + 0.03156737515368156, + 0.2035127023781756, + 0.38654380713595177, + -0.14385154646288661, + 0.2781673565418391, + 0.5253280193320048, + 0.25493990623497653, + 0.27062060265515064, + 0.13380912045228008, + -0.2755979323994831, + 0.5598281464570365, + 0.6065233186308296, + 0.4819712854928213, + 0.33117250829312705, + 0.4559284994821249, + 0.1753191528297882, + 0.4896906249226562, + 0.29153785588446396, + 0.09279895119973779, + -0.019934050983512745, + 0.3380452145113036, + -0.10794779548694886, + 0.5706981301745325, + -0.13142376135594033, + -0.3113019523254881, + 0.49452539913134974, + 0.18348677937169483, + 0.22354803788700944, + 0.36348784137196033, + 0.4486911626727906, + -0.3045453656363414, + 0.200340044085011, + 0.1644751776187944, + 0.3022850810712702, + -0.0745589384176942, + 0.5974200368550092, + 0.2816811629202907, + -0.32133867233466806, + -0.17233515558378887, + 0.039975867993967, + 0.5392369033092258, + 0.37754505438626357, + 0.4271651747912937 + ], + "de_corr": [ + 0.6897227764129639, + 0.8864696621894836, + 0.12434157729148865, + -0.10353509336709976, + 0.464594304561615, + -0.34774771332740784, + -0.27771130204200745, + 0.5040099024772644, + 0.5618944764137268, + 0.8007484674453735, + 0.22932881116867065, + 0.7646660804748535, + -0.044790640473365784, + -0.12429240345954895, + 0.03533187881112099, + -0.3967788815498352, + 0.6527127623558044, + 0.5345252752304077, + 0.5684742331504822, + -0.008411237969994545, + 0.8292492628097534, + 0.397029310464859, + -0.03514830768108368, + -0.5421869158744812, + 0.6469439268112183, + 0.730720043182373, + 0.12504473328590393, + 0.3785933256149292, + 0.57857745885849, + 0.2224082499742508, + 0.05066896975040436, + 0.5418410301208496, + 0.18211892247200012, + -0.16815973818302155, + -0.35439443588256836, + 0.7907842397689819, + 0.008259550668299198, + 0.4223805367946625, + 0.29191523790359497, + 0.7320380210876465, + -0.47352707386016846, + 0.8487730026245117, + 0.8590567708015442, + 0.5939439535140991, + 0.44766291975975037, + 0.5950221419334412, + -0.364464670419693, + 0.6831140518188477, + 0.5967739820480347, + 0.7253232002258301, + 0.6219841241836548, + 0.8695692420005798, + 0.02088513784110546, + 0.8723554015159607, + 0.6031555533409119, + 0.1709887832403183, + -0.5777860879898071, + 0.7562110424041748, + 0.21485689282417297, + 0.6774112582206726, + -0.259482741355896, + -0.42325034737586975, + 0.7348798513412476, + 0.19625763595104218, + 0.37473225593566895, + 0.1757725030183792, + 0.8413135409355164, + -0.3673400282859802, + 0.6309633255004883, + 0.5124343037605286, + 0.2654319703578949, + -0.25711071491241455, + 0.6562187671661377, + 0.39367181062698364, + -0.7479125261306763, + -0.31291115283966064, + -0.07712652534246445, + 0.820684015750885, + 0.7898404002189636, + 0.9160152077674866 + ], + "mmd": [ + 0.015612012265772357, + 0.07244129978026048, + 0.037323246311130265, + 0.017966079909563892, + 0.026677115551748676, + 0.04194380412900667, + 0.016127729490181042, + 0.013699043463356708, + 0.015605133216123113, + 0.3531200415957221, + 0.01629205845509074, + 0.029364245061944017, + 0.02297392911477736, + 0.019984428308042146, + 0.0426202242977991, + 0.017350741273044368, + 0.13386211423933303, + 0.026131277704813938, + 0.03322666060616575, + 0.018543118232818734, + 0.06982541647391594, + 0.020134879227340474, + 0.020461732595834392, + 0.05236381049286176, + 0.015794230805896148, + 0.04831285063738111, + 0.07089491254470959, + 0.04683226925414441, + 0.03707041441722303, + 0.03172074122933666, + 0.03209863117312117, + 0.10394586763225966, + 0.014881325502262732, + 0.03123161148303566, + 0.021796626732261903, + 0.1856120953338638, + 0.02859580932279815, + 0.01863937809511451, + 0.049704348785424846, + 0.042040765393655755, + 0.01811251040826045, + 0.11454712947438028, + 0.08671286605099227, + 0.04086847865650145, + 0.05490606824338029, + 0.03667710650340994, + 0.02549084091629339, + 0.0667841550847531, + 0.029319744194779784, + 0.023698859926828852, + 0.025432609755765778, + 0.02375579149193896, + 0.01672538336871099, + 0.02296611570991347, + 0.06483577064107693, + 0.07479617702428154, + 0.03087995600074156, + 0.12237882385158527, + 0.023488062777768826, + 0.027452723249229538, + 0.030384945921135165, + 0.01793141495997852, + 0.019243247666952623, + 0.028563930804721305, + 0.013113530908661741, + 0.022313901038961137, + 0.16970258084218226, + 0.020046126399569597, + 0.03186086311614589, + 0.02666626056421295, + 0.15998976095182682, + 0.029796040472007235, + 0.18131855588601598, + 0.033866676238364035, + 0.01996912456637656, + 0.049657362535969196, + 0.050930449245672915, + 0.07621316988507532, + 0.0808905825671209, + 0.06284590932538292 + ], + "wasserstein": [ + 0.20109765374154162, + 0.38141535845814334, + 0.33192984244805374, + 0.21404119225360824, + 0.25326986483802755, + 0.2864221796191552, + 0.19187670042300778, + 0.16402139777856714, + 0.2073851044581886, + 0.9984526134097971, + 0.1923749183023051, + 0.24002639069753087, + 0.2138438086901091, + 0.18659729013797335, + 0.34276256547473516, + 0.19914072402508542, + 0.5767906592102415, + 0.283577669826469, + 0.25615216609917574, + 0.22012732562144105, + 0.4078474990948107, + 0.19877238605731073, + 0.2189543272460741, + 0.3659234304059749, + 0.19020897003871554, + 0.319304124225768, + 0.3712434263786506, + 0.3158606655532214, + 0.2525657980324252, + 0.25984407213166355, + 0.22488612010391518, + 0.4700802544401308, + 0.19176222361719758, + 0.3158310116728782, + 0.20851331624214553, + 0.816144425031109, + 0.25133037663114616, + 0.20657744362919403, + 0.3529377000062945, + 0.3384781344159699, + 0.2080608437210573, + 0.5503556950978512, + 0.4237569522938746, + 0.3647263618096864, + 0.33440008784853403, + 0.2713213874763763, + 0.26364353985622474, + 0.36472518748423277, + 0.2584174006145461, + 0.22072431701581718, + 0.24307713276754325, + 0.24127512673258075, + 0.2015307135233507, + 0.22736481283579402, + 0.3971641853486821, + 0.4624401129355364, + 0.2616543491169932, + 0.46842239847386763, + 0.22826554245247604, + 0.2603108535131246, + 0.24459127344521975, + 0.21543957854417894, + 0.1865079242631454, + 0.2260926098407223, + 0.15413924348716793, + 0.2368527085665848, + 0.5070019407138747, + 0.2288087538210125, + 0.3105734593152807, + 0.22351871910528132, + 0.7094728677688954, + 0.27027927904847215, + 0.6563384825094751, + 0.27708720342334286, + 0.19827973314315703, + 0.34677151547271734, + 0.3281853009414478, + 0.3773901473939231, + 0.40244759763011634, + 0.368000764517111 + ] + }, + "n_perts": 80 + }, + "LinearResponse": { + "mse": 0.0034097483410732822, + "r2": -0.5520497649908066, + "pearson": 0.24553402810706756, + "spearman": 0.20520112447788003, + "de_corr": 0.3118328398792073, + "mmd": 0.0495994453304199, + "wasserstein": 0.3117223400859971, + "_per_pert": { + "mse": [ + 0.0005825996631756425, + 0.005143148358911276, + 0.002098729368299246, + 0.0008992542279884219, + 0.0010414323769509792, + 0.0019154194742441177, + 0.0005956629174761474, + 0.0007027163519524038, + 0.0006015186081640422, + 0.038948047906160355, + 0.0007910301210358739, + 0.0013747999910265207, + 0.0012303432449698448, + 0.0008642044849693775, + 0.002258309395983815, + 0.0008396321791224182, + 0.013300895690917969, + 0.0010313604725524783, + 0.0012407514732331038, + 0.0009599042823538184, + 0.004448774270713329, + 0.0008785222889855504, + 0.0009774609934538603, + 0.003598140785470605, + 0.0005044957506470382, + 0.002534718718379736, + 0.004212670028209686, + 0.0023060282692313194, + 0.0014529528561979532, + 0.0010840122122317553, + 0.0011912997579202056, + 0.007326717022806406, + 0.0008223413024097681, + 0.0015441392315551639, + 0.0008783813682384789, + 0.024095291271805763, + 0.0010421121260151267, + 0.0009365165024064481, + 0.0023957055527716875, + 0.0021362006664276123, + 0.000885691144503653, + 0.01343798078596592, + 0.006077340338379145, + 0.0020331263076514006, + 0.0034025576896965504, + 0.0017901232931762934, + 0.0014572122599929571, + 0.0034621572121977806, + 0.0012116493890061975, + 0.0010567789431661367, + 0.0011296012671664357, + 0.0013417028822004795, + 0.0006667430279776454, + 0.0009453942766413093, + 0.0031355093233287334, + 0.003944715950638056, + 0.0013999186921864748, + 0.009762878529727459, + 0.0010465530212968588, + 0.000922837236430496, + 0.0014585708267986774, + 0.000788472534622997, + 0.00033990430529229343, + 0.0012442066799849272, + 0.00043771331547759473, + 0.000813378719612956, + 0.01128991600126028, + 0.0009627271210774779, + 0.00234303530305624, + 0.0009936345741152763, + 0.014348622411489487, + 0.0012312277685850859, + 0.014440884813666344, + 0.0017932482296600938, + 0.000997941242530942, + 0.001919172122143209, + 0.0023601490538567305, + 0.004366768058389425, + 0.006941828411072493, + 0.0038117526564747095 + ], + "r2": [ + -1.830138921737671, + 0.281883180141449, + -0.41423749923706055, + -1.6038117408752441, + -0.20775723457336426, + -0.5091320276260376, + -1.154984951019287, + -0.08547317981719971, + -0.4793192148208618, + 0.10466241836547852, + -2.0214786529541016, + 0.33772653341293335, + -0.9537456035614014, + -1.0948641300201416, + -0.5996403694152832, + -2.167060136795044, + 0.1182662844657898, + 0.2704926133155823, + 0.37349504232406616, + -1.7797160148620605, + 0.17458868026733398, + -1.4836680889129639, + -2.557546615600586, + -0.2191455364227295, + -0.27525436878204346, + 0.2813529968261719, + 0.2203305959701538, + 0.29672926664352417, + 0.20658165216445923, + -0.6338722705841064, + 0.028239667415618896, + 0.14206808805465698, + -5.1482253074646, + -0.6012016534805298, + -0.668878436088562, + 0.10966914892196655, + -0.947182297706604, + -1.9279875755310059, + -0.03451097011566162, + 0.22045254707336426, + -2.4178738594055176, + -0.12351036071777344, + 0.22404712438583374, + 0.11473113298416138, + 0.1024894118309021, + -0.09018433094024658, + -1.9801685810089111, + 0.26959240436553955, + 0.3742772936820984, + 0.19591444730758667, + 0.11517184972763062, + 0.2898947596549988, + -0.72434401512146, + 0.36514168977737427, + 0.031814634799957275, + -0.08126378059387207, + -0.6727696657180786, + 0.13132917881011963, + -0.8928917646408081, + 0.2781773805618286, + -0.549697756767273, + -5.062529563903809, + 0.27878421545028687, + -0.22921323776245117, + -1.041421890258789, + -0.05579423904418945, + 0.1103176474571228, + -2.744757890701294, + 0.03040832281112671, + -0.09232842922210693, + 0.0724266767501831, + -1.260132074356079, + 0.16772770881652832, + 0.06387251615524292, + -2.993464708328247, + -0.6709173917770386, + -0.12444305419921875, + 0.2700226306915283, + 0.19145548343658447, + 0.19842296838760376 + ], + "pearson": [ + 0.061030276119709015, + 0.7549563646316528, + 0.057510942220687866, + -0.04369964450597763, + 0.2069295346736908, + -0.15974394977092743, + 0.10329174250364304, + 0.40790867805480957, + 0.3015618920326233, + 0.5682238340377808, + 0.2668113112449646, + 0.5916787981987, + -0.10949700325727463, + 0.1466212421655655, + -0.09198087453842163, + -0.1529911607503891, + 0.4653441905975342, + 0.5345771312713623, + 0.6695621609687805, + -0.20527154207229614, + 0.5552326440811157, + 0.03321441635489464, + -0.27204808592796326, + 0.06643685698509216, + 0.39208218455314636, + 0.5634338855743408, + 0.5327057838439941, + 0.5942004323005676, + 0.4918018877506256, + 0.14649567008018494, + 0.35414794087409973, + 0.4047507345676422, + -0.5029025077819824, + 0.020647387951612473, + 0.09025586396455765, + 0.6788249015808105, + -0.076981320977211, + 0.0033496576361358166, + 0.24272525310516357, + 0.48075905442237854, + -0.2761194109916687, + 0.35070180892944336, + 0.5991236567497253, + 0.3593235909938812, + 0.3373754024505615, + 0.18851718306541443, + -0.20311786234378815, + 0.6176603436470032, + 0.6711468696594238, + 0.5221259593963623, + 0.4010509252548218, + 0.6069756746292114, + 0.1967179924249649, + 0.6405487656593323, + 0.30995842814445496, + 0.06903589516878128, + -0.1344006508588791, + 0.4756820797920227, + -0.03781205788254738, + 0.6508035063743591, + -0.14448976516723633, + -0.33955514430999756, + 0.5968638062477112, + 0.2862204909324646, + 0.2697177529335022, + 0.41081514954566956, + 0.49110114574432373, + -0.37710583209991455, + 0.3493264615535736, + 0.26153215765953064, + 0.31147611141204834, + -0.11166010051965714, + 0.6022918224334717, + 0.34165677428245544, + -0.34675857424736023, + -0.2415717989206314, + 0.08801210671663284, + 0.6292260885238647, + 0.5155384540557861, + 0.5328304767608643 + ], + "spearman": [ + -0.012090679522669881, + 0.6977098954274739, + 0.1450131167532792, + -0.04749983087495771, + 0.08937925334481334, + -0.10078236419559106, + -0.0030729682682420667, + 0.2994747283686821, + 0.24681941720485429, + 0.4445384461346115, + 0.2151244512811128, + 0.5230794167698543, + -0.09617779354444839, + 0.14578339844584962, + 0.016144364536091135, + -0.18157021539255383, + 0.41461215765303944, + 0.37162702040675505, + 0.6083111270777818, + -0.19910819127704785, + 0.5227272826818207, + -0.07909027827256955, + -0.2634291793572948, + 0.06228738957184738, + 0.2968642672160668, + 0.47850050662512666, + 0.4742138535534634, + 0.4903983655995914, + 0.39055762963940743, + 0.13457892664473164, + 0.38983801045950256, + 0.28748236637059155, + -0.4623955155988788, + 0.015348783837195958, + 0.23744550536137635, + 0.6466587926646982, + -0.04224348756087189, + 0.03156737515368156, + 0.2035127023781756, + 0.38654380713595177, + -0.14385154646288661, + 0.2781673565418391, + 0.5253280193320048, + 0.25493990623497653, + 0.27062060265515064, + 0.13380912045228008, + -0.2755979323994831, + 0.5598281464570365, + 0.6065233186308296, + 0.4819712854928213, + 0.33117250829312705, + 0.4559284994821249, + 0.1753191528297882, + 0.4896906249226562, + 0.29153785588446396, + 0.09279895119973779, + -0.019934050983512745, + 0.3380452145113036, + -0.10794779548694886, + 0.5706981301745325, + -0.13142376135594033, + -0.3113019523254881, + 0.49452539913134974, + 0.18348677937169483, + 0.22354803788700944, + 0.36348784137196033, + 0.4486911626727906, + -0.3045453656363414, + 0.200340044085011, + 0.1644751776187944, + 0.3022850810712702, + -0.0745589384176942, + 0.597420161579073, + 0.2816811629202907, + -0.32133867233466806, + -0.17233515558378887, + 0.03997660149415037, + 0.5392369033092258, + 0.37754505438626357, + 0.4271651747912937 + ], + "de_corr": [ + 0.6897227168083191, + 0.8864696025848389, + 0.12434150278568268, + -0.10353504866361618, + 0.46459436416625977, + -0.3477477431297302, + -0.27771133184432983, + 0.5040098428726196, + 0.5618945360183716, + 0.8007484078407288, + 0.22932887077331543, + 0.7646661400794983, + -0.04479058086872101, + -0.12429239600896835, + 0.0353318490087986, + -0.3967788815498352, + 0.6527127027511597, + 0.5345252156257629, + 0.568474292755127, + -0.008411237969994545, + 0.8292492032051086, + 0.397029310464859, + -0.03514837101101875, + -0.5421869158744812, + 0.6469438672065735, + 0.7307199835777283, + 0.1250447779893875, + 0.3785933256149292, + 0.57857745885849, + 0.22240830957889557, + 0.05066901072859764, + 0.5418409705162048, + 0.18211904168128967, + -0.16815964877605438, + -0.35439446568489075, + 0.7907842397689819, + 0.008259566500782967, + 0.4223805367946625, + 0.2919152081012726, + 0.7320380210876465, + -0.4735271632671356, + 0.8487730026245117, + 0.8590567708015442, + 0.5939438939094543, + 0.44766297936439514, + 0.5950221419334412, + -0.36446473002433777, + 0.6831140518188477, + 0.5967740416526794, + 0.7253232002258301, + 0.6219841837882996, + 0.8695690631866455, + 0.02088514156639576, + 0.8723554015159607, + 0.6031556129455566, + 0.1709887981414795, + -0.5777860879898071, + 0.7562110424041748, + 0.21485692262649536, + 0.6774111986160278, + -0.2594827711582184, + -0.42325031757354736, + 0.7348797917366028, + 0.196257546544075, + 0.3747323751449585, + 0.17577248811721802, + 0.8413136005401611, + -0.36733999848365784, + 0.6309633255004883, + 0.5124343037605286, + 0.2654319405555725, + -0.2571105659008026, + 0.6562188267707825, + 0.39367181062698364, + -0.747912585735321, + -0.31291118264198303, + -0.07712656259536743, + 0.8206839561462402, + 0.7898404002189636, + 0.916015088558197 + ], + "mmd": [ + 0.015612012091381522, + 0.07244129962390444, + 0.03732324635244422, + 0.017966079853083072, + 0.026677115216721226, + 0.04194380406056519, + 0.016127729442044214, + 0.01369904368703756, + 0.015605133392873838, + 0.3531200418266687, + 0.01629205843466419, + 0.029364244570023956, + 0.022973928706202074, + 0.01998442841945902, + 0.04262022399414289, + 0.017350741184317564, + 0.1338621144736254, + 0.026131277525809682, + 0.0332266606644257, + 0.018543118315726526, + 0.0698254169995226, + 0.020134879337823763, + 0.020461732219173134, + 0.052363810031845426, + 0.01579423109582534, + 0.04831285054486123, + 0.07089491235511003, + 0.04683226907041538, + 0.037070414719125755, + 0.03172074092573296, + 0.03209863104703492, + 0.1039458682662725, + 0.01488132538187037, + 0.031231611265801873, + 0.02179662671362459, + 0.18561209522241373, + 0.028595809426273266, + 0.01863937783420777, + 0.04970434924224387, + 0.042040765423941306, + 0.018112510160466444, + 0.1145471296900692, + 0.08671286601753703, + 0.04086847870829802, + 0.05490606816994359, + 0.036677106487255196, + 0.025490840857557928, + 0.06678415441498087, + 0.0293197438013324, + 0.023698859622901747, + 0.0254326094080346, + 0.023755791637402823, + 0.016725383307354846, + 0.022966115705365775, + 0.06483577081145464, + 0.07479617700441044, + 0.03087995585038117, + 0.12237882456238658, + 0.023488062780999686, + 0.02745272302365309, + 0.030384945864470714, + 0.017931414862744743, + 0.019243247846852274, + 0.028563931044767732, + 0.013113530765531234, + 0.022313901202118513, + 0.1697025811948144, + 0.020046126236005213, + 0.03186086315320802, + 0.026666260846928247, + 0.15998976147536959, + 0.029796040331229512, + 0.18131855492208226, + 0.03386667549768585, + 0.019969124596080468, + 0.049657362418190076, + 0.05093044942678748, + 0.07621317055323595, + 0.08089058316486275, + 0.06284591005060025 + ], + "wasserstein": [ + 0.20109765342148145, + 0.38141535787733416, + 0.3319298437341127, + 0.21404119193806903, + 0.2532698630456588, + 0.2864221806507449, + 0.1918767008051754, + 0.16402139752201111, + 0.20738510618490602, + 0.9984526125146103, + 0.19237491673430016, + 0.2400263896477546, + 0.21384380788100174, + 0.18659729074990256, + 0.34276256590835635, + 0.19914072335293173, + 0.5767906602443618, + 0.2835776671282534, + 0.2561521667337757, + 0.2201273256478871, + 0.4078474988266915, + 0.1987723877740816, + 0.21895432608258783, + 0.3659234284390539, + 0.1902089705768061, + 0.3193041237788155, + 0.3712434263912862, + 0.3158606637327786, + 0.25256579776887184, + 0.2598440706180539, + 0.22488611941474054, + 0.4700802561379341, + 0.1917622225492707, + 0.31583101077819026, + 0.208513315658571, + 0.8161444245601654, + 0.2513303766326195, + 0.20657744191892527, + 0.3529377006426373, + 0.3384781339512265, + 0.2080608428568004, + 0.5503556958110224, + 0.42375695068357694, + 0.3647263607598856, + 0.3344000888887028, + 0.27132138763153973, + 0.263643538565005, + 0.36472518475081855, + 0.25841739989352047, + 0.2207243158781102, + 0.24307713096033048, + 0.24127512751103336, + 0.20153071205305628, + 0.22736481193372066, + 0.39716418564501965, + 0.46244011254729295, + 0.26165434701296114, + 0.4684223991128823, + 0.2282655417820913, + 0.26031085122150577, + 0.24459127527228708, + 0.2154395783006202, + 0.18650792584561024, + 0.2260926090488945, + 0.1541392421861213, + 0.23685270805218506, + 0.5070019415585381, + 0.22880875223607236, + 0.3105734593458843, + 0.22351871849931476, + 0.7094728692075722, + 0.27027927878296204, + 0.6563384802272009, + 0.27708720025610406, + 0.19827973354114609, + 0.3467715158384472, + 0.3281853019257982, + 0.377390148557869, + 0.4024475976889559, + 0.36800076705337303 + ] + }, + "n_perts": 80 + }, + "NearestPerturbationCentroid": { + "mse": 0.005708367927582003, + "r2": -3.6805508196353913, + "pearson": 0.0717333696957212, + "spearman": 0.05168680142046058, + "de_corr": 0.1191471351776272, + "mmd": 0.08350106605838317, + "wasserstein": 0.3939168081784377, + "_per_pert": { + "mse": [ + 0.002775015775114298, + 0.00555382901802659, + 0.003079378744587302, + 0.0026705777272582054, + 0.0032496952917426825, + 0.00409917626529932, + 0.002837124513462186, + 0.0025020253378897905, + 0.002442104509100318, + 0.04397495090961456, + 0.003224154468625784, + 0.003992007579654455, + 0.003761792555451393, + 0.003363958792760968, + 0.003967595752328634, + 0.00294712302275002, + 0.013915268704295158, + 0.003656757762655616, + 0.0042054434306919575, + 0.0028930201660841703, + 0.006417109631001949, + 0.002691220259293914, + 0.003484762739390135, + 0.008360848762094975, + 0.0020651340018957853, + 0.005540607962757349, + 0.007243866100907326, + 0.005349485669285059, + 0.0033827689476311207, + 0.003691357094794512, + 0.003909493796527386, + 0.012610295787453651, + 0.002954313298687339, + 0.003853762988001108, + 0.003202597377821803, + 0.02955160103738308, + 0.0032517006620764732, + 0.0031355612445622683, + 0.004354109987616539, + 0.004124769475311041, + 0.0028027717489749193, + 0.017625197768211365, + 0.008772746659815311, + 0.005304872523993254, + 0.007221899926662445, + 0.004211016930639744, + 0.003953391686081886, + 0.006835431791841984, + 0.003305850550532341, + 0.0035600934643298388, + 0.0036720691714435816, + 0.003941299859434366, + 0.0026211251970380545, + 0.003364634234458208, + 0.005260619334876537, + 0.00694065960124135, + 0.003300108015537262, + 0.0120886554941535, + 0.002798232948407531, + 0.002974161645397544, + 0.003024839796125889, + 0.0030078364070504904, + 0.001983393682166934, + 0.0031820693984627724, + 0.0025544248055666685, + 0.00316640711389482, + 0.00822489894926548, + 0.003703599562868476, + 0.004827210679650307, + 0.003453066572546959, + 0.01690433919429779, + 0.0034147275146096945, + 0.01725544035434723, + 0.00430149445310235, + 0.0032077908981591463, + 0.00511400680989027, + 0.005330512300133705, + 0.002068045549094677, + 0.009683116339147091, + 0.0034210081212222576 + ], + "r2": [ + -12.48040771484375, + 0.22454148530960083, + -1.075052261352539, + -6.732722759246826, + -2.7686970233917236, + -2.2296833992004395, + -9.264128684997559, + -2.8648335933685303, + -5.005887031555176, + -0.010895967483520508, + -11.315226554870605, + -0.9230440855026245, + -4.973606586456299, + -7.154361724853516, + -1.8103878498077393, + -10.116433143615723, + 0.07753872871398926, + -1.586517572402954, + -1.1234960556030273, + -7.377682685852051, + -0.19060981273651123, + -6.608341217041016, + -11.683067321777344, + -1.8328773975372314, + -4.220204830169678, + -0.5708807706832886, + -0.34067487716674805, + -0.6314358711242676, + -0.8472388982772827, + -4.563781261444092, + -2.189030408859253, + -0.4766199588775635, + -21.087886810302734, + -2.99617600440979, + -5.084766864776611, + -0.09194362163543701, + -5.075789451599121, + -8.803227424621582, + -0.880186915397644, + -0.5052205324172974, + -9.81586742401123, + -0.47359132766723633, + -0.12010157108306885, + -1.3098607063293457, + -0.9049588441848755, + -1.5645074844360352, + -7.085145950317383, + -0.44206368923187256, + -0.7072147130966187, + -1.7088160514831543, + -1.8763689994812012, + -1.0859594345092773, + -5.778806686401367, + -1.2594447135925293, + -0.6243786811828613, + -0.9024648666381836, + -2.943314790725708, + -0.07561123371124268, + -4.0611419677734375, + -1.3263227939605713, + -2.21382212638855, + -22.127117156982422, + -3.208404541015625, + -2.143723487854004, + -10.913413047790527, + -3.110107421875, + 0.3518509864807129, + -13.406034469604492, + -0.9975898265838623, + -2.7960469722747803, + -0.09278881549835205, + -5.268325328826904, + 0.005516231060028076, + -1.2455050945281982, + -11.836626052856445, + -3.4524831771850586, + -1.539609670639038, + 0.6542920470237732, + -0.12783396244049072, + 0.28059297800064087 + ], + "pearson": [ + 0.040950190275907516, + 0.5520670413970947, + 0.2949352264404297, + 0.1270199418067932, + 0.08379948139190674, + -0.13717515766620636, + -0.016923369839787483, + 0.3134763240814209, + 0.30443260073661804, + 0.12832075357437134, + -0.12237337231636047, + 0.18744917213916779, + -0.18839463591575623, + -0.0367615707218647, + 0.09397189319133759, + -0.013420735485851765, + 0.3006315231323242, + 0.07328757643699646, + 0.06809114664793015, + 0.06848412752151489, + 0.25736337900161743, + 0.16438859701156616, + -0.23693862557411194, + -0.32693690061569214, + 0.47959446907043457, + 0.10788293182849884, + 0.08924441039562225, + 0.10204946994781494, + 0.2674649655818939, + -0.09228792041540146, + 0.020998533815145493, + -0.120652936398983, + -0.29250749945640564, + -0.1349891573190689, + -0.045502446591854095, + 0.07061892002820969, + -0.06828445196151733, + -0.15700778365135193, + 0.11756821721792221, + 0.2323426455259323, + -0.0036526317708194256, + -0.01631554588675499, + 0.24310193955898285, + -0.08201180398464203, + -0.10763748735189438, + 0.042719706892967224, + -0.33680036664009094, + 0.06998996436595917, + 0.2448081523180008, + 0.09559549391269684, + 0.0318387970328331, + 0.10063743591308594, + 0.13100135326385498, + 0.18650692701339722, + 0.114848792552948, + -0.11383139342069626, + -0.012855380773544312, + 0.18107284605503082, + 0.0988704040646553, + 0.2811323404312134, + 0.10601114481687546, + -0.23266027867794037, + 0.433714896440506, + 0.17066460847854614, + 0.004271475598216057, + 0.13479971885681152, + 0.6769914627075195, + -0.4312286078929901, + 0.07111253589391708, + -0.01736949197947979, + 0.08847634494304657, + -0.03195406496524811, + 0.20600773394107819, + 0.07671056687831879, + -0.24971400201320648, + -0.38858258724212646, + -0.14172986149787903, + 0.8294099569320679, + 0.15451207756996155, + 0.5739294290542603 + ], + "spearman": [ + 0.047905532976383246, + 0.47972227943056983, + 0.35438633409658354, + 0.1889650012412503, + 0.043029619757404935, + -0.0804527976131994, + 0.04774034393508598, + 0.2034289063572266, + 0.27764595791148944, + -0.010549571637392908, + -0.1290160312540078, + 0.15639750609937653, + -0.14476797769199443, + -0.029059948264987066, + 0.08485366821341704, + 0.029388074847018712, + 0.26492527673131916, + -0.027749310937327732, + 0.007139812784953196, + 0.11300169575042392, + 0.1125054596263649, + 0.16361829640457407, + -0.22842103510525877, + -0.3638161089540272, + 0.3928561597140399, + -0.015466329866582466, + 0.15702613825653455, + 0.10211029952757489, + 0.19519428829857205, + 0.009164514791128698, + 0.038389935597483896, + -0.1353252668313167, + -0.22751582471427462, + -0.08258468664617166, + -0.04027246456811614, + 0.013012435753108936, + 0.0605511556377889, + -0.16589839147459787, + 0.06330627532656884, + 0.20405216101304025, + 0.07319584379896095, + -0.08014684403671102, + 0.22104831576207895, + -0.1242788755697189, + -0.14836346659086666, + 0.006126250531562632, + -0.3201874790468697, + 0.04472410118102529, + 0.19221744155436038, + 0.05398577699644425, + -0.008218883054720764, + -0.02939903084975771, + 0.12679051369762842, + 0.1686881011720253, + 0.14928983382245845, + -0.1299841659960415, + -0.018041637010409253, + 0.0035642086423887387, + 0.0964947196236799, + 0.22950836437709107, + 0.10839174009793502, + -0.16706155026538755, + 0.4460997875249469, + 0.09231464307866076, + -0.0076094584023645995, + 0.0994081328520332, + 0.6303141990785497, + -0.39193549698387425, + -0.02946834886708722, + -0.045404180351045084, + 0.0468091932022983, + -0.040534856648914735, + 0.158305472076368, + 0.01176827244206811, + -0.17812788103197025, + -0.3290287107571777, + -0.11575518943879735, + 0.7570251617562904, + 0.008476074619018654, + 0.444522636130659 + ], + "de_corr": [ + 0.4627281725406647, + 0.5847346782684326, + 0.4412709176540375, + 0.14813345670700073, + 0.08825524896383286, + -0.27607592940330505, + -0.14479492604732513, + 0.546600878238678, + 0.5115342140197754, + 0.6641858220100403, + -0.28457897901535034, + 0.4495815336704254, + -0.15066464245319366, + -0.17006850242614746, + -0.009612279944121838, + -0.12765875458717346, + 0.5670153498649597, + -0.04873455688357353, + 0.35111674666404724, + 0.07225529849529266, + 0.4345524311065674, + 0.3171515166759491, + -0.11298967897891998, + -0.8148773908615112, + 0.5133845210075378, + 0.3043084740638733, + -0.2143367975950241, + 0.01165721658617258, + 0.6780430674552917, + -0.1776619404554367, + -0.028674716129899025, + -0.09889513999223709, + -0.8606702089309692, + -0.6738924980163574, + -0.3771640658378601, + -0.11621119827032089, + -0.13651451468467712, + 0.06158582121133804, + 0.24193349480628967, + 0.7036532759666443, + -0.030576923862099648, + 0.24055425822734833, + 0.32758644223213196, + -0.0483349934220314, + -0.473682165145874, + 0.3182647228240967, + -0.3409825563430786, + 0.06544657796621323, + 0.2355632334947586, + 0.12002364546060562, + 0.32202965021133423, + 0.6086364984512329, + -0.018901970237493515, + 0.5017403364181519, + 0.5716832876205444, + -0.1689077764749527, + 0.1588938683271408, + 0.5301911234855652, + 0.024050481617450714, + 0.45638251304626465, + 0.3119121491909027, + -0.40808412432670593, + 0.4413498342037201, + 0.18814459443092346, + -0.11920631676912308, + -0.01477733813226223, + 0.8833320140838623, + -0.4483063817024231, + 0.44352641701698303, + 0.134155735373497, + 0.1720414161682129, + 0.18887881934642792, + 0.16141648590564728, + 0.08799620717763901, + -0.5561348795890808, + -0.7289972305297852, + -0.17588718235492706, + 0.9617927074432373, + 0.4150337278842926, + 0.8643184900283813 + ], + "mmd": [ + 0.05219576492141409, + 0.07889851084548927, + 0.05685209307502592, + 0.05036079613572131, + 0.05997102256627451, + 0.07902495455395231, + 0.05458265336941581, + 0.04300078471639002, + 0.048605035475542624, + 0.3795075166950631, + 0.05518389761308473, + 0.0666978378376617, + 0.06464007904260882, + 0.06168396124410891, + 0.07132592542765359, + 0.05387260600115307, + 0.14788053918231459, + 0.06838268723801932, + 0.0645274178387586, + 0.0548453751922916, + 0.09536479922651997, + 0.05169740047775495, + 0.06259930051564844, + 0.11330530294775598, + 0.037979138126772116, + 0.08684897495611299, + 0.11664089357715102, + 0.09296696881055322, + 0.06285983776124904, + 0.07591106665607972, + 0.07579793120005907, + 0.15635759141339822, + 0.054063013151032235, + 0.07251494934264324, + 0.05741446120047078, + 0.21674177593242716, + 0.07054485096755869, + 0.056492795915961014, + 0.07564508818180726, + 0.07322941227160695, + 0.05251724254643897, + 0.14800783440133236, + 0.119213497544428, + 0.09064955011213416, + 0.1066033967744785, + 0.0768935170317141, + 0.0696895982553487, + 0.11168089791869862, + 0.05884208580445183, + 0.06263915235879647, + 0.06850840450229234, + 0.058358386441722865, + 0.04976127332911029, + 0.061079495394589345, + 0.0986732719448915, + 0.11929611959002906, + 0.0648890561589458, + 0.14779170848680268, + 0.05589973994608988, + 0.06063605130958272, + 0.060537703596493686, + 0.05413708898822944, + 0.045527329704063924, + 0.06165017388451266, + 0.051743498354969275, + 0.058536877245668584, + 0.13233941730115428, + 0.06840988936568815, + 0.0653710993642529, + 0.06770299440648597, + 0.19207823966971782, + 0.06740186722182007, + 0.20982488716019465, + 0.0759745723884967, + 0.056828885344218816, + 0.10458242182325805, + 0.0985330630375586, + 0.04579248046816797, + 0.1057996180830374, + 0.05866788580628057 + ], + "wasserstein": [ + 0.3039940988700868, + 0.38383625909752667, + 0.3619847232645206, + 0.2894708433883089, + 0.35426734262803905, + 0.3315559903857778, + 0.299113843057621, + 0.2544418722122676, + 0.31385938457638674, + 1.0420245505605208, + 0.318993315295164, + 0.34349833104116767, + 0.3255579765134605, + 0.29442548758489373, + 0.3987172591117573, + 0.30073002794452897, + 0.546817744326097, + 0.3984919478712279, + 0.3464847453081712, + 0.3003434589469694, + 0.4776827711261298, + 0.28642210724056466, + 0.3349675443732946, + 0.5101122204609412, + 0.2578093204663691, + 0.4379814925673164, + 0.46996234220348776, + 0.4176002456558335, + 0.3626613959024197, + 0.367318734376945, + 0.3283973399386791, + 0.5654930460938172, + 0.3074792607067829, + 0.39911835113632416, + 0.2975595777600581, + 0.8527737502110125, + 0.34935037900258287, + 0.3241944378985564, + 0.380703337497849, + 0.41015493638972567, + 0.27823106110688073, + 0.5595640012574201, + 0.45242865538353927, + 0.46869732780340584, + 0.4203691195107247, + 0.3417242776927336, + 0.3769894280504267, + 0.4734618194291875, + 0.3366606326283432, + 0.3278343067365113, + 0.35367780008983263, + 0.36828200588807847, + 0.300196714524547, + 0.33575258901433847, + 0.4714105335450636, + 0.5390157962244185, + 0.35528148042904917, + 0.5649892233030266, + 0.33373166966799706, + 0.35630534277727477, + 0.3199586499121318, + 0.3124362928216076, + 0.2608963771398867, + 0.3244447414172524, + 0.27379280016397345, + 0.3471016966215186, + 0.4239616054803974, + 0.35016539524356705, + 0.41879757916712407, + 0.32877376158244176, + 0.736442759441735, + 0.3576513458788307, + 0.7281165419953295, + 0.39102381460751234, + 0.3011626440623093, + 0.4656381281636742, + 0.3941697954896977, + 0.29021867451031613, + 0.484998336526039, + 0.3426381359936866 + ] + }, + "n_perts": 80 + }, + "EndpointMLP": { + "mse": 0.003586451524461154, + "r2": -1.3224301666021347, + "pearson": 0.24995377372251823, + "spearman": 0.2152196827393794, + "de_corr": 0.30248952442780136, + "mmd": 0.05450787306777762, + "wasserstein": 0.32707639826764473, + "_per_pert": { + "mse": [ + 0.0011231935350224376, + 0.0038481440860778093, + 0.002897437894716859, + 0.0014841224765405059, + 0.0014127054018899798, + 0.002788542304188013, + 0.0010674939258024096, + 0.0010427202796563506, + 0.001191804651170969, + 0.03684421628713608, + 0.0013517566258087754, + 0.001576532842591405, + 0.0018845876911655068, + 0.00143820617813617, + 0.002818877110257745, + 0.0015250642318278551, + 0.013066423125565052, + 0.0012817876413464546, + 0.0010678012622520328, + 0.001714074402116239, + 0.0040601822547614574, + 0.0015582800842821598, + 0.00162046670448035, + 0.0036802988033741713, + 0.0008095600642263889, + 0.0024209285620599985, + 0.0035572254564613104, + 0.0021829064935445786, + 0.0013983584940433502, + 0.001724716741591692, + 0.0014995421515777707, + 0.007131185382604599, + 0.0015951752429828048, + 0.002267966279760003, + 0.0013007798697799444, + 0.021637924015522003, + 0.0016225995495915413, + 0.0016233180649578571, + 0.002925353590399027, + 0.001958011882379651, + 0.0016202294500544667, + 0.0119987353682518, + 0.005377018824219704, + 0.0021351317409425974, + 0.0035842773504555225, + 0.002342485124245286, + 0.002217607107013464, + 0.003091072663664818, + 0.0011216620914638042, + 0.0015049307839944959, + 0.001401988323777914, + 0.0015016567194834352, + 0.0011413900647312403, + 0.0011087291641160846, + 0.00333274994045496, + 0.004697535187005997, + 0.0019494163570925593, + 0.009752286598086357, + 0.0017165426397696137, + 0.000831128447316587, + 0.002194315893575549, + 0.0014752958668395877, + 0.0005743966321460903, + 0.0016120277578011155, + 0.0009139457833953202, + 0.0012527675135061145, + 0.01018777396529913, + 0.0015960449818521738, + 0.0026573443319648504, + 0.0011893102200701833, + 0.014509763568639755, + 0.001916231238283217, + 0.013134699314832687, + 0.0021499628201127052, + 0.0016677521634846926, + 0.0026040193624794483, + 0.0028869195375591516, + 0.0037208914291113615, + 0.006518505979329348, + 0.0037273080088198185 + ], + "r2": [ + -4.456223011016846, + 0.462699294090271, + -0.9524505138397217, + -3.2973122596740723, + -0.6383254528045654, + -1.1970527172088623, + -2.861971378326416, + -0.6106709241867065, + -1.9310145378112793, + 0.1530253291130066, + -4.163272380828857, + 0.24054712057113647, + -1.9926648139953613, + -2.4862654209136963, + -0.9967101812362671, + -4.7524824142456055, + 0.13380974531173706, + 0.09335917234420776, + 0.46082448959350586, + -3.96366024017334, + 0.2466869354248047, + -3.4054088592529297, + -4.897816181182861, + -0.24698293209075928, + -1.0463900566101074, + 0.3136148452758789, + 0.341638445854187, + 0.3342777490615845, + 0.23639416694641113, + -1.599571704864502, + -0.22319817543029785, + 0.16496407985687256, + -10.926308631896973, + -1.3517770767211914, + -1.4714136123657227, + 0.200469970703125, + -2.0318210124969482, + -4.075249195098877, + -0.26322293281555176, + 0.2854776978492737, + -5.252450466156006, + -0.003179192543029785, + 0.3134639859199524, + 0.07031571865081787, + 0.054556071758270264, + -0.42657244205474854, + -3.5352649688720703, + 0.3478798270225525, + 0.4207487106323242, + -0.14507687091827393, + -0.09819173812866211, + 0.20523810386657715, + -1.9518859386444092, + 0.25545793771743774, + -0.02908945083618164, + -0.28761470317840576, + -1.32936692237854, + 0.13227152824401855, + -2.10469651222229, + 0.3499099612236023, + -1.3314096927642822, + -10.343482971191406, + -0.21876633167266846, + -0.5926018953323364, + -3.2624902725219727, + -0.6261361837387085, + 0.19717007875442505, + -5.2081990242004395, + -0.09965872764587402, + -0.3074396848678589, + 0.06200963258743286, + -2.5175750255584717, + 0.2430071234703064, + -0.12234306335449219, + -5.673848628997803, + -1.2671759128570557, + -0.37541186809539795, + 0.3779914975166321, + 0.24076169729232788, + 0.21618086099624634 + ], + "pearson": [ + 0.09502872079610825, + 0.8276609778404236, + 0.0098153967410326, + -0.03348850458860397, + 0.23761668801307678, + -0.23369304835796356, + 0.14508400857448578, + 0.42380473017692566, + 0.21640387177467346, + 0.5708160996437073, + 0.3077346980571747, + 0.5381605625152588, + -0.12110285460948944, + 0.16989414393901825, + -0.05251817777752876, + -0.16090410947799683, + 0.41078901290893555, + 0.5201277136802673, + 0.682468831539154, + -0.2792333662509918, + 0.5496934056282043, + -0.01444360613822937, + -0.2130851447582245, + 0.1522333174943924, + 0.466625839471817, + 0.5611525774002075, + 0.6121105551719666, + 0.6051725149154663, + 0.5571375489234924, + 0.10451110452413559, + 0.3083171844482422, + 0.41998910903930664, + -0.5366905927658081, + 0.000531347468495369, + 0.1610681116580963, + 0.7156506776809692, + -0.08233795315027237, + -0.01220520306378603, + 0.2216917872428894, + 0.5380873084068298, + -0.3179812431335449, + 0.4823363423347473, + 0.6197230815887451, + 0.3984537720680237, + 0.325303316116333, + 0.13670040667057037, + -0.17350135743618011, + 0.6223534345626831, + 0.6872645020484924, + 0.43554720282554626, + 0.4145434498786926, + 0.5640649199485779, + 0.21518488228321075, + 0.6233673095703125, + 0.3257567584514618, + -0.012926502153277397, + -0.08529604971408844, + 0.4303792417049408, + -0.0655321329832077, + 0.6751367449760437, + -0.1996285617351532, + -0.3251562714576721, + 0.6130167841911316, + 0.33582597970962524, + 0.2716352045536041, + 0.39808833599090576, + 0.5406419634819031, + -0.3427172899246216, + 0.3430376648902893, + 0.3292279839515686, + 0.25142842531204224, + -0.12206794321537018, + 0.616595447063446, + 0.3371334671974182, + -0.3003275990486145, + -0.2181304544210434, + 0.06100047752261162, + 0.6749403476715088, + 0.5166482329368591, + 0.5245563387870789 + ], + "spearman": [ + 0.04040927810231952, + 0.7945701366425342, + 0.09768091245685846, + -0.023026624256656063, + 0.13992114348028586, + -0.17080410820102704, + 0.05282894070723518, + 0.3539240939810235, + 0.18567775941943984, + 0.4411194367798592, + 0.267870063967516, + 0.4500177255044313, + -0.08925348731337182, + 0.16713687128421784, + 0.057553602888400716, + -0.19270993967748493, + 0.4050699397674849, + 0.378477024119256, + 0.6054383533595884, + -0.24883885505302833, + 0.5116462939115735, + -0.12134902983725747, + -0.19610049702512425, + 0.1221400100350025, + 0.38952613038153255, + 0.47235107658776904, + 0.5748291252072812, + 0.4950337552584388, + 0.4478311134577783, + 0.10550843937710984, + 0.33710194977548746, + 0.2918662044665511, + -0.501767019941755, + 0.013856487464121865, + 0.2765249296312324, + 0.6810827272706818, + -0.047911741477935366, + 0.02981686795421699, + 0.19038085609521402, + 0.46201957800489446, + -0.18058462214615553, + 0.4113219068304767, + 0.5649539177384795, + 0.2964369066092266, + 0.23821018605254649, + 0.06525664931416232, + -0.23586021655350173, + 0.5536379779094944, + 0.6026745966686491, + 0.40842124510531125, + 0.3561744155436039, + 0.4032878043219511, + 0.2000863805215951, + 0.5035344478836119, + 0.3145049541262385, + 0.012540310635077658, + 0.009463059865764966, + 0.28397124499281123, + -0.11614487753621937, + 0.6168429687107422, + -0.17631247939423705, + -0.2980324525081131, + 0.5371960627990157, + 0.272148248037062, + 0.2312956838239209, + 0.36022520105630024, + 0.5021666635416658, + -0.26105880826470207, + 0.18966945391736348, + 0.2412429573107393, + 0.2559138894784723, + -0.08547737290139726, + 0.6048302427075607, + 0.2851739527934882, + -0.2597778139444535, + -0.15617122954280738, + 0.03141218585304646, + 0.6065327371331842, + 0.3592666468166617, + 0.42115206928801724 + ], + "de_corr": [ + 0.6472761631011963, + 0.936928391456604, + 0.011994044296443462, + -0.13699433207511902, + 0.6277158260345459, + -0.5803767442703247, + -0.2965421676635742, + 0.4722442626953125, + 0.341996431350708, + 0.7877206206321716, + 0.3056774139404297, + 0.7390341758728027, + -0.2040613889694214, + 0.007292353548109531, + 0.05458313971757889, + -0.3569762110710144, + 0.35677915811538696, + 0.5975459814071655, + 0.5686826109886169, + -0.20067548751831055, + 0.8186091780662537, + 0.31272706389427185, + 0.14163240790367126, + -0.3010391891002655, + 0.784136176109314, + 0.6897380948066711, + 0.16019591689109802, + 0.4378555417060852, + 0.6034249067306519, + 0.07066167891025543, + 0.01640596240758896, + 0.557061493396759, + 0.3083898425102234, + -0.3127228915691376, + -0.25274521112442017, + 0.8261823058128357, + 0.026458756998181343, + 0.3759273886680603, + 0.16771529614925385, + 0.7753993272781372, + -0.5850271582603455, + 0.8295484781265259, + 0.8234143257141113, + 0.6581287980079651, + 0.482865571975708, + 0.47781023383140564, + -0.46412166953086853, + 0.7015983462333679, + 0.690395712852478, + 0.6333187222480774, + 0.7163512706756592, + 0.8286314010620117, + 0.23394539952278137, + 0.8346809148788452, + 0.45052680373191833, + 0.04013703763484955, + -0.4896038770675659, + 0.7143059968948364, + 0.06568113714456558, + 0.6881471872329712, + -0.3480433523654938, + -0.37667202949523926, + 0.703842043876648, + 0.09472042322158813, + 0.503110945224762, + 0.12406931072473526, + 0.8758518099784851, + -0.3308272957801819, + 0.5972989201545715, + 0.484485387802124, + 0.02280898578464985, + -0.21749094128608704, + 0.7933748960494995, + 0.5636692047119141, + -0.7041853070259094, + -0.19212569296360016, + -0.1855798214673996, + 0.851809561252594, + 0.8253962397575378, + 0.8970557451248169 + ], + "mmd": [ + 0.02463338089666689, + 0.061118352609025206, + 0.04701877404125476, + 0.026328086324946653, + 0.03556345707978237, + 0.056143267087071846, + 0.024614084794407298, + 0.019563292855244163, + 0.024598541876872826, + 0.34318171601167036, + 0.024533945241825283, + 0.033093743528868846, + 0.03171331073415662, + 0.028588919375595823, + 0.04967152119291962, + 0.02854260550509613, + 0.13507994140325164, + 0.030823282454183976, + 0.033868835388561225, + 0.029766943355612918, + 0.06736793981491163, + 0.031209805811913904, + 0.030359512307540193, + 0.0554188666370129, + 0.022783877823716825, + 0.048888839490026426, + 0.06564858907081572, + 0.04694581811096599, + 0.039762084677964715, + 0.04409465727641426, + 0.03802612271945538, + 0.10443788737906956, + 0.026319538921031227, + 0.040225834500951074, + 0.03058273811910661, + 0.17678148810571837, + 0.039006513353560646, + 0.028852586954941306, + 0.05796604385292947, + 0.04102480260340102, + 0.029544813807473425, + 0.11128138551501499, + 0.07864118774851458, + 0.043709186116262955, + 0.05856426779864976, + 0.04321218980590946, + 0.035135762134303805, + 0.06228697626448809, + 0.02955174966057328, + 0.03124311942048863, + 0.03070603322107024, + 0.030185713764195654, + 0.02452224209971343, + 0.026536368397786947, + 0.0688983229049549, + 0.08431796511032275, + 0.03982676484273895, + 0.12354061216005785, + 0.033711159563934556, + 0.03214187150964787, + 0.04233924892905705, + 0.02941418700690135, + 0.02734623551123616, + 0.03533257594671135, + 0.022220362856682385, + 0.029646357860226624, + 0.15783773057552497, + 0.03009212755158308, + 0.039514509273090415, + 0.031120619890795842, + 0.16210299221294655, + 0.04084085209856725, + 0.17051654726137155, + 0.03907556724359751, + 0.030829974604091315, + 0.059193409426170795, + 0.05994613650534264, + 0.06754279151475417, + 0.08076118528191567, + 0.06324919273707941 + ], + "wasserstein": [ + 0.23871243307387188, + 0.3346605638732891, + 0.36242596563325186, + 0.24344671772501195, + 0.2745443047063416, + 0.3215851141593424, + 0.22905834674587658, + 0.18919363588845767, + 0.24696140223819363, + 0.9660656659758848, + 0.23047766967817598, + 0.24081510977251638, + 0.25071106650834596, + 0.21505683701619602, + 0.3522028329418898, + 0.23859097364679807, + 0.5685240632316683, + 0.2945361334383444, + 0.25097327951886134, + 0.2641573123398029, + 0.3921238063125795, + 0.23358588624033097, + 0.258829464722145, + 0.3675575747175733, + 0.21472938294125207, + 0.3268962698986273, + 0.35953168728206997, + 0.32604754880283104, + 0.2683668682129701, + 0.29172180321894836, + 0.24597304142515405, + 0.4686003536092056, + 0.2400623709156303, + 0.34607419682212487, + 0.22533276287498807, + 0.7793485619717433, + 0.2922014853762878, + 0.24238995518285553, + 0.37056931387263764, + 0.3271491512218677, + 0.24753720783273397, + 0.5228884588634822, + 0.39773862795954207, + 0.3682273858689565, + 0.33000614219723085, + 0.2870067178344599, + 0.308166873742198, + 0.3506969722901504, + 0.25450498341775074, + 0.24819729994462483, + 0.25367867457711624, + 0.25380467119181177, + 0.2366213651573321, + 0.2493617821479873, + 0.4161750938533445, + 0.49527504075101264, + 0.28748395122316284, + 0.47399615149356383, + 0.2708865904959986, + 0.2528456861799071, + 0.2716758617333982, + 0.26070560501618467, + 0.20397376880541473, + 0.24533908426196388, + 0.1881305956566167, + 0.27075120381550577, + 0.4818858304553284, + 0.2649333764589094, + 0.32524178418741145, + 0.2325449756187078, + 0.7112953348236656, + 0.30444613441982105, + 0.6257101114714256, + 0.29303119747492185, + 0.23838267783562317, + 0.3769536662683459, + 0.35780516860185474, + 0.35717986344210617, + 0.3953368078338088, + 0.36589822247425663 + ] + }, + "n_perts": 80 + }, + "Random": { + "mse": 0.006677762430626899, + "r2": -5.967309301346541, + "pearson": 0.07861651587300003, + "spearman": 0.060570616989009105, + "de_corr": 0.0693207856384106, + "mmd": 0.09497343294881602, + "wasserstein": 0.4134648025875022, + "_per_pert": { + "mse": [ + 0.0010714087402448058, + 0.008310685865581036, + 0.0011369789717718959, + 0.0006849450874142349, + 0.005138291511684656, + 0.001851098146289587, + 0.0004061683430336416, + 0.0009891907684504986, + 0.0038703458849340677, + 0.04388745501637459, + 0.0062617408111691475, + 0.004406824242323637, + 0.0012788097374141216, + 0.017692120745778084, + 0.013192248530685902, + 0.007360848132520914, + 0.013519012369215488, + 0.002311853924766183, + 0.002828317927196622, + 0.002092184964567423, + 0.00566841708496213, + 0.0009849645430222154, + 0.0010204187128692865, + 0.007547278888523579, + 0.0012441880535334349, + 0.004131954163312912, + 0.006882065907120705, + 0.003388597397133708, + 0.005721228662878275, + 0.007476172409951687, + 0.008691416122019291, + 0.00840443279594183, + 0.005043040495365858, + 0.0017399343196302652, + 0.0022140699438750744, + 0.028981423005461693, + 0.0007193429628387094, + 0.006111594382673502, + 0.00358200934715569, + 0.004431304056197405, + 0.00256142090074718, + 0.015168326906859875, + 0.008604544214904308, + 0.004340725485235453, + 0.004687618464231491, + 0.0017772462451830506, + 0.0012882775627076626, + 0.007022916339337826, + 0.004188288003206253, + 0.002025469671934843, + 0.0014687794027850032, + 0.0034318191464990377, + 0.002002198714762926, + 0.0020444062538444996, + 0.007825258187949657, + 0.004064055159687996, + 0.009291616268455982, + 0.014161684550344944, + 0.0009286750573664904, + 0.015186254866421223, + 0.009794197045266628, + 0.00303348689340055, + 0.0016723728040233254, + 0.005423890892416239, + 0.000797499087639153, + 0.008694063872098923, + 0.015244751237332821, + 0.019512606784701347, + 0.00812803115695715, + 0.0020775715820491314, + 0.01578957960009575, + 0.0007333817193284631, + 0.016681989654898643, + 0.0026479908265173435, + 0.021117519587278366, + 0.0018800348043441772, + 0.0021721101365983486, + 0.007371179759502411, + 0.008697140961885452, + 0.018407631665468216 + ], + "r2": [ + -4.204664707183838, + -0.16038739681243896, + 0.2338418960571289, + -0.9832749366760254, + -4.958916187286377, + -0.4584540128707886, + -0.4694328308105469, + -0.5279847383499146, + -8.51837158203125, + -0.008884668350219727, + -22.917821884155273, + -1.1228704452514648, + -1.0307087898254395, + -41.88637161254883, + -8.344533920288086, + -26.764829635620117, + 0.10380709171295166, + -0.6352325677871704, + -0.4281303882598877, + -5.058603286743164, + -0.05169975757598877, + -1.7845909595489502, + -2.713895082473755, + -1.5572185516357422, + -2.145033836364746, + -0.17149734497070312, + -0.27371394634246826, + -0.03342247009277344, + -2.124208688735962, + -10.268426895141602, + -6.089713096618652, + 0.01587134599685669, + -36.704227447509766, + -0.804232120513916, + -3.206615924835205, + -0.07087516784667969, + -0.3440892696380615, + -18.107694625854492, + -0.5467792749404907, + -0.6170817613601685, + -8.88449764251709, + -0.2681795358657837, + -0.09862542152404785, + -0.8900493383407593, + -0.23647797107696533, + -0.08234226703643799, + -1.6346771717071533, + -0.4816169738769531, + -1.1629254817962646, + -0.5411465167999268, + -0.15050983428955078, + -0.8163133859634399, + -4.178125858306885, + -0.3728752136230469, + -1.416290521621704, + -0.11397504806518555, + -10.102598190307617, + -0.26006317138671875, + -0.6796870231628418, + -10.878348350524902, + -9.406106948852539, + -22.324342727661133, + -2.548474073410034, + -4.358529567718506, + -2.719403028488159, + -10.285198211669922, + -0.20133602619171143, + -74.89895629882812, + -2.3635311126708984, + -1.2839291095733643, + -0.020724773406982422, + -0.34624946117401123, + 0.038565874099731445, + -0.3823281526565552, + -83.50604248046875, + -0.6368423700332642, + -0.03485584259033203, + -0.2322143316268921, + -0.012993216514587402, + -2.870957851409912 + ], + "pearson": [ + -0.07394499331712723, + 0.38327232003211975, + 0.4922369718551636, + 0.24992913007736206, + 0.1596084088087082, + -0.19467127323150635, + 0.41956645250320435, + 0.4732178747653961, + -0.19185833632946014, + 0.08126550167798996, + 0.38706615567207336, + 0.2664681375026703, + 0.047900907695293427, + 0.04526091739535332, + -0.1485881358385086, + 0.019804639741778374, + 0.34633946418762207, + 0.14941340684890747, + -0.11417879164218903, + 0.08445817977190018, + 0.2762632668018341, + -0.006389973685145378, + -0.25405484437942505, + 0.08582435548305511, + -0.19291524589061737, + 0.15713642537593842, + -0.03847264125943184, + 0.4006481170654297, + -0.2702997326850891, + 0.14122092723846436, + 0.12429474294185638, + 0.34660664200782776, + -0.3899100422859192, + -0.10831008106470108, + -0.3729121685028076, + 0.03619510680437088, + 0.122618667781353, + -0.03271499276161194, + -0.06665034592151642, + -0.18047432601451874, + -0.14742863178253174, + -0.07555611431598663, + 0.323653906583786, + 0.37756747007369995, + -0.1375139355659485, + 0.2685936987400055, + 0.0715644508600235, + 0.43730053305625916, + 0.6930716633796692, + 0.06588558107614517, + 0.16499271988868713, + 0.2805560231208801, + 0.05185748636722565, + 0.05179757624864578, + 0.2478073537349701, + 0.07464978098869324, + 0.07926826924085617, + -0.10503227263689041, + 0.0660596564412117, + 0.3947041928768158, + 0.10160470753908157, + -0.19572240114212036, + 0.37130066752433777, + 0.2726840078830719, + 0.022907119244337082, + -0.27365633845329285, + 0.15480978786945343, + -0.31564995646476746, + -0.05692048743367195, + -0.182278573513031, + 0.03785790875554085, + 0.20004452764987946, + 0.25373926758766174, + 0.0716307982802391, + -0.2189941555261612, + -0.10900420695543289, + 0.15471626818180084, + -0.18847469985485077, + 0.1833294779062271, + 0.16132734715938568 + ], + "spearman": [ + -0.07849068462267116, + 0.30678755719688927, + 0.41174785893696475, + 0.23385356846339211, + 0.07576654444163611, + -0.16115939760028417, + 0.3434800968700242, + 0.32284599571149886, + -0.14077782069445516, + 0.04210063952515988, + 0.3687185046796262, + 0.29953204038301007, + 0.11157820289455073, + 0.0967984831996208, + -0.11637577709394428, + -0.053502433375608345, + 0.07080860670215167, + 0.12068968367242092, + -0.04871761217940305, + 0.04542192685548171, + 0.33296408924102233, + -0.014562065140516283, + -0.24008624702156173, + 0.09855397863849466, + -0.27701493175373293, + 0.10664464466116116, + -0.06294865723716431, + 0.3998177454544363, + -0.11476578619144653, + 0.16068326217081552, + 0.17882643420660851, + 0.302157848039462, + -0.3247460566865142, + -0.06184557796139449, + -0.3915971973992993, + 0.026361654590413647, + 0.1266864856716214, + 0.010528205632051407, + -0.019259799314949828, + -0.24507676026919006, + -0.037412493353123334, + 0.01289440372360093, + 0.23188800247200061, + 0.3371470967867742, + -0.14896253074063268, + 0.26236363159090786, + -0.009572898893224723, + 0.42132701733175426, + 0.6350560047640011, + 0.12928892782223195, + 0.0139723099930775, + 0.18849124862281214, + 0.04936471534117884, + 0.014920121230030305, + 0.1981867065466766, + 0.05319609729902433, + 0.12103939975984992, + -0.17319207579801896, + -0.1592706608176652, + 0.3520647925161981, + 0.13248507912126978, + -0.16608574902143725, + 0.2810207052551763, + 0.19783968895992224, + -0.04469481669046473, + -0.3001235430308858, + 0.05074855118713779, + -0.25592724548181134, + -0.009142734785683697, + -0.25939038184759544, + -0.09470687667671916, + 0.09489236825867671, + 0.22501443225360804, + 0.03753443588360897, + -0.1431307047826762, + -0.09037451259362815, + 0.19620841305210324, + -0.06192992898248225, + 0.1700343740085935, + 0.1501607355401839 + ], + "de_corr": [ + -0.32088857889175415, + 0.6617040634155273, + 0.6333076357841492, + 0.3213602304458618, + 0.4129597842693329, + -0.18804456293582916, + 0.5866332650184631, + 0.559747040271759, + -0.46279603242874146, + 0.40259212255477905, + 0.49291524291038513, + 0.14782871305942535, + -0.027892543002963066, + -0.2740292251110077, + -0.3563442826271057, + 0.20284143090248108, + 0.8901569843292236, + -0.7518615126609802, + -0.8586809039115906, + 0.408949077129364, + 0.08098554611206055, + -0.044039927423000336, + -0.5801964998245239, + -0.4421440064907074, + -0.16913677752017975, + -0.18034014105796814, + -0.2936314642429352, + -0.13102726638317108, + -0.6457442045211792, + 0.3713304400444031, + -0.4108746647834778, + 0.49391791224479675, + 0.2540752589702606, + -0.43366679549217224, + -0.4742238223552704, + 0.42451393604278564, + 0.39721372723579407, + -0.4491572082042694, + -0.3588150441646576, + -0.5440247654914856, + -0.22610916197299957, + -0.10303591191768646, + 0.4854274094104767, + 0.6457905173301697, + -0.32313990592956543, + 0.5210903286933899, + 0.663002610206604, + 0.6283378005027771, + 0.7616052627563477, + -0.15083889663219452, + 0.6716588735580444, + 0.7218559384346008, + -0.17813625931739807, + 0.24978893995285034, + 0.07433053106069565, + 0.23308917880058289, + 0.10644571483135223, + -0.045061152428388596, + -0.023111285641789436, + 0.46189939975738525, + -0.0044242870062589645, + -0.3102158308029175, + 0.5429196953773499, + 0.17933528125286102, + 0.00861529540270567, + -0.8447163105010986, + 0.6266130805015564, + -0.48666831851005554, + -0.3957209885120392, + 0.11648251116275787, + 0.2739069163799286, + 0.8013081550598145, + 0.3441019058227539, + 0.04731523245573044, + -0.6044915318489075, + -0.18571841716766357, + 0.1742997169494629, + -0.06964527815580368, + 0.669327974319458, + 0.14267593622207642 + ], + "mmd": [ + 0.023361819692315988, + 0.11991498582481241, + 0.02902788114841104, + 0.01703074763030543, + 0.089267204030688, + 0.04164199862866447, + 0.011246776240074863, + 0.019437597731783884, + 0.06843637353444698, + 0.3785540252396259, + 0.09530218147629788, + 0.07680060161300617, + 0.028043164773075557, + 0.25012231208419133, + 0.19517817809472993, + 0.12053790970957434, + 0.14341016093879577, + 0.044139725868798, + 0.050116025524933216, + 0.04055495720516711, + 0.08860147664181484, + 0.024938358733045174, + 0.02268566014842599, + 0.10929050656236527, + 0.02037862908014365, + 0.07077262354248504, + 0.10532644785015932, + 0.06842675013511446, + 0.08618033384919344, + 0.13235742407315032, + 0.15137728753420598, + 0.11968189581847588, + 0.08058681985259308, + 0.03861924175921749, + 0.03741089217516791, + 0.21234647632091008, + 0.022517982397033176, + 0.09759288408087397, + 0.06725267356700204, + 0.07313712916350545, + 0.04205316558484051, + 0.11952604184516713, + 0.11634325042120608, + 0.0787533622768185, + 0.07071067238289885, + 0.04018412092917045, + 0.029017269102876275, + 0.11648044299755589, + 0.07374419318243786, + 0.037194556630563924, + 0.030520867999110024, + 0.05373025670939824, + 0.03897019026942894, + 0.03950973422831594, + 0.1317073195959726, + 0.07881308174989199, + 0.15225859928147412, + 0.16402045580075764, + 0.02147734853697192, + 0.23913407298803213, + 0.1616377042275131, + 0.055000486481153854, + 0.04324687824666806, + 0.09760134255865038, + 0.01739714095178002, + 0.13195441927403484, + 0.21522204351716923, + 0.28439446164951, + 0.10296098692498246, + 0.039942332645973244, + 0.16876522009051476, + 0.023749640097987967, + 0.20255240785124884, + 0.05031027754844952, + 0.29225970749432895, + 0.054662548981086356, + 0.04966368126933174, + 0.11317984100427314, + 0.09925529730225124, + 0.2483630950009037 + ], + "wasserstein": [ + 0.21347219026086123, + 0.4494622386443933, + 0.2877933864879557, + 0.21453263511135984, + 0.4221167087760322, + 0.29342267467812033, + 0.1569214618031662, + 0.18449419531745265, + 0.32653045628800476, + 1.1052152037194325, + 0.4064427401484893, + 0.38390341214936763, + 0.21790461402292138, + 0.6188617685855959, + 0.613310847805784, + 0.4223665147832428, + 0.5543359901180793, + 0.321716274662615, + 0.35140747128227473, + 0.2743211798597523, + 0.4586399266783008, + 0.2323164285515894, + 0.22670302280548707, + 0.46856100852557625, + 0.20684987304217642, + 0.3821340622360124, + 0.4294919699684128, + 0.36328782921000646, + 0.4178343463938501, + 0.4540987744115285, + 0.4173238035935458, + 0.4776058954077525, + 0.40909202752976886, + 0.31439748906022075, + 0.29034958360846846, + 0.9018220469542129, + 0.20832054959895682, + 0.40383294297472866, + 0.4215430413093263, + 0.4153511607276264, + 0.3040060184683039, + 0.5909209547001072, + 0.5252016968135108, + 0.4599711305638081, + 0.4028763044738926, + 0.27603554595178237, + 0.24730593333662124, + 0.4812876004229941, + 0.4153529655291879, + 0.2827917711403394, + 0.26403238104350213, + 0.34496110827984067, + 0.2574946766832668, + 0.28526289470853117, + 0.5343703165353291, + 0.46015227281029253, + 0.5573142707272042, + 0.5439680468918034, + 0.20203507141814026, + 0.6528582043242417, + 0.4434558241651954, + 0.2937875347742926, + 0.2823198756460329, + 0.39377588604530644, + 0.19827138791508972, + 0.4358747854706965, + 0.5858969220825007, + 0.7832207257932079, + 0.4482542198781067, + 0.27945395667774464, + 0.7060302432635049, + 0.23794314570148406, + 0.72580450643946, + 0.3407942808086171, + 0.8083209041970622, + 0.3202590619123087, + 0.3157979144019084, + 0.46454850156429583, + 0.47571157832840394, + 0.7313760400198122 + ] + }, + "n_perts": 80 + } + } +} \ No newline at end of file diff --git a/experiments/results/replogle_k562_forward_perturbation.json b/experiments/results/replogle_k562_forward_perturbation.json new file mode 100644 index 0000000000000000000000000000000000000000..99e83b19904ff01c0aabb24818c60def24740a97 --- /dev/null +++ b/experiments/results/replogle_k562_forward_perturbation.json @@ -0,0 +1,4693 @@ +{ + "split": "perturbation", + "models": { + "PIVOT": { + "mse": 0.003554715416976251, + "r2": -0.2246327854692936, + "pearson": 0.21947074643103406, + "spearman": 0.17854213131891875, + "de_corr": 0.2589861237909645, + "mmd": 0.30655516100487235, + "wasserstein": 0.5696810464977814, + "_per_pert": { + "mse": [ + 0.00032930049928836524, + 0.006490429397672415, + 0.0018926572520285845, + 0.0005527318571694195, + 0.0008916478254832327, + 0.001560605363920331, + 0.00032671570079401135, + 0.0007132198079489172, + 0.0005599033902399242, + 0.042110148817300797, + 0.00044550257734954357, + 0.0015542173059657216, + 0.0009289868758060038, + 0.000565923226531595, + 0.001961003290489316, + 0.0005967390607111156, + 0.01376851461827755, + 0.0009856144897639751, + 0.0016810069791972637, + 0.0006050305091775954, + 0.005214446224272251, + 0.0007103198440745473, + 0.0007001936319284141, + 0.003270950401201844, + 0.0005046002916060388, + 0.0033381113316863775, + 0.004670143127441406, + 0.0026250770315527916, + 0.0016742963343858719, + 0.0006901241722516716, + 0.0011189522920176387, + 0.007821081206202507, + 0.00038539074012078345, + 0.001142647466622293, + 0.0004883137298747897, + 0.027227012440562248, + 0.0007697748369537294, + 0.0004809390811715275, + 0.002268396783620119, + 0.002074942458420992, + 0.0005298242322169244, + 0.013472349382936954, + 0.007516087032854557, + 0.0019308102782815695, + 0.0037861643359065056, + 0.0015904039610177279, + 0.000831897952593863, + 0.003902199910953641, + 0.0016755323158577085, + 0.0009263086249120533, + 0.0011618288699537516, + 0.001902756397612393, + 0.00044743670150637627, + 0.000986164202913642, + 0.003232812974601984, + 0.0038625800516456366, + 0.000882247812114656, + 0.011747113429009914, + 0.0006094747805036604, + 0.0010782371973618865, + 0.0013474817387759686, + 0.0003855415852740407, + 0.0003481956373434514, + 0.001002608798444271, + 0.00042998738354071975, + 0.0006617963663302362, + 0.012336650863289833, + 0.0006963751511648297, + 0.002797316526994109, + 0.000990133616141975, + 0.014069371856749058, + 0.0008011366007849574, + 0.016097022220492363, + 0.0019295383244752884, + 0.0005816128104925156, + 0.0016392822144553065, + 0.0016513020964339375, + 0.005356024485081434, + 0.008628016337752342, + 0.004859996028244495 + ], + "r2": [ + -0.5996682643890381, + 0.09376770257949829, + -0.2753748893737793, + -0.6004486083984375, + -0.034050822257995605, + -0.22957897186279297, + -0.1819899082183838, + -0.10169768333435059, + -0.37697482109069824, + 0.03197205066680908, + -0.7016756534576416, + 0.2512969374656677, + -0.4752013683319092, + -0.3718189001083374, + -0.3890477418899536, + -1.2508764266967773, + 0.08726722002029419, + 0.3028499484062195, + 0.15119266510009766, + -0.7520633935928345, + 0.03252851963043213, + -1.008143663406372, + -1.548410177230835, + -0.1082848310470581, + -0.27551841735839844, + 0.0535740852355957, + 0.13566267490386963, + 0.19942885637283325, + 0.08571195602416992, + -0.040186405181884766, + 0.0872545838356018, + 0.08417963981628418, + -1.88136887550354, + -0.18487310409545898, + 0.07222956418991089, + -0.006049156188964844, + -0.4383211135864258, + -0.5036400556564331, + 0.02046346664428711, + 0.24280714988708496, + -1.0445866584777832, + -0.12638378143310547, + 0.04034847021102905, + 0.15928184986114502, + 0.0013033151626586914, + 0.03144478797912598, + -0.7013281583786011, + 0.17675703763961792, + 0.13471770286560059, + 0.29518723487854004, + 0.08992773294448853, + -0.0070465803146362305, + -0.15716981887817383, + 0.33776360750198364, + 0.0017691254615783691, + -0.05874991416931152, + -0.05420207977294922, + -0.045221924781799316, + -0.10235202312469482, + 0.15662705898284912, + -0.4316680431365967, + -1.9644114971160889, + 0.26119160652160645, + 0.009473264217376709, + -1.0053892135620117, + 0.14096510410308838, + 0.027831554412841797, + -1.7087178230285645, + -0.15758180618286133, + -0.08847999572753906, + 0.09047895669937134, + -0.47062551975250244, + 0.07227951288223267, + -0.007275104522705078, + -1.3274412155151367, + -0.42723238468170166, + 0.2132720947265625, + 0.1046520471572876, + -0.0049419403076171875, + -0.022013187408447266 + ], + "pearson": [ + 0.26537150144577026, + 0.4986146092414856, + -0.030948791652917862, + 0.049742989242076874, + 0.21616661548614502, + -0.04233824461698532, + 0.3672877550125122, + 0.22827021777629852, + 0.1915128231048584, + 0.3241843283176422, + 0.1154043897986412, + 0.5260506272315979, + 0.03633866831660271, + 0.189438134431839, + -0.20044033229351044, + -0.06577067822217941, + 0.4618605375289917, + 0.5715621709823608, + 0.5175926089286804, + -0.14318998157978058, + 0.3425867259502411, + -0.13327668607234955, + -0.28042325377464294, + 0.12712803483009338, + 0.228920578956604, + 0.25859424471855164, + 0.45665594935417175, + 0.5441624522209167, + 0.3281431496143341, + 0.28550025820732117, + 0.3914056718349457, + 0.32282158732414246, + -0.05497036501765251, + 0.08327391743659973, + 0.36992964148521423, + 0.33156469464302063, + 0.1025797575712204, + 0.2548922300338745, + 0.20976336300373077, + 0.5677556395530701, + -0.05649489536881447, + 0.3545188903808594, + 0.417149156332016, + 0.41567566990852356, + 0.17115721106529236, + 0.2218881994485855, + -0.09945430606603622, + 0.5406379699707031, + 0.38432061672210693, + 0.6151050925254822, + 0.33070674538612366, + 0.2050923854112625, + 0.2663467824459076, + 0.6627833247184753, + 0.203647643327713, + 0.021198933944106102, + 0.1890280544757843, + 0.010523666627705097, + 0.29445186257362366, + 0.6373123526573181, + -0.24844931066036224, + -0.020957428961992264, + 0.518423318862915, + 0.28891634941101074, + -0.0651136040687561, + 0.42380523681640625, + 0.3406451344490051, + -0.2087816596031189, + 0.06278211623430252, + 0.21950525045394897, + 0.4559398889541626, + 0.0546746701002121, + 0.3728513717651367, + 0.20449982583522797, + -0.130028635263443, + -0.3451337218284607, + 0.4718112647533417, + 0.3566877543926239, + 0.06649459898471832, + 0.1397743970155716 + ], + "spearman": [ + 0.30474930868732714, + 0.4436063129015782, + 0.1267164196791049, + 0.11442068710517177, + 0.02375587343896836, + -0.05141090485272621, + 0.30723087630771906, + 0.07581619145404786, + 0.14968814692203672, + 0.1320116085029021, + 0.10319062079765519, + 0.45633140258285065, + 0.13788577047144263, + 0.11892392673098166, + -0.11247694611923652, + 0.008530313132578283, + 0.43679167469791863, + 0.39602810700702673, + 0.3994883003720751, + -0.10469633017408253, + 0.2264781836195459, + -0.11632245508061376, + -0.16304675426168855, + 0.14393069548267387, + 0.1499695664923916, + 0.13398016399504098, + 0.43031625557906383, + 0.44898511474627867, + 0.2077433904358476, + 0.23067157566789392, + 0.43389115697278924, + 0.1706653191663298, + -0.02712928128232032, + 0.11702871825717956, + 0.30487990471997617, + 0.23782935545733883, + 0.12014821353705338, + 0.25233898058474513, + 0.09945630086407521, + 0.499128732282183, + 0.07254246013561502, + 0.3017429869357467, + 0.39555068238767055, + 0.30599354549838637, + 0.13660762015190503, + 0.15597005549251386, + -0.1463834735958684, + 0.5622873595718398, + 0.343553695888424, + 0.5489832817458203, + 0.2566462541615635, + -0.02394105748526437, + 0.25389471197367797, + 0.5864221796055449, + 0.15730372382593097, + -0.025056133764033442, + 0.19576498894124722, + -0.14151790737947684, + 0.19496943974235995, + 0.528260616065154, + -0.11127315381828845, + 0.033927189981797494, + 0.4081960815490204, + 0.15194906598726649, + -0.009700355425088856, + 0.36471295317823826, + 0.31817852704463173, + -0.14397946949486734, + -0.10789237497309373, + 0.09454306013576502, + 0.39934510673603113, + 0.05631493034985069, + 0.3542738430684607, + 0.21055180063795015, + -0.12416341604085401, + -0.25591575997893995, + 0.4217505544376386, + 0.2594537853634463, + -0.08323989130997282, + 0.019218501304625324 + ], + "de_corr": [ + 0.7246350646018982, + 0.7319033145904541, + 0.054268915206193924, + -0.22486615180969238, + 0.5038124322891235, + 0.13352273404598236, + 0.39656126499176025, + 0.34915807843208313, + 0.6377226114273071, + 0.5154032707214355, + -0.2342742532491684, + 0.6024112701416016, + -0.06243693456053734, + 0.2799561321735382, + -0.10675972700119019, + -0.4079165756702423, + 0.4609520137310028, + 0.6039023995399475, + 0.8041831254959106, + 0.047678522765636444, + 0.6009846329689026, + -0.18138673901557922, + -0.6741742491722107, + -0.2759869396686554, + 0.43559950590133667, + 0.17277102172374725, + 0.09817469865083694, + 0.44085729122161865, + 0.5419723391532898, + 0.1824272871017456, + 0.04657416045665741, + 0.43541303277015686, + -0.3722631633281708, + -0.10018515586853027, + 0.3158436417579651, + 0.4314202070236206, + 0.10723572224378586, + 0.672145426273346, + 0.33215904235839844, + 0.7844231128692627, + -0.19178099930286407, + 0.6551980376243591, + 0.7716969847679138, + 0.5754251480102539, + 0.11847108602523804, + 0.5213597416877747, + -0.4718112349510193, + 0.5342247486114502, + 0.37475740909576416, + 0.7432939410209656, + 0.494350790977478, + 0.4358676075935364, + -0.2157966047525406, + 0.8942629098892212, + 0.06154732033610344, + -0.05262061208486557, + 0.22690793871879578, + 0.06019843369722366, + 0.58687824010849, + 0.8507490754127502, + -0.37286269664764404, + -0.07353424280881882, + 0.8160513639450073, + 0.2746753692626953, + -0.28310683369636536, + 0.4886159598827362, + 0.48748862743377686, + -0.3577195107936859, + 0.2033940851688385, + 0.6220433115959167, + 0.41393980383872986, + 0.11771465837955475, + 0.3430480659008026, + 0.09543273597955704, + -0.45248568058013916, + -0.43215566873550415, + 0.796271800994873, + 0.5698059797286987, + 0.2158425748348236, + 0.46942785382270813 + ], + "mmd": [ + 0.2683781243861453, + 0.32850269760230155, + 0.2811607297783786, + 0.25794614275260463, + 0.31145132644146456, + 0.3055893003288833, + 0.2717431321511854, + 0.2696866259662054, + 0.27007526487886335, + 0.5662474862072095, + 0.26815015781397766, + 0.3025331335955389, + 0.2879901738752575, + 0.27619543376294564, + 0.29985685562309883, + 0.27505244367837, + 0.37344490341898917, + 0.3162635634816996, + 0.3204444846832273, + 0.2784619295870072, + 0.33433094087180404, + 0.2560057323718411, + 0.27383629158089684, + 0.30916302916961735, + 0.2756114553602913, + 0.32899820734096363, + 0.34101265411251824, + 0.3097124864661477, + 0.31911349790338994, + 0.2648704031001786, + 0.27872853309728185, + 0.35075860288917105, + 0.24548134482514217, + 0.27852506522780374, + 0.2843783936229587, + 0.4362311498769911, + 0.26983669683441647, + 0.25304827217160164, + 0.3123033327059467, + 0.3334298814936568, + 0.26131701325141066, + 0.3714038305766021, + 0.36901473267055074, + 0.31231698011858333, + 0.32356384513239855, + 0.2773129525124596, + 0.271235418193157, + 0.34577740311593874, + 0.29907940186663484, + 0.27697409659038674, + 0.2781607299776756, + 0.3040293292173668, + 0.27602687651044466, + 0.28163700362244604, + 0.31731293795026183, + 0.3243168804100506, + 0.26683095834080073, + 0.3703102790212843, + 0.2720098976334817, + 0.3028213008400885, + 0.28018049180705806, + 0.2649461532327111, + 0.29251140745724835, + 0.28730332616713405, + 0.2556147010210059, + 0.2954794600656738, + 0.43515360140558823, + 0.2626603986013447, + 0.3179085573261484, + 0.30194839064418644, + 0.3790339186487026, + 0.2589807795469199, + 0.41698772002966467, + 0.30829042256480144, + 0.26725383385579027, + 0.29772911855415585, + 0.3076005208495345, + 0.3577646994657846, + 0.31721759930741744, + 0.33384603125092416 + ], + "wasserstein": [ + 0.5099603627800954, + 0.6917120977062357, + 0.5155713197148523, + 0.46302096806911797, + 0.5543428112935735, + 0.5228969362251378, + 0.4782137066307795, + 0.5296925667095226, + 0.4775708744456096, + 1.2290466520841574, + 0.49468418343217535, + 0.5368647282400762, + 0.507759788610062, + 0.4940188372388846, + 0.49448673956507155, + 0.5050448355122379, + 0.8393223118530484, + 0.5946246798646834, + 0.7026705928072707, + 0.4993581982123028, + 0.6140928182359946, + 0.46659045423814294, + 0.4896416651071622, + 0.5613228416156167, + 0.5097543717952078, + 0.6128060082294324, + 0.5359031763476042, + 0.5202820832073491, + 0.6366815798682094, + 0.4395347241991868, + 0.5195349263873211, + 0.7817015857456389, + 0.4557433298761462, + 0.48429852548099595, + 0.5107770861035005, + 1.0578833828046599, + 0.4392818949064734, + 0.4683571650491791, + 0.5094571454472759, + 0.5682001650597482, + 0.47234741282426973, + 0.8356612929034216, + 0.6866021117758077, + 0.4847028031974682, + 0.5857846398028916, + 0.5115257258540794, + 0.5035317237686432, + 0.641889569322154, + 0.5539867098043059, + 0.4891082542947305, + 0.5096841132160485, + 0.6107473182842663, + 0.5071050162471387, + 0.52563586979024, + 0.5258723606277357, + 0.5578524200276737, + 0.46979118165073125, + 0.7756005657952388, + 0.4517533326356575, + 0.5467457305568595, + 0.48784795861107966, + 0.49544274989395426, + 0.5324496813015005, + 0.5271193349240708, + 0.4824543676271139, + 0.5351844358122362, + 0.7117753997107993, + 0.4546136850759747, + 0.6376655470203271, + 0.5628939394918145, + 0.8688451848225653, + 0.41875057576312213, + 0.8723196935160378, + 0.5420656561438675, + 0.5018393146274653, + 0.4198310630676122, + 0.47626396773347596, + 0.6223850886274808, + 0.7245937626591105, + 0.6315060403137932 + ] + }, + "n_perts": 80 + }, + "MeanControl": { + "mse": 0.0037895372535786008, + "r2": -0.14595426246523857, + "pearson": 0.004628284195496235, + "spearman": -0.0007950709341632556, + "de_corr": 0.04540404016152024, + "mmd": 0.05126164886924247, + "wasserstein": 0.315581765298599, + "_per_pert": { + "mse": [ + 0.0003135530569124967, + 0.007565529551357031, + 0.00159974314738065, + 0.00041434390004724264, + 0.001057824119925499, + 0.0012700867373496294, + 0.00038360542384907603, + 0.0007954876637086272, + 0.00036734045716002584, + 0.043578583747148514, + 0.0003190361603628844, + 0.0018691866425797343, + 0.0007922602817416191, + 0.0005640173912979662, + 0.0019363417522981763, + 0.0003283632395323366, + 0.01513818558305502, + 0.0012615913292393088, + 0.002322000451385975, + 0.00032479382934980094, + 0.0060366191901266575, + 0.0003600410127546638, + 0.000425270787673071, + 0.003986774943768978, + 0.0005002517718821764, + 0.003681278321892023, + 0.005548305343836546, + 0.003138965228572488, + 0.0023685062769800425, + 0.0007131922757253051, + 0.0014301537303254008, + 0.008315999060869217, + 0.0001708415220491588, + 0.0010590070160105824, + 0.0007750060758553445, + 0.02823387086391449, + 0.0005691223777830601, + 0.00048193958355113864, + 0.002259922446683049, + 0.00288140750490129, + 0.0002663169871084392, + 0.01575201004743576, + 0.0074739293195307255, + 0.002405586652457714, + 0.003736414248123765, + 0.001574805355630815, + 0.0008918708772398531, + 0.0047572460025548935, + 0.00199701264500618, + 0.0012434691889211535, + 0.0013458342291414738, + 0.001973779173567891, + 0.0004220961418468505, + 0.0013149866135790944, + 0.0036813688930124044, + 0.003905249759554863, + 0.0010133606847375631, + 0.011102687567472458, + 0.0006861949223093688, + 0.0016585740959271789, + 0.0010814041597768664, + 0.00024069577921181917, + 0.0005773089942522347, + 0.0012526626233011484, + 0.00018732303578872234, + 0.0007441260968334973, + 0.013510736636817455, + 0.0004385379434097558, + 0.002753057749941945, + 0.0010888457763940096, + 0.01575833186507225, + 0.0006098570302128792, + 0.017467468976974487, + 0.0019267674069851637, + 0.00036751842708326876, + 0.0013341380981728435, + 0.0023392115253955126, + 0.006029635202139616, + 0.008585790172219276, + 0.004528419580310583 + ], + "r2": [ + -0.5231711864471436, + -0.05634415149688721, + -0.07799339294433594, + -0.19974267482757568, + -0.22676682472229004, + -0.0006834268569946289, + -0.38780510425567627, + -0.22877538204193115, + 0.0965965986251831, + -0.001784205436706543, + -0.21861493587493896, + 0.09956890344619751, + -0.2580840587615967, + -0.36719954013824463, + -0.37157928943634033, + -0.23857367038726807, + -0.003530144691467285, + 0.10764443874359131, + -0.17247068881988525, + 0.05945366621017456, + -0.12001478672027588, + -0.01787126064300537, + -0.5478066205978394, + -0.35082530975341797, + -0.26452624797821045, + -0.04372107982635498, + -0.026865124702453613, + 0.04270803928375244, + -0.2933773994445801, + -0.07495594024658203, + -0.1665971279144287, + 0.026226699352264404, + -0.2772942781448364, + -0.09814167022705078, + -0.4724709987640381, + -0.04325294494628906, + -0.06340301036834717, + -0.506767988204956, + 0.02412283420562744, + -0.051489949226379395, + -0.02771472930908203, + -0.31697940826416016, + 0.04573112726211548, + -0.047446489334106445, + 0.014426171779632568, + 0.04094433784484863, + -0.8239794969558716, + -0.003631114959716797, + -0.03130173683166504, + 0.05386495590209961, + -0.05420541763305664, + -0.044635891914367676, + -0.09163355827331543, + 0.11695027351379395, + -0.13673663139343262, + -0.07044601440429688, + -0.21086955070495605, + 0.01211702823638916, + -0.24111509323120117, + -0.2972996234893799, + -0.14896690845489502, + -0.8506988286972046, + -0.2249457836151123, + -0.23756706714630127, + 0.12635666131973267, + 0.03409820795059204, + -0.0646902322769165, + -0.7057983875274658, + -0.13926661014556885, + -0.19699668884277344, + -0.018704533576965332, + -0.1194983720779419, + -0.006703615188598633, + -0.0058286190032958984, + -0.4706993103027344, + -0.1615605354309082, + -0.11446774005889893, + -0.007953166961669922, + -2.372264862060547e-05, + 0.04771435260772705 + ], + "pearson": [ + -0.21644580364227295, + 0.23483537137508392, + -0.0005317168543115258, + 0.13985086977481842, + -0.19946721196174622, + 0.12864293158054352, + -0.17481984198093414, + -0.08385606855154037, + 0.3229949176311493, + 0.21456672251224518, + 0.2328033596277237, + 0.4145580232143402, + -0.13677188754081726, + -0.17974503338336945, + -0.39655187726020813, + 0.005322729703038931, + -0.006408405490219593, + 0.3476566970348358, + -0.1814814656972885, + 0.2700512111186981, + -0.02433258295059204, + 0.23205558955669403, + -0.16232909262180328, + -0.4905180335044861, + -0.15489687025547028, + 0.030346892774105072, + -0.01711989752948284, + 0.22520405054092407, + -0.49047741293907166, + 0.17707619071006775, + 0.10346878319978714, + 0.19181008636951447, + 0.005092858336865902, + 0.001998172141611576, + -0.3337039649486542, + 0.13154591619968414, + 0.13716234266757965, + -0.059707339853048325, + 0.1893775761127472, + -0.07294772565364838, + 0.19234991073608398, + -0.27877891063690186, + 0.5292931199073792, + 0.0016831941902637482, + 0.2390739917755127, + 0.20871996879577637, + -0.3954814076423645, + 0.03975219652056694, + -0.013415707275271416, + 0.297225683927536, + -0.10845108330249786, + 0.011838908307254314, + 0.01556172501295805, + 0.41493499279022217, + -0.34031444787979126, + -0.08314527571201324, + -0.2369609922170639, + 0.21057097613811493, + -0.2164701670408249, + 0.031848758459091187, + -0.026233380660414696, + -0.22145536541938782, + -0.16412152349948883, + -0.15401369333267212, + 0.39817267656326294, + 0.20422665774822235, + 0.044849853962659836, + -0.24804569780826569, + -0.2711746096611023, + -0.02838457189500332, + -0.05656258761882782, + 0.032593511044979095, + 0.06686683744192123, + 0.08814864605665207, + -0.2807958126068115, + -0.08646180480718613, + -0.18501870334148407, + 0.003255219431594014, + 0.06884294003248215, + 0.3114296495914459 + ], + "spearman": [ + -0.2851684687921172, + 0.18591052347763085, + 0.13271025017756255, + 0.05135837783959445, + -0.15025929906482477, + 0.21325318131329532, + -0.18941778535444634, + -0.13669835067458766, + 0.300010524002631, + 0.09210283402570851, + 0.2110023167505792, + 0.40189168497292116, + -0.20181448902930268, + -0.13205372101343024, + -0.29198597949649485, + 0.020944190236047558, + -0.06909687377421844, + 0.20337405284351323, + -0.10753343688335922, + 0.183818571954643, + 0.18253596063399016, + 0.1583769215942304, + -0.19224566706141677, + -0.43370692692673174, + -0.16069139367284843, + -0.0416239664059916, + -0.08766276391569097, + 0.14421377405344352, + -0.38826861706715426, + 0.16490751972687992, + 0.24116854979213745, + 0.11093013823253456, + 0.019369273842318457, + -0.1248129132032283, + -0.14487105771776443, + 0.1018206524551631, + 0.203658827914707, + -0.04712122928030731, + 0.18127947331986835, + -0.058616794154198536, + 0.22842015310503827, + -0.30412270303067573, + 0.4070255847563962, + -0.05600968100242024, + 0.17591752547938136, + 0.15667782466945615, + -0.31603919950979986, + 0.09133868883467221, + 0.04179049194762298, + 0.2560116460029115, + -0.08960167790041947, + -0.03707772326943081, + 0.035176136794034196, + 0.3319584829896208, + -0.19913080378270093, + 0.00488276372069093, + -0.1056317949079487, + 0.1540773460193365, + -0.3700087555021888, + -0.013817911954477987, + -0.07298156724539182, + -0.2183526860881715, + -0.14457450164362542, + -0.20079306819826703, + 0.22561346340336583, + 0.2607705396926349, + 0.0680014940003735, + -0.2806478936619734, + -0.27926900181725045, + -0.13235749058937263, + -0.0347957651989413, + 0.03108909628393249, + 0.16514513778628445, + 0.092081288020322, + -0.2940463365115841, + -0.04698630501419612, + -0.16146368386592094, + -0.02465691016422754, + 0.06946570286642571, + 0.26232855408213857 + ], + "de_corr": [ + -0.03622117266058922, + 0.38581591844558716, + -0.14239460229873657, + 0.25438085198402405, + -0.41583362221717834, + 0.09088260680437088, + -0.12077896296977997, + -0.1917145848274231, + 0.5123120546340942, + 0.4046861231327057, + 0.18827927112579346, + 0.5574362277984619, + 0.09375456720590591, + -0.2718583941459656, + -0.5104306936264038, + -0.5676153302192688, + 0.3156948983669281, + 0.322867214679718, + -0.5215802788734436, + 0.4345199167728424, + 0.06456367671489716, + 0.6212273836135864, + 0.19039873778820038, + -0.8067936301231384, + -0.3089989125728607, + -0.0720198005437851, + 0.1396462768316269, + 0.23179945349693298, + -0.4338334798812866, + 0.6532859802246094, + -0.39463475346565247, + 0.4321286976337433, + 0.7570123076438904, + 0.3519911468029022, + -0.4297088086605072, + 0.6994672417640686, + 0.39958176016807556, + 0.21339523792266846, + 0.2590979039669037, + 0.020650368183851242, + 0.3302801549434662, + -0.0758761391043663, + 0.7354885339736938, + 0.07079026848077774, + 0.4478318393230438, + 0.5474826693534851, + -0.5955069661140442, + 0.06928855180740356, + 0.09410976618528366, + 0.4581683576107025, + -0.20661939680576324, + 0.43922898173332214, + -0.21895869076251984, + 0.6155246496200562, + -0.3576083779335022, + -0.17602530121803284, + -0.7841647267341614, + 0.3739136755466461, + 0.14571139216423035, + 0.06457892805337906, + -0.567622721195221, + -0.17433546483516693, + -0.12049046158790588, + -0.18151460587978363, + 0.569500744342804, + -0.6042268872261047, + 0.15831904113292694, + 0.14160946011543274, + -0.5267399549484253, + 0.49778082966804504, + 0.20231327414512634, + 0.06598637253046036, + -0.3405483365058899, + -0.12429828196763992, + -0.2662769854068756, + -0.27096059918403625, + -0.6506903767585754, + -0.2900505065917969, + 0.0674976333975792, + 0.698974072933197 + ], + "mmd": [ + 0.010865961643706346, + 0.09526803016238794, + 0.03414542261651732, + 0.012655752878092397, + 0.020272161876809136, + 0.03169784512836238, + 0.012166964591528107, + 0.01378156822626575, + 0.013161506222683972, + 0.37595768685632247, + 0.00979984181546123, + 0.03530890636017725, + 0.019015213965092426, + 0.016309612110020577, + 0.040693729480614116, + 0.009759099563166451, + 0.14422062098187793, + 0.027405931798592853, + 0.04225285796992073, + 0.010066702584036746, + 0.08631084179813242, + 0.012579651170403272, + 0.012571158202939525, + 0.0544084974687441, + 0.012538237883627623, + 0.06043173430925475, + 0.08415542539479814, + 0.05715046483273867, + 0.04296383245996371, + 0.02260156939242275, + 0.0338301946091234, + 0.11078357005261896, + 0.0071098337255163235, + 0.02846107153033539, + 0.01655639894340888, + 0.2022278918590673, + 0.019905650584001622, + 0.013048109468331903, + 0.04738914453273513, + 0.04963575853962676, + 0.009787493390163271, + 0.12135835753612634, + 0.10259814448897608, + 0.044599706333020905, + 0.05809130101644877, + 0.037120262448286256, + 0.020010755580221473, + 0.08324532378402383, + 0.038567341303705316, + 0.02504657601456528, + 0.028043479383941916, + 0.023580868962572943, + 0.012261321474307363, + 0.027796521068628288, + 0.07056779294627435, + 0.07657451455867637, + 0.025348443521783004, + 0.13396834963126292, + 0.018971953290527366, + 0.02903435660072362, + 0.02464025590783192, + 0.008471001609619888, + 0.014732542644024682, + 0.02646491019592334, + 0.0070139229480018495, + 0.02027082889037135, + 0.19370670986072713, + 0.013233911056386383, + 0.029681144910746693, + 0.025743508355221656, + 0.1718426196244932, + 0.02127572315814641, + 0.20704057653164076, + 0.03706375762058778, + 0.010190299027388883, + 0.04344536307718494, + 0.050616749429386876, + 0.09801713847439131, + 0.09072148283962445, + 0.07072214645403596 + ], + "wasserstein": [ + 0.15711171303839613, + 0.46294023210619595, + 0.3097925711144535, + 0.17848178976516998, + 0.2558425643177493, + 0.2559152283247149, + 0.1523232535434542, + 0.17515080517140816, + 0.18794170353453712, + 1.0767282260840905, + 0.14973940369189223, + 0.279458939479929, + 0.19361002135671626, + 0.18688165814365643, + 0.3416138962697992, + 0.15231269407574455, + 0.6458083908505542, + 0.29904459258819494, + 0.3109280343243505, + 0.16514292369639894, + 0.46247828408234914, + 0.17453192282617175, + 0.1799635980070445, + 0.3773264153691337, + 0.1796078135437695, + 0.36506880286672433, + 0.402834494647329, + 0.32065433000087823, + 0.27373344862515764, + 0.2363612896551318, + 0.23622519191526703, + 0.5010627928032602, + 0.1411489296246058, + 0.28799037297564195, + 0.19722798016939144, + 0.8823939408087295, + 0.19776941429478243, + 0.18002998043785606, + 0.3522717425518259, + 0.36787555259517163, + 0.16654906649832257, + 0.590740631660127, + 0.45340379509496137, + 0.37648602514223944, + 0.36508584566005786, + 0.2806632280416268, + 0.22090181440084117, + 0.412335814245266, + 0.2943957928612708, + 0.23597065972378597, + 0.2635683387412148, + 0.27071601108734555, + 0.17228513715224256, + 0.22547700541340243, + 0.3958086142289623, + 0.44895616289241747, + 0.2428726501733131, + 0.5025236601456261, + 0.19312637648130992, + 0.3033928678882094, + 0.242617471118807, + 0.16100098873112714, + 0.1985966803497802, + 0.24211151260618344, + 0.1368346764976086, + 0.2179747543314429, + 0.5514291366005489, + 0.19428340909703454, + 0.332240721408113, + 0.22806193624733515, + 0.7469295981944093, + 0.24160652140088504, + 0.7204070569427354, + 0.30062080042906286, + 0.14440878485734845, + 0.32440906950855847, + 0.3228828228753159, + 0.43125815121369304, + 0.45646147939874554, + 0.3838212152690147 + ] + }, + "n_perts": 80 + }, + "AvgPerturbationEffect": { + "mse": 0.0034106687711755513, + "r2": -0.5598821602761745, + "pearson": 0.24505942618707194, + "spearman": 0.20495787831150186, + "de_corr": 0.31165603089029903, + "mmd": 0.049649688791485846, + "wasserstein": 0.31183738580365433, + "_per_pert": { + "mse": [ + 0.0005894884234294295, + 0.005133030004799366, + 0.002097384538501501, + 0.0009051811648532748, + 0.0010524397948756814, + 0.0019171570893377066, + 0.0005996071849949658, + 0.0006937250727787614, + 0.0006017289124429226, + 0.038852035999298096, + 0.0007957180496305227, + 0.0013922436628490686, + 0.0012487592175602913, + 0.0008711018599569798, + 0.0022619040682911873, + 0.0008431424503214657, + 0.013289649039506912, + 0.0010381487663835287, + 0.0012427716283127666, + 0.0009599572513252497, + 0.004432815592736006, + 0.0008816534536890686, + 0.000990309752523899, + 0.003627948695793748, + 0.0004956180928274989, + 0.0025439485907554626, + 0.0042213439010083675, + 0.00230471626855433, + 0.001441224361769855, + 0.0011011951137334108, + 0.0011994002852588892, + 0.007305716630071402, + 0.0008306491654366255, + 0.0015610003611072898, + 0.0008835542248561978, + 0.024105127900838852, + 0.0010445668594911695, + 0.0009483625763095915, + 0.0023823522496968508, + 0.0021485495381057262, + 0.0008916195365600288, + 0.013441819697618484, + 0.006088176742196083, + 0.0020531434565782547, + 0.0034099421463906765, + 0.0018019744893535972, + 0.001463604625314474, + 0.0034932252019643784, + 0.0012204849626868963, + 0.0010706600733101368, + 0.001149422605521977, + 0.0013255140511319041, + 0.0006668244604952633, + 0.0009478386491537094, + 0.003121105255559087, + 0.003949003294110298, + 0.0014148567570373416, + 0.009705149568617344, + 0.001050600316375494, + 0.0009364152210764587, + 0.0014723913045600057, + 0.0007964519318193197, + 0.0003402339934837073, + 0.0012345650466158986, + 0.00044358207378536463, + 0.0008176034898497164, + 0.011230789124965668, + 0.0009742280235514045, + 0.0023226593621075153, + 0.0009950286475941539, + 0.01436714455485344, + 0.0012383308494463563, + 0.014483560808002949, + 0.0018183455104008317, + 0.0010009249672293663, + 0.001923806150443852, + 0.002366699744015932, + 0.004320385865867138, + 0.006899230182170868, + 0.0037709311582148075 + ], + "r2": [ + -1.8636033535003662, + 0.28329581022262573, + -0.4133310317993164, + -1.620973825454712, + -0.22052252292633057, + -0.5105007886886597, + -1.1692540645599365, + -0.07158458232879639, + -0.47983646392822266, + 0.106869637966156, + -2.0393857955932617, + 0.3293234705924988, + -0.9829897880554199, + -1.1115834712982178, + -0.6021864414215088, + -2.18030047416687, + 0.11901187896728516, + 0.26569104194641113, + 0.37247514724731445, + -1.7798693180084229, + 0.17754977941513062, + -1.4925205707550049, + -2.6043107509613037, + -0.22924542427062988, + -0.25281357765197754, + 0.2787362337112427, + 0.21872520446777344, + 0.29712939262390137, + 0.2129862904548645, + -0.6597713232040405, + 0.021631956100463867, + 0.14452707767486572, + -5.210337162017822, + -0.6186857223510742, + -0.6787067651748657, + 0.109305739402771, + -0.9517689943313599, + -1.9650239944458008, + -0.02874457836151123, + 0.2159462571144104, + -2.440751075744629, + -0.12383127212524414, + 0.2226635217666626, + 0.10601520538330078, + 0.10054141283035278, + -0.09740173816680908, + -1.993241548538208, + 0.2630380392074585, + 0.3697144389152527, + 0.18535268306732178, + 0.09964555501937866, + 0.2984629273414612, + -0.7245548963546753, + 0.3635002374649048, + 0.036262333393096924, + -0.08243894577026367, + -0.6906191110610962, + 0.1364656686782837, + -0.9002121686935425, + 0.26755696535110474, + -0.56438148021698, + -5.123881816864014, + 0.2780846953392029, + -0.21968770027160645, + -1.0687928199768066, + -0.06127774715423584, + 0.11497700214385986, + -2.7894928455352783, + 0.038840293884277344, + -0.09386086463928223, + 0.07122939825057983, + -1.2731711864471436, + 0.16526806354522705, + 0.05077087879180908, + -3.005403995513916, + -0.6749517917633057, + -0.1275639533996582, + 0.27777618169784546, + 0.19641715288162231, + 0.20700734853744507 + ], + "pearson": [ + 0.05896278843283653, + 0.7570440173149109, + 0.06085492670536041, + -0.0444427952170372, + 0.20180360972881317, + -0.1554526537656784, + 0.10319400578737259, + 0.4175525903701782, + 0.30598771572113037, + 0.576377272605896, + 0.2685815095901489, + 0.5839219093322754, + -0.11628152430057526, + 0.14456048607826233, + -0.08896316587924957, + -0.1473231464624405, + 0.46514055132865906, + 0.5309261083602905, + 0.6671335101127625, + -0.19793277978897095, + 0.5580198168754578, + 0.03759147226810455, + -0.2749055325984955, + 0.05794224515557289, + 0.4054983854293823, + 0.5591835975646973, + 0.5282705426216125, + 0.593189001083374, + 0.4981309771537781, + 0.13972818851470947, + 0.3488367199897766, + 0.4086320698261261, + -0.5068380832672119, + 0.012276927009224892, + 0.09146756678819656, + 0.6736096739768982, + -0.07635803520679474, + -0.006184936501085758, + 0.24896512925624847, + 0.4740391671657562, + -0.275273859500885, + 0.35014304518699646, + 0.5953029990196228, + 0.35119494795799255, + 0.33516788482666016, + 0.1833677440881729, + -0.20259007811546326, + 0.6047167181968689, + 0.6642725467681885, + 0.5131582021713257, + 0.38888785243034363, + 0.6151023507118225, + 0.20143264532089233, + 0.6383336782455444, + 0.3160689175128937, + 0.06969551742076874, + -0.14262492954730988, + 0.4861581325531006, + -0.03801296278834343, + 0.641390860080719, + -0.14751948416233063, + -0.34102919697761536, + 0.598152756690979, + 0.29214373230934143, + 0.2671343684196472, + 0.41020670533180237, + 0.501792848110199, + -0.38497403264045715, + 0.3590739667415619, + 0.26316532492637634, + 0.30733630061149597, + -0.11251591891050339, + 0.5925635695457458, + 0.3308599889278412, + -0.3465830981731415, + -0.24093352258205414, + 0.08624634891748428, + 0.6415207982063293, + 0.5249453783035278, + 0.544535219669342 + ], + "spearman": [ + -0.01638680209670052, + 0.6991641532910383, + 0.1459635784908946, + -0.0485118016279504, + 0.08736281484070371, + -0.09393846698461675, + -0.0015370143842535961, + 0.3141665315416329, + 0.2528314757078689, + 0.45694526423631604, + 0.2189729827432457, + 0.5146091816522954, + -0.10450368962592241, + 0.14341695885423972, + 0.018052651513162876, + -0.1742145660536415, + 0.4155495198873799, + 0.36810455752613935, + 0.6055375918843979, + -0.1919158904789726, + 0.525489116372279, + -0.07361702590425648, + -0.2693049448262362, + 0.052945267236316804, + 0.31205626801406705, + 0.4720206085051521, + 0.47176803994201, + 0.491057880698617, + 0.40113138428284606, + 0.12810103102525774, + 0.38358404689601167, + 0.29447608661902164, + -0.46608878352219585, + 0.00616226404056601, + 0.23716670629167655, + 0.6393868368364793, + -0.039781007445251856, + 0.01971894942973736, + 0.2125560981390245, + 0.37921048080262015, + -0.1445625931406483, + 0.27824656106164025, + 0.5181496385374096, + 0.2449543422385856, + 0.2700904620226155, + 0.12699000924750228, + -0.2723268620817155, + 0.5427764466941116, + 0.5966722191680548, + 0.47040537527774595, + 0.3152481473120368, + 0.4700351705087926, + 0.18122576730644183, + 0.4869994307498577, + 0.30007192501798124, + 0.09613929753482438, + -0.031189023297255824, + 0.3521854300463575, + -0.1071594987898747, + 0.5581242360310589, + -0.136246096061524, + -0.31312273528068385, + 0.4983976155994038, + 0.19216711804177952, + 0.21977423944355987, + 0.3615016233754058, + 0.4596529179132295, + -0.3130108367527092, + 0.2142681295670324, + 0.16687258171814542, + 0.29828115707028924, + -0.07609282833027672, + 0.5848952026931363, + 0.26780474795118697, + -0.3191299622824906, + -0.17027802956950736, + 0.03906446026611506, + 0.5521481865370467, + 0.38927847431961854, + 0.43961948290487074 + ], + "de_corr": [ + 0.6850724220275879, + 0.8879386186599731, + 0.1328609138727188, + -0.10727494210004807, + 0.45528504252433777, + -0.3523196876049042, + -0.27665653824806213, + 0.5069816708564758, + 0.5711830854415894, + 0.8087224960327148, + 0.22876529395580292, + 0.7591655254364014, + -0.04666735231876373, + -0.11922423541545868, + 0.03902706503868103, + -0.400743693113327, + 0.6545570492744446, + 0.5364365577697754, + 0.5674976110458374, + 0.004549222532659769, + 0.8275271058082581, + 0.39404574036598206, + -0.04165051877498627, + -0.5498351454734802, + 0.655297040939331, + 0.7302966713905334, + 0.11759544909000397, + 0.38393616676330566, + 0.5820118188858032, + 0.21148277819156647, + 0.04829304292798042, + 0.5449263453483582, + 0.16939057409763336, + -0.17490263283252716, + -0.3492230176925659, + 0.789750874042511, + 0.011864609085023403, + 0.4153749346733093, + 0.30131974816322327, + 0.7311798930168152, + -0.47079288959503174, + 0.8483207821846008, + 0.8571071624755859, + 0.5815296173095703, + 0.4406147301197052, + 0.5927321314811707, + -0.3578053116798401, + 0.6760474443435669, + 0.5958951711654663, + 0.719107449054718, + 0.6221590042114258, + 0.8742597103118896, + 0.034995585680007935, + 0.8716983199119568, + 0.6109908223152161, + 0.16803662478923798, + -0.5775253772735596, + 0.7642053365707397, + 0.21314768493175507, + 0.670685887336731, + -0.25614023208618164, + -0.422098308801651, + 0.7319802641868591, + 0.19940131902694702, + 0.3634165823459625, + 0.18028034269809723, + 0.8474987149238586, + -0.3782942593097687, + 0.6447136998176575, + 0.5108279585838318, + 0.26227447390556335, + -0.25329533219337463, + 0.645876407623291, + 0.38376161456108093, + -0.7495485544204712, + -0.31595396995544434, + -0.07863935828208923, + 0.8323890566825867, + 0.7960889935493469, + 0.9186955690383911 + ], + "mmd": [ + 0.01570141535697922, + 0.07236881385040284, + 0.037287025601551727, + 0.01805368439601196, + 0.0268440716667373, + 0.04196085758615642, + 0.016207096554945455, + 0.013603214818082465, + 0.015615680272564436, + 0.3526261618546107, + 0.016375076103583353, + 0.02958753273187409, + 0.023261326360364487, + 0.02008443138579241, + 0.042676154771774, + 0.017421556486625134, + 0.13379725174921775, + 0.02621740216769297, + 0.033256648635204056, + 0.018537467890297976, + 0.06963659580119275, + 0.020178456353972307, + 0.020673964970429393, + 0.05276733228447872, + 0.015654474466824997, + 0.04843898016557413, + 0.07113911868166678, + 0.04682247522234795, + 0.036954148077606885, + 0.032001918370707205, + 0.03224842369704184, + 0.10374977711656097, + 0.015001290416029267, + 0.031453395199351974, + 0.021869091865155066, + 0.18566640880250618, + 0.028644790001729037, + 0.01883323500282663, + 0.049566648196372465, + 0.042203605920825504, + 0.01821352169845636, + 0.11465443402645903, + 0.08683565374987745, + 0.041138241579298795, + 0.055039365110732597, + 0.036846260676270504, + 0.025576655517632885, + 0.06722397326047114, + 0.029473445840719537, + 0.023928544499040805, + 0.0257686491203879, + 0.023757746700988513, + 0.016767739883091215, + 0.022989242716960234, + 0.0646401997530438, + 0.07486265516742052, + 0.03112522612336066, + 0.12182827212009395, + 0.023530761047959148, + 0.027681769218509067, + 0.030600634868444665, + 0.018047285048507722, + 0.019263644187130113, + 0.02847296895279461, + 0.013243274211557354, + 0.02232678961989798, + 0.16901661003307655, + 0.020222435732057398, + 0.03183148839294914, + 0.026693513670877467, + 0.16023573560636417, + 0.029895594608512877, + 0.18175112566433405, + 0.03426230391050167, + 0.020031202808421855, + 0.04975504528544761, + 0.05102439434377237, + 0.07558778371237596, + 0.08054294275581675, + 0.06230097124158451 + ], + "wasserstein": [ + 0.20111146712226444, + 0.38092888295247795, + 0.33122355096848066, + 0.21403848827216748, + 0.25374966467696297, + 0.28586573059583237, + 0.19180188058905212, + 0.16353703492028926, + 0.2073218926507884, + 0.9982994008567991, + 0.19254250378916038, + 0.24084339143861644, + 0.21488397883059396, + 0.18665246717933998, + 0.3425077429717615, + 0.19941959929077985, + 0.5766726168598839, + 0.2845452850917858, + 0.2564523260410695, + 0.2197301658325513, + 0.4082561855431376, + 0.19872550000954575, + 0.2193092444751932, + 0.36715441431884516, + 0.18903115836554335, + 0.32021330332983455, + 0.371765380702717, + 0.3161846387506113, + 0.2526341380517888, + 0.2606270717209236, + 0.2254998060630447, + 0.469299890231332, + 0.1920270258135351, + 0.31663756363906054, + 0.20840670257685714, + 0.8163757074917574, + 0.25131046836409593, + 0.2069735079389802, + 0.35184276078341786, + 0.33890180205250436, + 0.20761094525615423, + 0.550458133129081, + 0.42413707398106626, + 0.36567601166984426, + 0.33425921961829835, + 0.2711914023016828, + 0.26399402229429586, + 0.36587054980039546, + 0.2586192540738769, + 0.2216899219193495, + 0.24436702220320772, + 0.24110536462445042, + 0.20120086778480242, + 0.2273513594661153, + 0.39648333752323744, + 0.462822917519662, + 0.26236716171914654, + 0.4677155564519627, + 0.2285666452298058, + 0.2613657348095476, + 0.2448754617774722, + 0.2157963275293286, + 0.1861228340973911, + 0.22609126669536883, + 0.1543711049919832, + 0.2367365821391758, + 0.5052214355252471, + 0.229492066092707, + 0.31008677428741954, + 0.22366068102455, + 0.7102428151070243, + 0.2702743251721207, + 0.6575921629804722, + 0.27845383405465574, + 0.19817825591374877, + 0.34706644128131403, + 0.3284483302837195, + 0.3758530865347564, + 0.4018658281232308, + 0.36640440815129544 + ] + }, + "n_perts": 80 + }, + "Additive": { + "mse": 0.0034106687711755513, + "r2": -0.5598821602761745, + "pearson": 0.24505942618707194, + "spearman": 0.20495787831150186, + "de_corr": 0.31165603089029903, + "mmd": 0.049649688791485846, + "wasserstein": 0.31183738580365433, + "_per_pert": { + "mse": [ + 0.0005894884234294295, + 0.005133030004799366, + 0.002097384538501501, + 0.0009051811648532748, + 0.0010524397948756814, + 0.0019171570893377066, + 0.0005996071849949658, + 0.0006937250727787614, + 0.0006017289124429226, + 0.038852035999298096, + 0.0007957180496305227, + 0.0013922436628490686, + 0.0012487592175602913, + 0.0008711018599569798, + 0.0022619040682911873, + 0.0008431424503214657, + 0.013289649039506912, + 0.0010381487663835287, + 0.0012427716283127666, + 0.0009599572513252497, + 0.004432815592736006, + 0.0008816534536890686, + 0.000990309752523899, + 0.003627948695793748, + 0.0004956180928274989, + 0.0025439485907554626, + 0.0042213439010083675, + 0.00230471626855433, + 0.001441224361769855, + 0.0011011951137334108, + 0.0011994002852588892, + 0.007305716630071402, + 0.0008306491654366255, + 0.0015610003611072898, + 0.0008835542248561978, + 0.024105127900838852, + 0.0010445668594911695, + 0.0009483625763095915, + 0.0023823522496968508, + 0.0021485495381057262, + 0.0008916195365600288, + 0.013441819697618484, + 0.006088176742196083, + 0.0020531434565782547, + 0.0034099421463906765, + 0.0018019744893535972, + 0.001463604625314474, + 0.0034932252019643784, + 0.0012204849626868963, + 0.0010706600733101368, + 0.001149422605521977, + 0.0013255140511319041, + 0.0006668244604952633, + 0.0009478386491537094, + 0.003121105255559087, + 0.003949003294110298, + 0.0014148567570373416, + 0.009705149568617344, + 0.001050600316375494, + 0.0009364152210764587, + 0.0014723913045600057, + 0.0007964519318193197, + 0.0003402339934837073, + 0.0012345650466158986, + 0.00044358207378536463, + 0.0008176034898497164, + 0.011230789124965668, + 0.0009742280235514045, + 0.0023226593621075153, + 0.0009950286475941539, + 0.01436714455485344, + 0.0012383308494463563, + 0.014483560808002949, + 0.0018183455104008317, + 0.0010009249672293663, + 0.001923806150443852, + 0.002366699744015932, + 0.004320385865867138, + 0.006899230182170868, + 0.0037709311582148075 + ], + "r2": [ + -1.8636033535003662, + 0.28329581022262573, + -0.4133310317993164, + -1.620973825454712, + -0.22052252292633057, + -0.5105007886886597, + -1.1692540645599365, + -0.07158458232879639, + -0.47983646392822266, + 0.106869637966156, + -2.0393857955932617, + 0.3293234705924988, + -0.9829897880554199, + -1.1115834712982178, + -0.6021864414215088, + -2.18030047416687, + 0.11901187896728516, + 0.26569104194641113, + 0.37247514724731445, + -1.7798693180084229, + 0.17754977941513062, + -1.4925205707550049, + -2.6043107509613037, + -0.22924542427062988, + -0.25281357765197754, + 0.2787362337112427, + 0.21872520446777344, + 0.29712939262390137, + 0.2129862904548645, + -0.6597713232040405, + 0.021631956100463867, + 0.14452707767486572, + -5.210337162017822, + -0.6186857223510742, + -0.6787067651748657, + 0.109305739402771, + -0.9517689943313599, + -1.9650239944458008, + -0.02874457836151123, + 0.2159462571144104, + -2.440751075744629, + -0.12383127212524414, + 0.2226635217666626, + 0.10601520538330078, + 0.10054141283035278, + -0.09740173816680908, + -1.993241548538208, + 0.2630380392074585, + 0.3697144389152527, + 0.18535268306732178, + 0.09964555501937866, + 0.2984629273414612, + -0.7245548963546753, + 0.3635002374649048, + 0.036262333393096924, + -0.08243894577026367, + -0.6906191110610962, + 0.1364656686782837, + -0.9002121686935425, + 0.26755696535110474, + -0.56438148021698, + -5.123881816864014, + 0.2780846953392029, + -0.21968770027160645, + -1.0687928199768066, + -0.06127774715423584, + 0.11497700214385986, + -2.7894928455352783, + 0.038840293884277344, + -0.09386086463928223, + 0.07122939825057983, + -1.2731711864471436, + 0.16526806354522705, + 0.05077087879180908, + -3.005403995513916, + -0.6749517917633057, + -0.1275639533996582, + 0.27777618169784546, + 0.19641715288162231, + 0.20700734853744507 + ], + "pearson": [ + 0.05896278843283653, + 0.7570440173149109, + 0.06085492670536041, + -0.0444427952170372, + 0.20180360972881317, + -0.1554526537656784, + 0.10319400578737259, + 0.4175525903701782, + 0.30598771572113037, + 0.576377272605896, + 0.2685815095901489, + 0.5839219093322754, + -0.11628152430057526, + 0.14456048607826233, + -0.08896316587924957, + -0.1473231464624405, + 0.46514055132865906, + 0.5309261083602905, + 0.6671335101127625, + -0.19793277978897095, + 0.5580198168754578, + 0.03759147226810455, + -0.2749055325984955, + 0.05794224515557289, + 0.4054983854293823, + 0.5591835975646973, + 0.5282705426216125, + 0.593189001083374, + 0.4981309771537781, + 0.13972818851470947, + 0.3488367199897766, + 0.4086320698261261, + -0.5068380832672119, + 0.012276927009224892, + 0.09146756678819656, + 0.6736096739768982, + -0.07635803520679474, + -0.006184936501085758, + 0.24896512925624847, + 0.4740391671657562, + -0.275273859500885, + 0.35014304518699646, + 0.5953029990196228, + 0.35119494795799255, + 0.33516788482666016, + 0.1833677440881729, + -0.20259007811546326, + 0.6047167181968689, + 0.6642725467681885, + 0.5131582021713257, + 0.38888785243034363, + 0.6151023507118225, + 0.20143264532089233, + 0.6383336782455444, + 0.3160689175128937, + 0.06969551742076874, + -0.14262492954730988, + 0.4861581325531006, + -0.03801296278834343, + 0.641390860080719, + -0.14751948416233063, + -0.34102919697761536, + 0.598152756690979, + 0.29214373230934143, + 0.2671343684196472, + 0.41020670533180237, + 0.501792848110199, + -0.38497403264045715, + 0.3590739667415619, + 0.26316532492637634, + 0.30733630061149597, + -0.11251591891050339, + 0.5925635695457458, + 0.3308599889278412, + -0.3465830981731415, + -0.24093352258205414, + 0.08624634891748428, + 0.6415207982063293, + 0.5249453783035278, + 0.544535219669342 + ], + "spearman": [ + -0.01638680209670052, + 0.6991641532910383, + 0.1459635784908946, + -0.0485118016279504, + 0.08736281484070371, + -0.09393846698461675, + -0.0015370143842535961, + 0.3141665315416329, + 0.2528314757078689, + 0.45694526423631604, + 0.2189729827432457, + 0.5146091816522954, + -0.10450368962592241, + 0.14341695885423972, + 0.018052651513162876, + -0.1742145660536415, + 0.4155495198873799, + 0.36810455752613935, + 0.6055375918843979, + -0.1919158904789726, + 0.525489116372279, + -0.07361702590425648, + -0.2693049448262362, + 0.052945267236316804, + 0.31205626801406705, + 0.4720206085051521, + 0.47176803994201, + 0.491057880698617, + 0.40113138428284606, + 0.12810103102525774, + 0.38358404689601167, + 0.29447608661902164, + -0.46608878352219585, + 0.00616226404056601, + 0.23716670629167655, + 0.6393868368364793, + -0.039781007445251856, + 0.01971894942973736, + 0.2125560981390245, + 0.37921048080262015, + -0.1445625931406483, + 0.27824656106164025, + 0.5181496385374096, + 0.2449543422385856, + 0.2700904620226155, + 0.12699000924750228, + -0.2723268620817155, + 0.5427764466941116, + 0.5966722191680548, + 0.47040537527774595, + 0.3152481473120368, + 0.4700351705087926, + 0.18122576730644183, + 0.4869994307498577, + 0.30007192501798124, + 0.09613929753482438, + -0.031189023297255824, + 0.3521854300463575, + -0.1071594987898747, + 0.5581242360310589, + -0.136246096061524, + -0.31312273528068385, + 0.4983976155994038, + 0.19216711804177952, + 0.21977423944355987, + 0.3615016233754058, + 0.4596529179132295, + -0.3130108367527092, + 0.2142681295670324, + 0.16687258171814542, + 0.29828115707028924, + -0.07609282833027672, + 0.5848952026931363, + 0.26780474795118697, + -0.3191299622824906, + -0.17027802956950736, + 0.03906446026611506, + 0.5521481865370467, + 0.38927847431961854, + 0.43961948290487074 + ], + "de_corr": [ + 0.6850724220275879, + 0.8879386186599731, + 0.1328609138727188, + -0.10727494210004807, + 0.45528504252433777, + -0.3523196876049042, + -0.27665653824806213, + 0.5069816708564758, + 0.5711830854415894, + 0.8087224960327148, + 0.22876529395580292, + 0.7591655254364014, + -0.04666735231876373, + -0.11922423541545868, + 0.03902706503868103, + -0.400743693113327, + 0.6545570492744446, + 0.5364365577697754, + 0.5674976110458374, + 0.004549222532659769, + 0.8275271058082581, + 0.39404574036598206, + -0.04165051877498627, + -0.5498351454734802, + 0.655297040939331, + 0.7302966713905334, + 0.11759544909000397, + 0.38393616676330566, + 0.5820118188858032, + 0.21148277819156647, + 0.04829304292798042, + 0.5449263453483582, + 0.16939057409763336, + -0.17490263283252716, + -0.3492230176925659, + 0.789750874042511, + 0.011864609085023403, + 0.4153749346733093, + 0.30131974816322327, + 0.7311798930168152, + -0.47079288959503174, + 0.8483207821846008, + 0.8571071624755859, + 0.5815296173095703, + 0.4406147301197052, + 0.5927321314811707, + -0.3578053116798401, + 0.6760474443435669, + 0.5958951711654663, + 0.719107449054718, + 0.6221590042114258, + 0.8742597103118896, + 0.034995585680007935, + 0.8716983199119568, + 0.6109908223152161, + 0.16803662478923798, + -0.5775253772735596, + 0.7642053365707397, + 0.21314768493175507, + 0.670685887336731, + -0.25614023208618164, + -0.422098308801651, + 0.7319802641868591, + 0.19940131902694702, + 0.3634165823459625, + 0.18028034269809723, + 0.8474987149238586, + -0.3782942593097687, + 0.6447136998176575, + 0.5108279585838318, + 0.26227447390556335, + -0.25329533219337463, + 0.645876407623291, + 0.38376161456108093, + -0.7495485544204712, + -0.31595396995544434, + -0.07863935828208923, + 0.8323890566825867, + 0.7960889935493469, + 0.9186955690383911 + ], + "mmd": [ + 0.01570141535697922, + 0.07236881385040284, + 0.037287025601551727, + 0.01805368439601196, + 0.0268440716667373, + 0.04196085758615642, + 0.016207096554945455, + 0.013603214818082465, + 0.015615680272564436, + 0.3526261618546107, + 0.016375076103583353, + 0.02958753273187409, + 0.023261326360364487, + 0.02008443138579241, + 0.042676154771774, + 0.017421556486625134, + 0.13379725174921775, + 0.02621740216769297, + 0.033256648635204056, + 0.018537467890297976, + 0.06963659580119275, + 0.020178456353972307, + 0.020673964970429393, + 0.05276733228447872, + 0.015654474466824997, + 0.04843898016557413, + 0.07113911868166678, + 0.04682247522234795, + 0.036954148077606885, + 0.032001918370707205, + 0.03224842369704184, + 0.10374977711656097, + 0.015001290416029267, + 0.031453395199351974, + 0.021869091865155066, + 0.18566640880250618, + 0.028644790001729037, + 0.01883323500282663, + 0.049566648196372465, + 0.042203605920825504, + 0.01821352169845636, + 0.11465443402645903, + 0.08683565374987745, + 0.041138241579298795, + 0.055039365110732597, + 0.036846260676270504, + 0.025576655517632885, + 0.06722397326047114, + 0.029473445840719537, + 0.023928544499040805, + 0.0257686491203879, + 0.023757746700988513, + 0.016767739883091215, + 0.022989242716960234, + 0.0646401997530438, + 0.07486265516742052, + 0.03112522612336066, + 0.12182827212009395, + 0.023530761047959148, + 0.027681769218509067, + 0.030600634868444665, + 0.018047285048507722, + 0.019263644187130113, + 0.02847296895279461, + 0.013243274211557354, + 0.02232678961989798, + 0.16901661003307655, + 0.020222435732057398, + 0.03183148839294914, + 0.026693513670877467, + 0.16023573560636417, + 0.029895594608512877, + 0.18175112566433405, + 0.03426230391050167, + 0.020031202808421855, + 0.04975504528544761, + 0.05102439434377237, + 0.07558778371237596, + 0.08054294275581675, + 0.06230097124158451 + ], + "wasserstein": [ + 0.20111146712226444, + 0.38092888295247795, + 0.33122355096848066, + 0.21403848827216748, + 0.25374966467696297, + 0.28586573059583237, + 0.19180188058905212, + 0.16353703492028926, + 0.2073218926507884, + 0.9982994008567991, + 0.19254250378916038, + 0.24084339143861644, + 0.21488397883059396, + 0.18665246717933998, + 0.3425077429717615, + 0.19941959929077985, + 0.5766726168598839, + 0.2845452850917858, + 0.2564523260410695, + 0.2197301658325513, + 0.4082561855431376, + 0.19872550000954575, + 0.2193092444751932, + 0.36715441431884516, + 0.18903115836554335, + 0.32021330332983455, + 0.371765380702717, + 0.3161846387506113, + 0.2526341380517888, + 0.2606270717209236, + 0.2254998060630447, + 0.469299890231332, + 0.1920270258135351, + 0.31663756363906054, + 0.20840670257685714, + 0.8163757074917574, + 0.25131046836409593, + 0.2069735079389802, + 0.35184276078341786, + 0.33890180205250436, + 0.20761094525615423, + 0.550458133129081, + 0.42413707398106626, + 0.36567601166984426, + 0.33425921961829835, + 0.2711914023016828, + 0.26399402229429586, + 0.36587054980039546, + 0.2586192540738769, + 0.2216899219193495, + 0.24436702220320772, + 0.24110536462445042, + 0.20120086778480242, + 0.2273513594661153, + 0.39648333752323744, + 0.462822917519662, + 0.26236716171914654, + 0.4677155564519627, + 0.2285666452298058, + 0.2613657348095476, + 0.2448754617774722, + 0.2157963275293286, + 0.1861228340973911, + 0.22609126669536883, + 0.1543711049919832, + 0.2367365821391758, + 0.5052214355252471, + 0.229492066092707, + 0.31008677428741954, + 0.22366068102455, + 0.7102428151070243, + 0.2702743251721207, + 0.6575921629804722, + 0.27845383405465574, + 0.19817825591374877, + 0.34706644128131403, + 0.3284483302837195, + 0.3758530865347564, + 0.4018658281232308, + 0.36640440815129544 + ] + }, + "n_perts": 80 + }, + "LinearResponse": { + "mse": 0.0034106687755411256, + "r2": -0.5598821468651295, + "pearson": 0.24505942570394837, + "spearman": 0.2049578819994904, + "de_corr": 0.31165603364934213, + "mmd": 0.04964968872868484, + "wasserstein": 0.31183738544721346, + "_per_pert": { + "mse": [ + 0.0005894883652217686, + 0.005133030004799366, + 0.002097384538501501, + 0.0009051811066456139, + 0.0010524396784603596, + 0.0019171570893377066, + 0.0005996071267873049, + 0.0006937250727787614, + 0.0006017289124429226, + 0.038852035999298096, + 0.0007957181078381836, + 0.0013922437792643905, + 0.0012487592175602913, + 0.0008711018017493188, + 0.0022619040682911873, + 0.0008431424503214657, + 0.013289649039506912, + 0.0010381487663835287, + 0.0012427716283127666, + 0.0009599572513252497, + 0.004432815592736006, + 0.0008816534536890686, + 0.000990309868939221, + 0.003627948695793748, + 0.0004956181510351598, + 0.0025439488235861063, + 0.0042213439010083675, + 0.00230471626855433, + 0.001441224361769855, + 0.0011011951137334108, + 0.0011994001688435674, + 0.007305716630071402, + 0.0008306491654366255, + 0.0015610003611072898, + 0.0008835542830638587, + 0.024105127900838852, + 0.0010445668594911695, + 0.0009483625763095915, + 0.0023823522496968508, + 0.0021485495381057262, + 0.0008916195365600288, + 0.013441819697618484, + 0.006088176742196083, + 0.0020531434565782547, + 0.00340994237922132, + 0.001801974605768919, + 0.001463604625314474, + 0.0034932249691337347, + 0.0012204849626868963, + 0.0010706600733101368, + 0.001149422605521977, + 0.0013255139347165823, + 0.0006668244604952633, + 0.0009478385909460485, + 0.003121105022728443, + 0.003949003294110298, + 0.0014148567570373416, + 0.009705149568617344, + 0.001050600316375494, + 0.0009364152792841196, + 0.001472391071729362, + 0.0007964518154039979, + 0.0003402339934837073, + 0.0012345650466158986, + 0.00044358207378536463, + 0.0008176034316420555, + 0.011230789124965668, + 0.0009742280235514045, + 0.0023226593621075153, + 0.0009950286475941539, + 0.01436714455485344, + 0.0012383308494463563, + 0.014483560808002949, + 0.0018183455104008317, + 0.0010009248508140445, + 0.001923806150443852, + 0.002366699744015932, + 0.004320385865867138, + 0.0068992311134934425, + 0.0037709311582148075 + ], + "r2": [ + -1.863602876663208, + 0.28329581022262573, + -0.4133310317993164, + -1.620973825454712, + -0.22052252292633057, + -0.5105007886886597, + -1.1692540645599365, + -0.07158446311950684, + -0.4798363447189331, + 0.106869637966156, + -2.039386034011841, + 0.3293234705924988, + -0.9829896688461304, + -1.1115834712982178, + -0.6021864414215088, + -2.180300235748291, + 0.11901187896728516, + 0.2656911015510559, + 0.37247514724731445, + -1.7798690795898438, + 0.17754977941513062, + -1.4925205707550049, + -2.6043107509613037, + -0.22924530506134033, + -0.25281357765197754, + 0.2787361741065979, + 0.21872520446777344, + 0.29712939262390137, + 0.2129862904548645, + -0.6597713232040405, + 0.021631956100463867, + 0.14452707767486572, + -5.210337162017822, + -0.6186857223510742, + -0.6787067651748657, + 0.109305739402771, + -0.9517689943313599, + -1.9650239944458008, + -0.02874457836151123, + 0.2159462571144104, + -2.440751314163208, + -0.12383139133453369, + 0.2226635217666626, + 0.10601532459259033, + 0.10054135322570801, + -0.09740173816680908, + -1.993241548538208, + 0.2630380392074585, + 0.3697144389152527, + 0.18535268306732178, + 0.09964555501937866, + 0.2984629273414612, + -0.7245548963546753, + 0.3635002374649048, + 0.036262333393096924, + -0.08243894577026367, + -0.6906191110610962, + 0.1364656686782837, + -0.9002121686935425, + 0.26755690574645996, + -0.56438148021698, + -5.123881816864014, + 0.2780846953392029, + -0.21968770027160645, + -1.0687928199768066, + -0.06127774715423584, + 0.11497700214385986, + -2.789492607116699, + 0.038840293884277344, + -0.09386086463928223, + 0.07122939825057983, + -1.2731711864471436, + 0.16526806354522705, + 0.05077087879180908, + -3.005403995513916, + -0.6749517917633057, + -0.1275639533996582, + 0.27777618169784546, + 0.19641715288162231, + 0.20700734853744507 + ], + "pearson": [ + 0.05896279215812683, + 0.7570438981056213, + 0.060854919254779816, + -0.0444427989423275, + 0.20180360972881317, + -0.1554526388645172, + 0.10319399833679199, + 0.4175525903701782, + 0.30598771572113037, + 0.5763773322105408, + 0.26858147978782654, + 0.5839219093322754, + -0.11628151684999466, + 0.14456047117710114, + -0.08896317332983017, + -0.1473231464624405, + 0.46514055132865906, + 0.5309261083602905, + 0.6671335101127625, + -0.19793274998664856, + 0.5580198168754578, + 0.03759148716926575, + -0.27490556240081787, + 0.05794225260615349, + 0.4054984152317047, + 0.5591835975646973, + 0.5282705426216125, + 0.593189001083374, + 0.4981309473514557, + 0.13972818851470947, + 0.3488367795944214, + 0.4086320698261261, + -0.5068380832672119, + 0.012276935391128063, + 0.09146754443645477, + 0.6736096143722534, + -0.07635803520679474, + -0.006184935104101896, + 0.24896511435508728, + 0.4740391969680786, + -0.2752738296985626, + 0.35014304518699646, + 0.595302939414978, + 0.35119494795799255, + 0.3351678252220154, + 0.1833677440881729, + -0.20259007811546326, + 0.6047167181968689, + 0.6642725467681885, + 0.5131582021713257, + 0.3888879120349884, + 0.6151022911071777, + 0.20143263041973114, + 0.6383337378501892, + 0.31606897711753845, + 0.06969551742076874, + -0.14262491464614868, + 0.4861580729484558, + -0.038012921810150146, + 0.6413908004760742, + -0.14751943945884705, + -0.34102919697761536, + 0.5981526970863342, + 0.2921437919139862, + 0.26713433861732483, + 0.41020670533180237, + 0.501792848110199, + -0.38497400283813477, + 0.3590739667415619, + 0.26316529512405396, + 0.30733633041381836, + -0.11251591891050339, + 0.5925635695457458, + 0.3308600187301636, + -0.3465830683708191, + -0.24093353748321533, + 0.08624634891748428, + 0.6415208578109741, + 0.5249453186988831, + 0.544535219669342 + ], + "spearman": [ + -0.01638680209670052, + 0.6991641532910383, + 0.1459635784908946, + -0.0485118016279504, + 0.08736281484070371, + -0.09393846698461675, + -0.0015370143842535961, + 0.3141665315416329, + 0.2528314757078689, + 0.45694526423631604, + 0.2189729827432457, + 0.5146091816522954, + -0.10450368962592241, + 0.14341695885423972, + 0.018052651513162876, + -0.1742145660536415, + 0.4155495198873799, + 0.36810455752613935, + 0.6055375918843979, + -0.1919158904789726, + 0.525489116372279, + -0.07361702590425648, + -0.2693049448262362, + 0.052945267236316804, + 0.31205626801406705, + 0.4720206085051521, + 0.47176803994201, + 0.491057880698617, + 0.40113138428284606, + 0.12810103102525774, + 0.38358404689601167, + 0.29447608661902164, + -0.46608878352219585, + 0.00616226404056601, + 0.23716670629167655, + 0.6393870113467528, + -0.039781007445251856, + 0.01971894942973736, + 0.2125560981390245, + 0.37921048080262015, + -0.1445625931406483, + 0.27824656106164025, + 0.5181496385374096, + 0.2449543422385856, + 0.2700904620226155, + 0.12699000924750228, + -0.2723268620817155, + 0.5427764466941116, + 0.5966722191680548, + 0.470405228101307, + 0.3152481473120368, + 0.47003447000861753, + 0.18122576730644183, + 0.4869994307498577, + 0.30007192501798124, + 0.09613929753482438, + -0.031189023297255824, + 0.3521854300463575, + -0.1071594987898747, + 0.5581242360310589, + -0.136246096061524, + -0.31312273528068385, + 0.4983976155994038, + 0.19216711804177952, + 0.21977423944355987, + 0.3615016233754058, + 0.45965288041322006, + -0.3130108367527092, + 0.2142681295670324, + 0.16687258171814542, + 0.29828115707028924, + -0.07609224255159523, + 0.5848961782240445, + 0.26780474795118697, + -0.3191299622824906, + -0.17027802956950736, + 0.03906446026611506, + 0.5521481865370467, + 0.3892779187154589, + 0.43961948290487074 + ], + "de_corr": [ + 0.6850724816322327, + 0.8879385590553284, + 0.1328609138727188, + -0.10727494955062866, + 0.45528504252433777, + -0.35231971740722656, + -0.2766565680503845, + 0.5069816708564758, + 0.5711831450462341, + 0.8087224364280701, + 0.22876524925231934, + 0.7591655850410461, + -0.04666728898882866, + -0.11922430247068405, + 0.03902702406048775, + -0.40074366331100464, + 0.6545571684837341, + 0.5364365577697754, + 0.5674976110458374, + 0.0045492458157241344, + 0.8275271058082581, + 0.39404574036598206, + -0.041650574654340744, + -0.5498350858688354, + 0.6552971005439758, + 0.7302966713905334, + 0.11759539693593979, + 0.38393616676330566, + 0.5820119380950928, + 0.21148280799388885, + 0.04829303175210953, + 0.5449264049530029, + 0.1693905144929886, + -0.17490263283252716, + -0.3492230474948883, + 0.789750874042511, + 0.011864597909152508, + 0.41537487506866455, + 0.30131977796554565, + 0.7311798334121704, + -0.47079285979270935, + 0.848320722579956, + 0.8571071624755859, + 0.5815296769142151, + 0.4406146705150604, + 0.5927321314811707, + -0.3578052222728729, + 0.6760474443435669, + 0.5958951711654663, + 0.719107449054718, + 0.6221588850021362, + 0.8742597103118896, + 0.03499555587768555, + 0.8716983795166016, + 0.6109908819198608, + 0.16803669929504395, + -0.5775253772735596, + 0.7642053365707397, + 0.21314778923988342, + 0.670685887336731, + -0.2561401426792145, + -0.4220983386039734, + 0.7319803237915039, + 0.199401393532753, + 0.36341655254364014, + 0.1802803874015808, + 0.8474987149238586, + -0.37829431891441345, + 0.6447136998176575, + 0.5108279585838318, + 0.26227447390556335, + -0.2532954216003418, + 0.6458763480186462, + 0.38376158475875854, + -0.7495485544204712, + -0.31595394015312195, + -0.07863932102918625, + 0.8323891162872314, + 0.7960888743400574, + 0.9186955094337463 + ], + "mmd": [ + 0.015701415085111803, + 0.07236881425542652, + 0.03728702580665255, + 0.01805368442763322, + 0.026844071753004295, + 0.04196085733993882, + 0.016207096209267524, + 0.01360321481591753, + 0.015615680109558938, + 0.35262616220724097, + 0.01637507628330581, + 0.02958753273579462, + 0.02326132599426367, + 0.020084431574357797, + 0.042676155088270606, + 0.01742155633890463, + 0.1337972514727953, + 0.026217401855542888, + 0.033256648448745096, + 0.01853746766876141, + 0.0696365958898364, + 0.020178456062405314, + 0.02067396490915807, + 0.05276733232282771, + 0.015654474572235122, + 0.048438980415314914, + 0.07113911875887347, + 0.04682247512038762, + 0.03695414773336114, + 0.03200191804777652, + 0.032248423411975646, + 0.10374977711951672, + 0.015001290274507029, + 0.031453394743859775, + 0.021869091775785665, + 0.18566640887044095, + 0.028644789789983305, + 0.018833234833686596, + 0.04956664825995927, + 0.04220360614670671, + 0.01821352164957324, + 0.11465443482517734, + 0.08683565414088468, + 0.04113824145638989, + 0.055039365604822366, + 0.03684626054434259, + 0.025576654768226348, + 0.06722397278061332, + 0.029473445706092782, + 0.023928543953844916, + 0.025768649141482136, + 0.023757746784734635, + 0.016767739821357153, + 0.022989242297438928, + 0.06464019909216268, + 0.07486265526139813, + 0.031125225886093566, + 0.12182827198147472, + 0.023530760511205617, + 0.02768176939936351, + 0.030600634906595592, + 0.01804728488673102, + 0.01926364432122618, + 0.028472968784408748, + 0.01324327407273962, + 0.02232678921748832, + 0.1690166103206, + 0.020222435944549533, + 0.0318314884326778, + 0.026693513994190954, + 0.1602357354185736, + 0.029895594311953655, + 0.18175112589217635, + 0.03426230360401683, + 0.02003120261731628, + 0.04975504535414732, + 0.051024394185701816, + 0.07558778367536234, + 0.08054294313404009, + 0.062300971084520596 + ], + "wasserstein": [ + 0.20111146597723617, + 0.380928885158753, + 0.3312235519582824, + 0.21403848773657339, + 0.2537496641217966, + 0.2858657296129732, + 0.19180187925318917, + 0.16353703467818448, + 0.20732189182602542, + 0.9982994011933171, + 0.1925425030719112, + 0.24084339307747732, + 0.21488397696559963, + 0.18665246735283142, + 0.3425077460479741, + 0.1994195977537639, + 0.5766726171511596, + 0.28454528274616697, + 0.2564523281674767, + 0.21973016365804796, + 0.4082561861702181, + 0.19872549939366954, + 0.21930924357170187, + 0.3671544138037004, + 0.18903115800211368, + 0.3202133039443945, + 0.37176538250138846, + 0.31618463755050663, + 0.2526341372474146, + 0.2606270698990527, + 0.22549980673768388, + 0.46929989179079146, + 0.19202702404231897, + 0.3166375624023779, + 0.2084067027586887, + 0.8163757077038646, + 0.2513104665496838, + 0.20697350670670578, + 0.3518427609249717, + 0.33890180333289427, + 0.20761094407723849, + 0.5504581355822209, + 0.42413707606641887, + 0.36567601053154797, + 0.3342592224407616, + 0.2711914026813523, + 0.2639940183639752, + 0.3658705497520569, + 0.2586192553892088, + 0.2216899197361412, + 0.24436702204988847, + 0.24110536485475084, + 0.2012008667327346, + 0.22735135789327654, + 0.39648333597861424, + 0.4628229170869114, + 0.2623671596350758, + 0.46771555613279114, + 0.2285666420485974, + 0.261365734838736, + 0.24487546231889504, + 0.2157963255955879, + 0.18612283455472936, + 0.22609126496811752, + 0.15437110301326856, + 0.23673657975939938, + 0.5052214353182999, + 0.22949206467571812, + 0.3100867737324422, + 0.22366068228183247, + 0.7102428150661442, + 0.27027432312108257, + 0.6575921640176751, + 0.27845383242695926, + 0.1981782540153246, + 0.3470664417268754, + 0.328448330708526, + 0.37585308682049773, + 0.40186582966267503, + 0.3664044075778468 + ] + }, + "n_perts": 80 + }, + "NearestPerturbationCentroid": { + "mse": 0.005708367927582003, + "r2": -3.6805508196353913, + "pearson": 0.0717333696957212, + "spearman": 0.05168680142046058, + "de_corr": 0.1191471351776272, + "mmd": 0.08350106605838317, + "wasserstein": 0.3939168081784377, + "_per_pert": { + "mse": [ + 0.002775015775114298, + 0.00555382901802659, + 0.003079378744587302, + 0.0026705777272582054, + 0.0032496952917426825, + 0.00409917626529932, + 0.002837124513462186, + 0.0025020253378897905, + 0.002442104509100318, + 0.04397495090961456, + 0.003224154468625784, + 0.003992007579654455, + 0.003761792555451393, + 0.003363958792760968, + 0.003967595752328634, + 0.00294712302275002, + 0.013915268704295158, + 0.003656757762655616, + 0.0042054434306919575, + 0.0028930201660841703, + 0.006417109631001949, + 0.002691220259293914, + 0.003484762739390135, + 0.008360848762094975, + 0.0020651340018957853, + 0.005540607962757349, + 0.007243866100907326, + 0.005349485669285059, + 0.0033827689476311207, + 0.003691357094794512, + 0.003909493796527386, + 0.012610295787453651, + 0.002954313298687339, + 0.003853762988001108, + 0.003202597377821803, + 0.02955160103738308, + 0.0032517006620764732, + 0.0031355612445622683, + 0.004354109987616539, + 0.004124769475311041, + 0.0028027717489749193, + 0.017625197768211365, + 0.008772746659815311, + 0.005304872523993254, + 0.007221899926662445, + 0.004211016930639744, + 0.003953391686081886, + 0.006835431791841984, + 0.003305850550532341, + 0.0035600934643298388, + 0.0036720691714435816, + 0.003941299859434366, + 0.0026211251970380545, + 0.003364634234458208, + 0.005260619334876537, + 0.00694065960124135, + 0.003300108015537262, + 0.0120886554941535, + 0.002798232948407531, + 0.002974161645397544, + 0.003024839796125889, + 0.0030078364070504904, + 0.001983393682166934, + 0.0031820693984627724, + 0.0025544248055666685, + 0.00316640711389482, + 0.00822489894926548, + 0.003703599562868476, + 0.004827210679650307, + 0.003453066572546959, + 0.01690433919429779, + 0.0034147275146096945, + 0.01725544035434723, + 0.00430149445310235, + 0.0032077908981591463, + 0.00511400680989027, + 0.005330512300133705, + 0.002068045549094677, + 0.009683116339147091, + 0.0034210081212222576 + ], + "r2": [ + -12.48040771484375, + 0.22454148530960083, + -1.075052261352539, + -6.732722759246826, + -2.7686970233917236, + -2.2296833992004395, + -9.264128684997559, + -2.8648335933685303, + -5.005887031555176, + -0.010895967483520508, + -11.315226554870605, + -0.9230440855026245, + -4.973606586456299, + -7.154361724853516, + -1.8103878498077393, + -10.116433143615723, + 0.07753872871398926, + -1.586517572402954, + -1.1234960556030273, + -7.377682685852051, + -0.19060981273651123, + -6.608341217041016, + -11.683067321777344, + -1.8328773975372314, + -4.220204830169678, + -0.5708807706832886, + -0.34067487716674805, + -0.6314358711242676, + -0.8472388982772827, + -4.563781261444092, + -2.189030408859253, + -0.4766199588775635, + -21.087886810302734, + -2.99617600440979, + -5.084766864776611, + -0.09194362163543701, + -5.075789451599121, + -8.803227424621582, + -0.880186915397644, + -0.5052205324172974, + -9.81586742401123, + -0.47359132766723633, + -0.12010157108306885, + -1.3098607063293457, + -0.9049588441848755, + -1.5645074844360352, + -7.085145950317383, + -0.44206368923187256, + -0.7072147130966187, + -1.7088160514831543, + -1.8763689994812012, + -1.0859594345092773, + -5.778806686401367, + -1.2594447135925293, + -0.6243786811828613, + -0.9024648666381836, + -2.943314790725708, + -0.07561123371124268, + -4.0611419677734375, + -1.3263227939605713, + -2.21382212638855, + -22.127117156982422, + -3.208404541015625, + -2.143723487854004, + -10.913413047790527, + -3.110107421875, + 0.3518509864807129, + -13.406034469604492, + -0.9975898265838623, + -2.7960469722747803, + -0.09278881549835205, + -5.268325328826904, + 0.005516231060028076, + -1.2455050945281982, + -11.836626052856445, + -3.4524831771850586, + -1.539609670639038, + 0.6542920470237732, + -0.12783396244049072, + 0.28059297800064087 + ], + "pearson": [ + 0.040950190275907516, + 0.5520670413970947, + 0.2949352264404297, + 0.1270199418067932, + 0.08379948139190674, + -0.13717515766620636, + -0.016923369839787483, + 0.3134763240814209, + 0.30443260073661804, + 0.12832075357437134, + -0.12237337231636047, + 0.18744917213916779, + -0.18839463591575623, + -0.0367615707218647, + 0.09397189319133759, + -0.013420735485851765, + 0.3006315231323242, + 0.07328757643699646, + 0.06809114664793015, + 0.06848412752151489, + 0.25736337900161743, + 0.16438859701156616, + -0.23693862557411194, + -0.32693690061569214, + 0.47959446907043457, + 0.10788293182849884, + 0.08924441039562225, + 0.10204946994781494, + 0.2674649655818939, + -0.09228792041540146, + 0.020998533815145493, + -0.120652936398983, + -0.29250749945640564, + -0.1349891573190689, + -0.045502446591854095, + 0.07061892002820969, + -0.06828445196151733, + -0.15700778365135193, + 0.11756821721792221, + 0.2323426455259323, + -0.0036526317708194256, + -0.01631554588675499, + 0.24310193955898285, + -0.08201180398464203, + -0.10763748735189438, + 0.042719706892967224, + -0.33680036664009094, + 0.06998996436595917, + 0.2448081523180008, + 0.09559549391269684, + 0.0318387970328331, + 0.10063743591308594, + 0.13100135326385498, + 0.18650692701339722, + 0.114848792552948, + -0.11383139342069626, + -0.012855380773544312, + 0.18107284605503082, + 0.0988704040646553, + 0.2811323404312134, + 0.10601114481687546, + -0.23266027867794037, + 0.433714896440506, + 0.17066460847854614, + 0.004271475598216057, + 0.13479971885681152, + 0.6769914627075195, + -0.4312286078929901, + 0.07111253589391708, + -0.01736949197947979, + 0.08847634494304657, + -0.03195406496524811, + 0.20600773394107819, + 0.07671056687831879, + -0.24971400201320648, + -0.38858258724212646, + -0.14172986149787903, + 0.8294099569320679, + 0.15451207756996155, + 0.5739294290542603 + ], + "spearman": [ + 0.047905532976383246, + 0.47972227943056983, + 0.35438633409658354, + 0.1889650012412503, + 0.043029619757404935, + -0.0804527976131994, + 0.04774034393508598, + 0.2034289063572266, + 0.27764595791148944, + -0.010549571637392908, + -0.1290160312540078, + 0.15639750609937653, + -0.14476797769199443, + -0.029059948264987066, + 0.08485366821341704, + 0.029388074847018712, + 0.26492527673131916, + -0.027749310937327732, + 0.007139812784953196, + 0.11300169575042392, + 0.1125054596263649, + 0.16361829640457407, + -0.22842103510525877, + -0.3638161089540272, + 0.3928561597140399, + -0.015466329866582466, + 0.15702613825653455, + 0.10211029952757489, + 0.19519428829857205, + 0.009164514791128698, + 0.038389935597483896, + -0.1353252668313167, + -0.22751582471427462, + -0.08258468664617166, + -0.04027246456811614, + 0.013012435753108936, + 0.0605511556377889, + -0.16589839147459787, + 0.06330627532656884, + 0.20405216101304025, + 0.07319584379896095, + -0.08014684403671102, + 0.22104831576207895, + -0.1242788755697189, + -0.14836346659086666, + 0.006126250531562632, + -0.3201874790468697, + 0.04472410118102529, + 0.19221744155436038, + 0.05398577699644425, + -0.008218883054720764, + -0.02939903084975771, + 0.12679051369762842, + 0.1686881011720253, + 0.14928983382245845, + -0.1299841659960415, + -0.018041637010409253, + 0.0035642086423887387, + 0.0964947196236799, + 0.22950836437709107, + 0.10839174009793502, + -0.16706155026538755, + 0.4460997875249469, + 0.09231464307866076, + -0.0076094584023645995, + 0.0994081328520332, + 0.6303141990785497, + -0.39193549698387425, + -0.02946834886708722, + -0.045404180351045084, + 0.0468091932022983, + -0.040534856648914735, + 0.158305472076368, + 0.01176827244206811, + -0.17812788103197025, + -0.3290287107571777, + -0.11575518943879735, + 0.7570251617562904, + 0.008476074619018654, + 0.444522636130659 + ], + "de_corr": [ + 0.4627281725406647, + 0.5847346782684326, + 0.4412709176540375, + 0.14813345670700073, + 0.08825524896383286, + -0.27607592940330505, + -0.14479492604732513, + 0.546600878238678, + 0.5115342140197754, + 0.6641858220100403, + -0.28457897901535034, + 0.4495815336704254, + -0.15066464245319366, + -0.17006850242614746, + -0.009612279944121838, + -0.12765875458717346, + 0.5670153498649597, + -0.04873455688357353, + 0.35111674666404724, + 0.07225529849529266, + 0.4345524311065674, + 0.3171515166759491, + -0.11298967897891998, + -0.8148773908615112, + 0.5133845210075378, + 0.3043084740638733, + -0.2143367975950241, + 0.01165721658617258, + 0.6780430674552917, + -0.1776619404554367, + -0.028674716129899025, + -0.09889513999223709, + -0.8606702089309692, + -0.6738924980163574, + -0.3771640658378601, + -0.11621119827032089, + -0.13651451468467712, + 0.06158582121133804, + 0.24193349480628967, + 0.7036532759666443, + -0.030576923862099648, + 0.24055425822734833, + 0.32758644223213196, + -0.0483349934220314, + -0.473682165145874, + 0.3182647228240967, + -0.3409825563430786, + 0.06544657796621323, + 0.2355632334947586, + 0.12002364546060562, + 0.32202965021133423, + 0.6086364984512329, + -0.018901970237493515, + 0.5017403364181519, + 0.5716832876205444, + -0.1689077764749527, + 0.1588938683271408, + 0.5301911234855652, + 0.024050481617450714, + 0.45638251304626465, + 0.3119121491909027, + -0.40808412432670593, + 0.4413498342037201, + 0.18814459443092346, + -0.11920631676912308, + -0.01477733813226223, + 0.8833320140838623, + -0.4483063817024231, + 0.44352641701698303, + 0.134155735373497, + 0.1720414161682129, + 0.18887881934642792, + 0.16141648590564728, + 0.08799620717763901, + -0.5561348795890808, + -0.7289972305297852, + -0.17588718235492706, + 0.9617927074432373, + 0.4150337278842926, + 0.8643184900283813 + ], + "mmd": [ + 0.05219576492141409, + 0.07889851084548927, + 0.05685209307502592, + 0.05036079613572131, + 0.05997102256627451, + 0.07902495455395231, + 0.05458265336941581, + 0.04300078471639002, + 0.048605035475542624, + 0.3795075166950631, + 0.05518389761308473, + 0.0666978378376617, + 0.06464007904260882, + 0.06168396124410891, + 0.07132592542765359, + 0.05387260600115307, + 0.14788053918231459, + 0.06838268723801932, + 0.0645274178387586, + 0.0548453751922916, + 0.09536479922651997, + 0.05169740047775495, + 0.06259930051564844, + 0.11330530294775598, + 0.037979138126772116, + 0.08684897495611299, + 0.11664089357715102, + 0.09296696881055322, + 0.06285983776124904, + 0.07591106665607972, + 0.07579793120005907, + 0.15635759141339822, + 0.054063013151032235, + 0.07251494934264324, + 0.05741446120047078, + 0.21674177593242716, + 0.07054485096755869, + 0.056492795915961014, + 0.07564508818180726, + 0.07322941227160695, + 0.05251724254643897, + 0.14800783440133236, + 0.119213497544428, + 0.09064955011213416, + 0.1066033967744785, + 0.0768935170317141, + 0.0696895982553487, + 0.11168089791869862, + 0.05884208580445183, + 0.06263915235879647, + 0.06850840450229234, + 0.058358386441722865, + 0.04976127332911029, + 0.061079495394589345, + 0.0986732719448915, + 0.11929611959002906, + 0.0648890561589458, + 0.14779170848680268, + 0.05589973994608988, + 0.06063605130958272, + 0.060537703596493686, + 0.05413708898822944, + 0.045527329704063924, + 0.06165017388451266, + 0.051743498354969275, + 0.058536877245668584, + 0.13233941730115428, + 0.06840988936568815, + 0.0653710993642529, + 0.06770299440648597, + 0.19207823966971782, + 0.06740186722182007, + 0.20982488716019465, + 0.0759745723884967, + 0.056828885344218816, + 0.10458242182325805, + 0.0985330630375586, + 0.04579248046816797, + 0.1057996180830374, + 0.05866788580628057 + ], + "wasserstein": [ + 0.3039940988700868, + 0.38383625909752667, + 0.3619847232645206, + 0.2894708433883089, + 0.35426734262803905, + 0.3315559903857778, + 0.299113843057621, + 0.2544418722122676, + 0.31385938457638674, + 1.0420245505605208, + 0.318993315295164, + 0.34349833104116767, + 0.3255579765134605, + 0.29442548758489373, + 0.3987172591117573, + 0.30073002794452897, + 0.546817744326097, + 0.3984919478712279, + 0.3464847453081712, + 0.3003434589469694, + 0.4776827711261298, + 0.28642210724056466, + 0.3349675443732946, + 0.5101122204609412, + 0.2578093204663691, + 0.4379814925673164, + 0.46996234220348776, + 0.4176002456558335, + 0.3626613959024197, + 0.367318734376945, + 0.3283973399386791, + 0.5654930460938172, + 0.3074792607067829, + 0.39911835113632416, + 0.2975595777600581, + 0.8527737502110125, + 0.34935037900258287, + 0.3241944378985564, + 0.380703337497849, + 0.41015493638972567, + 0.27823106110688073, + 0.5595640012574201, + 0.45242865538353927, + 0.46869732780340584, + 0.4203691195107247, + 0.3417242776927336, + 0.3769894280504267, + 0.4734618194291875, + 0.3366606326283432, + 0.3278343067365113, + 0.35367780008983263, + 0.36828200588807847, + 0.300196714524547, + 0.33575258901433847, + 0.4714105335450636, + 0.5390157962244185, + 0.35528148042904917, + 0.5649892233030266, + 0.33373166966799706, + 0.35630534277727477, + 0.3199586499121318, + 0.3124362928216076, + 0.2608963771398867, + 0.3244447414172524, + 0.27379280016397345, + 0.3471016966215186, + 0.4239616054803974, + 0.35016539524356705, + 0.41879757916712407, + 0.32877376158244176, + 0.736442759441735, + 0.3576513458788307, + 0.7281165419953295, + 0.39102381460751234, + 0.3011626440623093, + 0.4656381281636742, + 0.3941697954896977, + 0.29021867451031613, + 0.484998336526039, + 0.3426381359936866 + ] + }, + "n_perts": 80 + }, + "EndpointMLP": { + "mse": 0.0035895461776817685, + "r2": -1.3284923076629638, + "pearson": 0.250856500925147, + "spearman": 0.21472794816122348, + "de_corr": 0.31181388201657684, + "mmd": 0.05454873588020891, + "wasserstein": 0.32661930937190176, + "_per_pert": { + "mse": [ + 0.001127122319303453, + 0.0038664289750158787, + 0.0028467837255448103, + 0.0014646187191829085, + 0.0014612936647608876, + 0.0027446665335446596, + 0.001064940239302814, + 0.0010063059162348509, + 0.0011649008374661207, + 0.03693429380655289, + 0.0013625678839161992, + 0.0016081273788586259, + 0.0019149566069245338, + 0.0014659998705610633, + 0.0028379117138683796, + 0.0015390942571684718, + 0.012924129143357277, + 0.0012876329710707068, + 0.001074393279850483, + 0.0016928522381931543, + 0.0040303501300513744, + 0.0015720046358183026, + 0.0016548498533666134, + 0.0037606824189424515, + 0.0008038526284508407, + 0.002440518932417035, + 0.003538229502737522, + 0.0022133872844278812, + 0.001383767114020884, + 0.0017376290634274483, + 0.0014654132537543774, + 0.0071283262223005295, + 0.0015974423149600625, + 0.002283883048221469, + 0.00131800037343055, + 0.02171213924884796, + 0.001595949288457632, + 0.0016505932435393333, + 0.0028888792730867863, + 0.0019500732887536287, + 0.0016206128057092428, + 0.012017425149679184, + 0.0053583537228405476, + 0.0021676411852240562, + 0.0036209614481776953, + 0.0023437028285115957, + 0.0021934770047664642, + 0.0032005722168833017, + 0.001168526359833777, + 0.0015326308785006404, + 0.001444273511879146, + 0.001496295677497983, + 0.0011518372921273112, + 0.0010961294174194336, + 0.0032753730192780495, + 0.004643608815968037, + 0.001971352379769087, + 0.009740659967064857, + 0.0017063472187146544, + 0.0008499862160533667, + 0.0021783006377518177, + 0.0014902957482263446, + 0.0005737734027206898, + 0.0015710580628365278, + 0.0009349629981443286, + 0.0012233067536726594, + 0.010088352486491203, + 0.00160489440895617, + 0.002638954669237137, + 0.0011848068097606301, + 0.014412730000913143, + 0.0019192028557881713, + 0.013400845229625702, + 0.0022466478403657675, + 0.001667212462052703, + 0.0026062768884003162, + 0.0028331680223345757, + 0.0036577777937054634, + 0.006516745314002037, + 0.0036996235139667988 + ], + "r2": [ + -4.475307941436768, + 0.4601461887359619, + -0.9183167219161987, + -3.2408390045166016, + -0.6946736574172974, + -1.1624839305877686, + -2.8527326583862305, + -0.5544227361679077, + -1.86484956741333, + 0.15095454454421997, + -4.204568862915039, + 0.22532737255096436, + -2.0408895015716553, + -2.5536386966705322, + -1.01019287109375, + -4.805403232574463, + 0.1432427167892456, + 0.08922445774078369, + 0.4574959874153137, + -3.902205467224121, + 0.2522217631340027, + -3.4442105293273926, + -5.022956371307373, + -0.2742190361022949, + -1.0319631099700928, + 0.3080607056617737, + 0.34515416622161865, + 0.3249819874763489, + 0.2443619966506958, + -1.6190335750579834, + -0.19535863399505615, + 0.16529887914657593, + -10.943259239196777, + -1.3682820796966553, + -1.5041310787200928, + 0.19772768020629883, + -1.982025146484375, + -4.160525321960449, + -0.2474726438522339, + 0.2883746027946472, + -5.253929138183594, + -0.004741787910461426, + 0.3158472180366516, + 0.05616021156311035, + 0.0448797345161438, + -0.4273139238357544, + -3.4859161376953125, + 0.32477867603302, + 0.39654701948165894, + -0.1661534309387207, + -0.13131403923034668, + 0.20807552337646484, + -1.9789049625396729, + 0.26391881704330444, + -0.011372566223144531, + -0.2728334665298462, + -1.3555784225463867, + 0.133306086063385, + -2.086256265640259, + 0.3351597785949707, + -1.3143935203552246, + -10.458815574645996, + -0.2174438238143921, + -0.5521258115768433, + -3.360511302947998, + -0.5878949165344238, + 0.20500487089157104, + -5.242621421813965, + -0.0920487642288208, + -0.3024890422821045, + 0.06828242540359497, + -2.5230295658111572, + 0.2276684045791626, + -0.172815203666687, + -5.671688556671143, + -1.2691419124603271, + -0.34980273246765137, + 0.3885419964790344, + 0.24096685647964478, + 0.22200262546539307 + ], + "pearson": [ + 0.09920099377632141, + 0.8252658247947693, + 0.026621999219059944, + -0.01993238367140293, + 0.21930858492851257, + -0.20804326236248016, + 0.15599864721298218, + 0.44416487216949463, + 0.2452300637960434, + 0.5643652081489563, + 0.30330172181129456, + 0.5278626680374146, + -0.13110049068927765, + 0.15480056405067444, + -0.05504201352596283, + -0.16380605101585388, + 0.4292604327201843, + 0.5186289548873901, + 0.6800394654273987, + -0.2522020936012268, + 0.5551401376724243, + -0.010929271578788757, + -0.23540940880775452, + 0.13279856741428375, + 0.47484058141708374, + 0.5563710331916809, + 0.6156711578369141, + 0.596950888633728, + 0.563797116279602, + 0.10106407850980759, + 0.32546499371528625, + 0.4203547239303589, + -0.5379387736320496, + -0.004426982253789902, + 0.15489937365055084, + 0.7091835737228394, + -0.059812210500240326, + -0.026333341374993324, + 0.2314753234386444, + 0.5407662987709045, + -0.3146686851978302, + 0.47836756706237793, + 0.6216529607772827, + 0.39034759998321533, + 0.316742479801178, + 0.13878190517425537, + -0.15456554293632507, + 0.5966429710388184, + 0.6699447631835938, + 0.42552998661994934, + 0.39872393012046814, + 0.5657221674919128, + 0.21035589277744293, + 0.6282144784927368, + 0.3426300883293152, + 0.003669868456199765, + -0.09659762680530548, + 0.4318743944168091, + -0.05355288088321686, + 0.6674680113792419, + -0.18533453345298767, + -0.33597368001937866, + 0.6175138354301453, + 0.3519861400127411, + 0.25243276357650757, + 0.4155485928058624, + 0.5535225868225098, + -0.3515574336051941, + 0.3490539491176605, + 0.33445024490356445, + 0.2636698782444, + -0.1220380961894989, + 0.5844600200653076, + 0.30654919147491455, + -0.29786574840545654, + -0.2186243087053299, + 0.08098004013299942, + 0.6871530413627625, + 0.5168100595474243, + 0.5306476354598999 + ], + "spearman": [ + 0.03888177372044343, + 0.7877927014481753, + 0.10631400557850139, + -0.011374595847914436, + 0.12234314758578688, + -0.15317153179288293, + 0.06716425679106419, + 0.3705997331499333, + 0.20893809273452318, + 0.4379899829974958, + 0.2659442344860586, + 0.43818040504510125, + -0.09872413118103279, + 0.1473797063449266, + 0.04972579543144886, + -0.18694469573617395, + 0.41547474786868693, + 0.3735217218804305, + 0.598626148156537, + -0.22816709104177274, + 0.5126583951645988, + -0.11385004946251236, + -0.21070932917733232, + 0.1012825828206457, + 0.40264614916153724, + 0.4631380022845006, + 0.5822098950524738, + 0.4923821175955293, + 0.45330493282623324, + 0.09507180576795143, + 0.34267341554685643, + 0.29803271650817914, + -0.5064749511187377, + 0.0038176524544131134, + 0.26502053275513315, + 0.6677267269316818, + -0.026703093675773417, + 0.016423606105901527, + 0.19855922413980603, + 0.4673530183382545, + -0.17711831027957756, + 0.4058125419531355, + 0.5617419819354955, + 0.2862896410724103, + 0.2336820944205236, + 0.06965975941493983, + -0.22701436925359228, + 0.5253518378379595, + 0.5855090363772592, + 0.390338484084621, + 0.3386314326578581, + 0.4083484845871211, + 0.2015706503926626, + 0.5082925195731298, + 0.32312988728247183, + 0.02577252044313011, + -0.007738872434718108, + 0.28880041455840383, + -0.10372452943113236, + 0.6056198714049679, + -0.17284013221003303, + -0.3049524912381228, + 0.5373508553377139, + 0.28237011909252974, + 0.22148375387093847, + 0.36935717933929485, + 0.5145281696320425, + -0.2672186348046587, + 0.1988961572240393, + 0.24093372023343002, + 0.25703488025872007, + -0.07829796010385177, + 0.5759205749801437, + 0.2561287105321776, + -0.255496731874183, + -0.15104875376218846, + 0.04211718452929613, + 0.6201021815255454, + 0.3619787166304212, + 0.4318755194688798 + ], + "de_corr": [ + 0.6556634902954102, + 0.9295767545700073, + 0.0986441969871521, + -0.1064327210187912, + 0.5840852856636047, + -0.527094841003418, + -0.2910711467266083, + 0.5150852799415588, + 0.4458194077014923, + 0.7847771644592285, + 0.29429423809051514, + 0.7123333215713501, + -0.20313183963298798, + -0.012626292183995247, + 0.057278137654066086, + -0.38771840929985046, + 0.4616764783859253, + 0.6080045104026794, + 0.6075575351715088, + -0.09871281683444977, + 0.8424193859100342, + 0.3145641088485718, + 0.027318671345710754, + -0.3820090591907501, + 0.7703711986541748, + 0.710444986820221, + 0.16406331956386566, + 0.41748231649398804, + 0.6342500448226929, + 0.08571234345436096, + 0.07436976581811905, + 0.5517820119857788, + 0.24646161496639252, + -0.28841596841812134, + -0.2558394968509674, + 0.8195428848266602, + 0.07226260006427765, + 0.34800562262535095, + 0.21656228601932526, + 0.7648283839225769, + -0.558334469795227, + 0.8481343388557434, + 0.8454898595809937, + 0.6628275513648987, + 0.465027391910553, + 0.48272183537483215, + -0.37859174609184265, + 0.6653576493263245, + 0.6579313278198242, + 0.6672171354293823, + 0.7007299661636353, + 0.8282674551010132, + 0.18529564142227173, + 0.8476783037185669, + 0.5111265778541565, + 0.10663695633411407, + -0.4882339835166931, + 0.7139249444007874, + 0.1261516958475113, + 0.6795224547386169, + -0.3049156665802002, + -0.3979416787624359, + 0.725322425365448, + 0.19080251455307007, + 0.44852665066719055, + 0.2152465134859085, + 0.8711315393447876, + -0.386370986700058, + 0.6103668808937073, + 0.5270323753356934, + 0.09399121254682541, + -0.2579564154148102, + 0.7273256778717041, + 0.4903419613838196, + -0.7069991827011108, + -0.20244602859020233, + -0.11720681935548782, + 0.8679450750350952, + 0.8224426507949829, + 0.8974062204360962 + ], + "mmd": [ + 0.024653153190544952, + 0.06127176870561479, + 0.04634376996303646, + 0.02611160850714056, + 0.03603681103133838, + 0.05539728873844485, + 0.02459695745893975, + 0.019133813399382915, + 0.024211087467252645, + 0.34368256233904115, + 0.02468863291009049, + 0.03355942509663845, + 0.032276374628334104, + 0.0290244567070167, + 0.0499887140267915, + 0.028786502510649514, + 0.1340164149472719, + 0.03093230873014219, + 0.033786824798984205, + 0.029508048803127407, + 0.0670713897721198, + 0.03144507398922669, + 0.030947747740915243, + 0.05630633331367474, + 0.022610202593777662, + 0.049025620370418355, + 0.06549815728229424, + 0.04743400320247082, + 0.03956673969518243, + 0.044191567688844136, + 0.037424826205595396, + 0.10444054870519415, + 0.026427951403992345, + 0.04053298469962496, + 0.030629547268314017, + 0.17701698889208728, + 0.03856779024594914, + 0.029335227702307343, + 0.05743547493701562, + 0.04101052917325143, + 0.029719230014992104, + 0.11146467775571389, + 0.07849297343193429, + 0.04403525436740352, + 0.05902696296544263, + 0.043307341203411065, + 0.034891593873145554, + 0.06379470643298735, + 0.030228242442304265, + 0.03172892032296515, + 0.031475752208932284, + 0.030200567643126064, + 0.02468015780363353, + 0.026277682480642994, + 0.06806273709414734, + 0.0835141295861489, + 0.04019696725146871, + 0.12342623685309673, + 0.03352031158869151, + 0.03236418610562419, + 0.04213719706030161, + 0.029605143428978353, + 0.027356019972873513, + 0.034830210403563266, + 0.022547693309341033, + 0.029182291628671253, + 0.15669001873723076, + 0.03024801574765612, + 0.039434428594297355, + 0.030950871102351285, + 0.16154659082894107, + 0.04092216394331383, + 0.17306863761551583, + 0.04058085300740888, + 0.030883202072969018, + 0.059347669800112635, + 0.05905166708060805, + 0.066710402899674, + 0.08058609365880587, + 0.06288583925624835 + ], + "wasserstein": [ + 0.23776511139385448, + 0.3364284084244217, + 0.35765990670818076, + 0.24152438211619118, + 0.27512441943364635, + 0.3167653773874684, + 0.2286023680512722, + 0.1857123780190565, + 0.2439388541279063, + 0.9686100128801676, + 0.2304241505811, + 0.24136214811172535, + 0.2518370420730539, + 0.21498344630634478, + 0.3512967282528351, + 0.2382904125742072, + 0.5662853500112085, + 0.29596386578016104, + 0.25133390621651264, + 0.26067101865666215, + 0.3913021335276099, + 0.23386440101379094, + 0.2594231680199838, + 0.3700112151469803, + 0.21237847280929653, + 0.3262517072693322, + 0.3604945402932277, + 0.32724404673703555, + 0.2672680495862581, + 0.291577965638649, + 0.24402843320200676, + 0.4673749505744415, + 0.23885585908448267, + 0.3458751174776898, + 0.22581349775003873, + 0.7820921988899502, + 0.2902141703873509, + 0.24359597830292704, + 0.3668434292577229, + 0.3290019206424651, + 0.24583592709992036, + 0.522698232879563, + 0.39863848415809194, + 0.3697300799116194, + 0.33073865963713034, + 0.2857184881552608, + 0.30690648540576987, + 0.3543514376901068, + 0.25576173876632796, + 0.2489677803839507, + 0.25598481405892826, + 0.2533597145788599, + 0.236076622373976, + 0.2483129948160706, + 0.4133199638572333, + 0.4915354404177103, + 0.2886243939983514, + 0.47345976067167816, + 0.27025756150643726, + 0.2549727460875848, + 0.26924674606045734, + 0.26020824418398986, + 0.20265782611895256, + 0.2431515374652639, + 0.18820524008924572, + 0.2681499793150304, + 0.4802112048374144, + 0.26468327551589665, + 0.3234147999692766, + 0.23194208844515546, + 0.710669573736546, + 0.3042050845999533, + 0.6314052137459768, + 0.2964225534702546, + 0.23675289325137938, + 0.37575594444429816, + 0.3549486222362011, + 0.35521879210008267, + 0.3945653728777734, + 0.3643878881172084 + ] + }, + "n_perts": 80 + }, + "Random": { + "mse": 0.005731495868167258, + "r2": -3.9902552790939807, + "pearson": 0.10796300343354233, + "spearman": 0.08541606175072594, + "de_corr": 0.1627314489451237, + "mmd": 0.08371733022570763, + "wasserstein": 0.38453655729876746, + "_per_pert": { + "mse": [ + 0.0007089351420290768, + 0.008821012452244759, + 0.0020469308365136385, + 0.0006849450874142349, + 0.0043114470317959785, + 0.0011423415271565318, + 0.0005672886618413031, + 0.0008336359751410782, + 0.001187272253446281, + 0.03069770522415638, + 0.0004783738113474101, + 0.0028958283364772797, + 0.0010876067681238055, + 0.0008117451798170805, + 0.00293916929513216, + 0.015011005103588104, + 0.014183963648974895, + 0.001030571642331779, + 0.0021483555901795626, + 0.0015328560257330537, + 0.005268664564937353, + 0.001961674075573683, + 0.0027573390398174524, + 0.007547278888523579, + 0.004847305361181498, + 0.005906644742935896, + 0.005155965685844421, + 0.004809620790183544, + 0.007731635123491287, + 0.014606324955821037, + 0.005666823126375675, + 0.00840443279594183, + 0.0013987176353111863, + 0.0017399343196302652, + 0.001193966600112617, + 0.028056053444743156, + 0.001200039405375719, + 0.006111594382673502, + 0.0035988478921353817, + 0.0032748717349022627, + 0.0009552983101457357, + 0.017926927655935287, + 0.008409296162426472, + 0.0024114453699439764, + 0.002974275965243578, + 0.0029405285604298115, + 0.0011780605418607593, + 0.005296229384839535, + 0.009673490189015865, + 0.0022123013623058796, + 0.002645889762789011, + 0.001855160458944738, + 0.0011057923547923565, + 0.0012199480552226305, + 0.004115937277674675, + 0.004064055159687996, + 0.002257246058434248, + 0.011709428392350674, + 0.0013004689244553447, + 0.0018484409665688872, + 0.011370744556188583, + 0.0013224276481196284, + 0.01560946088284254, + 0.008660782128572464, + 0.0005711812991648912, + 0.0009117615409195423, + 0.013189605437219143, + 0.005606173072010279, + 0.010388492606580257, + 0.0014824297977611423, + 0.016576426103711128, + 0.0026809414848685265, + 0.021129455417394638, + 0.0035076194908469915, + 0.011581161059439182, + 0.002110458677634597, + 0.00973776075989008, + 0.008015350438654423, + 0.008832697756588459, + 0.004775794222950935 + ], + "r2": [ + -2.4438490867614746, + -0.2316422462463379, + -0.3793325424194336, + -0.9832749366760254, + -4.00001859664917, + 0.09996533393859863, + -1.0523324012756348, + -0.2877023220062256, + -1.919867992401123, + 0.29432129859924316, + -0.8272333145141602, + -0.39498841762542725, + -0.7270846366882324, + -0.9677009582519531, + -1.0819175243377686, + -55.62091827392578, + 0.05972665548324585, + 0.2710505723953247, + -0.08479034900665283, + -3.438884735107422, + 0.02246910333633423, + -4.545844078063965, + -9.035552024841309, + -1.5572185516357422, + -11.252920150756836, + -0.674660325050354, + 0.04574793577194214, + -0.4667927026748657, + -3.2220373153686523, + -21.015317916870117, + -3.622509479522705, + 0.01587134599685669, + -9.45749568939209, + -0.804232120513916, + -1.2684741020202637, + -0.03668248653411865, + -1.2422685623168945, + -18.107694625854492, + -0.5540505647659302, + -0.1950739622116089, + -2.6864869594573975, + -0.4988182783126831, + -0.07369625568389893, + -0.04999744892120361, + 0.21545934677124023, + -0.7907806634902954, + -1.4092705249786377, + -0.1173398494720459, + -3.99560546875, + -0.6833033561706543, + -1.0725524425506592, + 0.018143832683563232, + -1.8598225116729736, + 0.1807713508605957, + -0.27092278003692627, + -0.11397504806518555, + -1.6971940994262695, + -0.04186892509460449, + -1.3521475791931152, + -0.44580912590026855, + -11.081151962280273, + -9.168085098266602, + -32.12046432495117, + -7.556413650512695, + -1.6638939380645752, + -0.18349850177764893, + -0.03938400745391846, + -20.80655288696289, + -3.298952579498291, + -0.6296738386154175, + -0.07159066200256348, + -3.9213337898254395, + -0.21775519847869873, + -0.8310792446136475, + -45.34436798095703, + -0.837459921836853, + -3.639350414276123, + -0.33989834785461426, + -0.028782248497009277, + -0.004306197166442871 + ], + "pearson": [ + -0.03264115750789642, + 0.37640857696533203, + 0.1704404205083847, + 0.24992913007736206, + 0.1759069710969925, + 0.34897348284721375, + 0.13758163154125214, + 0.25945958495140076, + 0.3447943925857544, + 0.7289174199104309, + 0.29581159353256226, + 0.050837207585573196, + -0.007263404317200184, + 0.20736688375473022, + -0.4038006663322449, + -0.06737647205591202, + 0.28492170572280884, + 0.5648890733718872, + 0.14326710999011993, + 0.03372571989893913, + 0.37945669889450073, + -0.3558201491832733, + -0.02925780415534973, + 0.08582435548305511, + 0.3391045033931732, + 0.5492660403251648, + 0.2652689218521118, + -0.3427378535270691, + 0.5268052220344543, + 0.08322452753782272, + 0.3598836660385132, + 0.34660664200782776, + -0.2665960192680359, + -0.10831008106470108, + 0.15123681724071503, + 0.07758726924657822, + 0.07023529708385468, + -0.03271499276161194, + -0.15625835955142975, + -0.006269107572734356, + 0.06777283549308777, + -0.12218057364225388, + 0.1114918664097786, + 0.1986599713563919, + 0.497450590133667, + 0.28565484285354614, + -0.14018990099430084, + 0.20590418577194214, + 0.38167285919189453, + 0.27619796991348267, + -0.1401526927947998, + 0.2846909761428833, + 0.24297642707824707, + 0.4516053795814514, + -0.19533981382846832, + 0.07464978098869324, + -0.18705028295516968, + 0.0044649564661085606, + -0.031111685559153557, + -0.2127649039030075, + -0.05407530069351196, + -0.09476955235004425, + 0.489244282245636, + -0.07461868226528168, + 0.41796621680259705, + 0.5020486116409302, + 0.15493178367614746, + -0.31572845578193665, + 0.016724226996302605, + -0.022708933800458908, + 0.17155121266841888, + -0.22111478447914124, + -0.4945182502269745, + -0.19132943451404572, + -0.18722616136074066, + -0.033592499792575836, + 0.04519905522465706, + -0.008302725851535797, + 0.3888622224330902, + 0.2954098582267761 + ], + "spearman": [ + -0.031472436368109084, + 0.36377203294300825, + 0.2784006951001738, + 0.23385356846339211, + 0.09119668229917056, + 0.30063254652087384, + 0.07676169269042317, + 0.1923917535979384, + 0.24188834297208575, + 0.6900908445227111, + 0.2406512311628078, + 0.03258383314595829, + 0.0048122412030603, + 0.06511725427931357, + -0.38705258476314613, + -0.09572314343078585, + 0.1825278456319614, + 0.3952074598018649, + 0.189902751975688, + -0.040943993735998434, + 0.29137957134489284, + -0.337684051921013, + -0.1316582969145742, + 0.09855397863849466, + 0.288224448056112, + 0.5037910424477606, + 0.18371428142857035, + -0.3074484633621158, + 0.35612232353058093, + 0.14097573974393493, + 0.3850985492746373, + 0.302157848039462, + -0.2684021996005499, + -0.06184557796139449, + 0.18101824975456243, + 0.047986931996733, + 0.08109663977415994, + 0.010528205632051407, + -0.19117999329499832, + 0.09171125742781436, + 0.019397244349311085, + -0.10510627877656968, + 0.14704211626052907, + 0.19621192905298226, + 0.5504714396178599, + 0.2355586188896547, + -0.22620812205203053, + 0.16451522462880616, + 0.2549421382355346, + 0.22291997572999395, + 0.017544934386233595, + 0.2569640832410208, + 0.21000122550030637, + 0.4383962415990603, + -0.12217191354297839, + 0.05319609729902433, + -0.2388941172235293, + 0.08584532746133187, + -0.12853041663260414, + -0.15257854714463678, + -0.10307449526862382, + -0.13765548591387147, + 0.47913844178461046, + -0.12709402677350667, + 0.3588674637168659, + 0.3673061673265418, + 0.1233695468423867, + -0.2793412148353037, + 0.049922046480511614, + -0.11380587145146785, + 0.18356440139110036, + -0.3357247380570813, + -0.3455031338757834, + -0.13551736137934034, + -0.12551400887850223, + -0.15291013272753318, + -0.01451540912885228, + 0.014072033018008256, + 0.3758475804618951, + 0.1835968343992086 + ], + "de_corr": [ + -0.1082049310207367, + 0.5698527097702026, + -0.1347426027059555, + 0.3213602304458618, + 0.6617529988288879, + 0.5688132047653198, + 0.07467389106750488, + 0.42532825469970703, + 0.6664502620697021, + 0.859486997127533, + 0.2401142120361328, + 0.2947470247745514, + 0.07618106156587601, + 0.6178860664367676, + -0.6350488662719727, + -0.17713627219200134, + 0.3953942060470581, + 0.8594666719436646, + 0.5670092105865479, + 0.3741897940635681, + 0.463733434677124, + -0.2644274830818176, + 0.6111770272254944, + -0.4421440064907074, + 0.3524090051651001, + 0.7179036140441895, + 0.7089893221855164, + -0.41583314538002014, + 0.733873724937439, + -0.0783306136727333, + 0.23620223999023438, + 0.49391791224479675, + 0.4121815264225006, + -0.43366679549217224, + -0.1844785064458847, + 0.3163650333881378, + 0.13560013473033905, + -0.4491572082042694, + -0.04302889481186867, + -0.1760999858379364, + 0.4785613417625427, + -0.28885379433631897, + 0.08850489556789398, + 0.29578810930252075, + 0.8189729452133179, + 0.7343198657035828, + -0.13016217947006226, + 0.4582599103450775, + 0.7011787295341492, + 0.13158321380615234, + -0.3754953444004059, + 0.5352423787117004, + 0.16164542734622955, + 0.46075722575187683, + -0.4569880962371826, + 0.23308917880058289, + -0.347536563873291, + -0.03363665193319321, + -0.18536072969436646, + -0.5357833504676819, + -0.25681763887405396, + 0.10249623656272888, + 0.645117461681366, + -0.17644517123699188, + 0.7056043148040771, + 0.7571634650230408, + 0.2653813064098358, + -0.7877164483070374, + -0.1543988287448883, + 0.3882720172405243, + 0.0012331822654232383, + -0.0011248275404796004, + -0.753126859664917, + -0.6021104454994202, + -0.5806601047515869, + 0.08803112059831619, + 0.27570709586143494, + 0.01906701549887657, + 0.5531787872314453, + 0.5728172659873962 + ], + "mmd": [ + 0.017040960627189183, + 0.12762710416924128, + 0.043614174499218894, + 0.01703074763030543, + 0.08608555619181135, + 0.029307035678301507, + 0.015620226370748669, + 0.014739199570294126, + 0.02688087840319664, + 0.3103851149616311, + 0.012419129690211084, + 0.04821032652165069, + 0.024915695636547164, + 0.019904058844330663, + 0.05756215367551276, + 0.22688003112811095, + 0.14946043653557928, + 0.024377214054675855, + 0.03891728897881741, + 0.031151292500181227, + 0.07961855477854618, + 0.0387180916666533, + 0.049320283835500245, + 0.10929050656236527, + 0.08947331400540615, + 0.10263237604448283, + 0.07999009045144079, + 0.08146212979090539, + 0.13080403966718868, + 0.23251270890067544, + 0.10768434642778202, + 0.11968189581847588, + 0.02318412173851392, + 0.03861924175921749, + 0.025899351425215933, + 0.20075982334220988, + 0.030900076112676866, + 0.09759288408087397, + 0.06278547697702563, + 0.05409812554652049, + 0.021804517046842165, + 0.14552957218177598, + 0.11302131785337155, + 0.0485056771595489, + 0.052564125641590076, + 0.05981986855132693, + 0.024462870859493835, + 0.09118629333483519, + 0.15587417993663688, + 0.03882152332416389, + 0.04866481096793529, + 0.03038698461661493, + 0.024237022205199188, + 0.02737090200271053, + 0.07627184757257366, + 0.07881308174989199, + 0.04595556954818625, + 0.14072345282255772, + 0.028391002764072915, + 0.03225373790299624, + 0.18447054124540585, + 0.02823458138307844, + 0.2396437421981179, + 0.136355319052629, + 0.015153257880321291, + 0.02230819627452696, + 0.18993385185878298, + 0.09678458754134678, + 0.14099192784577796, + 0.03353027221344851, + 0.18623763107501834, + 0.052406721669188916, + 0.23740676581797515, + 0.0623453204302542, + 0.17856562397705666, + 0.05369014915966652, + 0.16325322691059663, + 0.12652258184410903, + 0.11175393771078013, + 0.07598375932697232 + ], + "wasserstein": [ + 0.18125505934357045, + 0.4763378522954169, + 0.3221971306491185, + 0.21453263511135984, + 0.3415888394775791, + 0.24896675124426385, + 0.16171736934417968, + 0.19061504712406088, + 0.2351030487826268, + 0.917685229808255, + 0.15826188632003924, + 0.315405553973528, + 0.21966924510576583, + 0.20942248705342412, + 0.3878459020864893, + 0.5809931614125661, + 0.5802167511401514, + 0.273105176338688, + 0.2925864512199406, + 0.2677524767029161, + 0.4285773209972749, + 0.23214074122004089, + 0.3369914958282573, + 0.46856100852557625, + 0.3707866466947403, + 0.46320398130438667, + 0.3997358219943042, + 0.3917988252263729, + 0.49646901407489513, + 0.6106102266948075, + 0.3786992390374902, + 0.4776058954077525, + 0.20576249914242248, + 0.31439748906022075, + 0.2288918153223997, + 0.8614028548624288, + 0.22532772648416638, + 0.40383294297472866, + 0.3873290032406132, + 0.41115783310067655, + 0.2198695507631816, + 0.6494952187971902, + 0.46384177394574044, + 0.3852701550000657, + 0.34761787903974584, + 0.38206792052351907, + 0.24612131580434268, + 0.41792360586905153, + 0.4368049241141519, + 0.2596746186533206, + 0.31592883806747346, + 0.289266271668532, + 0.2487129545048104, + 0.22746089872692335, + 0.42943393196961, + 0.46015227281029253, + 0.27482747868594837, + 0.5248028773455765, + 0.2284900322245324, + 0.30579941197622346, + 0.47054243049407324, + 0.23502897115245447, + 0.6824241816884907, + 0.44887703813027097, + 0.1646972199050915, + 0.22522403300645064, + 0.5578620819484975, + 0.3940362365335743, + 0.4447584296896948, + 0.24638999929905492, + 0.7572366094956215, + 0.3126774707574512, + 0.7798956522083212, + 0.36408013555467456, + 0.5974972662231495, + 0.350913207560339, + 0.5700202124541115, + 0.46143657338822963, + 0.5038806099839521, + 0.4133438582101676 + ] + }, + "n_perts": 80 + } + } +} \ No newline at end of file diff --git a/experiments/results/replogle_k562_nom_cell.json b/experiments/results/replogle_k562_nom_cell.json new file mode 100644 index 0000000000000000000000000000000000000000..4dec2b68471190ed6845b0d0d4cc8d3384ccc1ce --- /dev/null +++ b/experiments/results/replogle_k562_nom_cell.json @@ -0,0 +1,2773 @@ +{ + "split": "cell", + "reward": "centroid", + "n_candidates": 1805, + "n_targets": 40, + "methods": { + "PIVOT-ranking": { + "top1": 0.025, + "top5": 0.15, + "ndcg": 0.13208262047549235, + "func_top5": 0.725, + "func_ndcg": 0.3600167442071178, + "rank": 194.2, + "endpoint_dist": 1.2301450669765472, + "spec_margin": 1.2941597983241082, + "off_dist": 2.5243048638105394, + "clf": null, + "_per": { + "top1": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "top5": [ + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "ndcg": [ + 0.0, + 0.6309297535708266, + 0.0, + 0.0, + 0.3154648767854133, + 0.0, + 0.9999999999989999, + 0.6309297535708266, + 0.0, + 0.0, + 0.0, + 0.43067655807296235, + 0.0, + 0.0, + 0.30102999566368016, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.6309297535708266, + 0.6309297535708266, + 0.0, + 0.0, + 0.0, + 0.35620718710766597, + 0.0, + 0.0, + 0.0, + 0.35620718710766597, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "func_top5": [ + 0.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.0, + 0.9334235959386915, + 0.0, + 0.8196104814589247, + 0.8765170386988083, + 0.672509164948684, + 0.9999999999996392, + 0.7863222794284508, + 0.06978419164316285, + 0.05430265598413898, + 0.0, + 0.8973000184165562, + 0.11381311447976694, + 0.0, + 0.10860531196827795, + 0.5254078484384431, + 0.408403654282275, + 0.1803895185407146, + 0.0, + 0.3821659031256479, + 0.06012983951357154, + 0.5499613274410384, + 0.7988275017411766, + 0.6053288745612656, + 0.3123817114824851, + 0.3155894990647003, + 0.12851208596631675, + 0.39817794789001704, + 0.4170340107060822, + 0.4057842583350576, + 0.883866524442083, + 0.20400787375012425, + 0.5065517856223779, + 0.11381311447976694, + 0.0, + 0.13459609419751495, + 0.0, + 0.30055347352382106, + 0.29250165296063585, + 0.1444974152544963 + ], + "rank": [ + 410, + 2, + 134, + 23, + 8, + 49, + 1, + 2, + 25, + 108, + 290, + 4, + 41, + 561, + 9, + 77, + 151, + 185, + 103, + 59, + 232, + 2, + 2, + 758, + 121, + 346, + 6, + 20, + 28, + 606, + 6, + 145, + 651, + 281, + 789, + 92, + 332, + 739, + 105, + 265 + ], + "endpoint_dist": [ + 0.7788619995117188, + 1.0574302673339844, + 0.8981291055679321, + 1.1791802644729614, + 1.9431819915771484, + 1.2274514436721802, + 1.570068120956421, + 0.8371641039848328, + 1.1645748615264893, + 1.684700846672058, + 1.1419439315795898, + 1.1942020654678345, + 1.262868881225586, + 0.9169500470161438, + 0.9980612397193909, + 1.5394939184188843, + 1.5928115844726562, + 0.8636164665222168, + 0.5946463346481323, + 0.6364714503288269, + 0.9466665983200073, + 1.3823901414871216, + 1.350879430770874, + 1.2104741334915161, + 1.2438565492630005, + 1.172344446182251, + 1.0656836032867432, + 0.8919244408607483, + 1.4183169603347778, + 1.4194594621658325, + 2.675273895263672, + 1.5831266641616821, + 1.1956759691238403, + 1.3301384449005127, + 1.29361891746521, + 0.6645997166633606, + 1.49713134765625, + 1.0711721181869507, + 1.2048929929733276, + 1.5063679218292236 + ], + "spec_margin": [ + 1.6204698085784912, + 1.539564609527588, + 1.7510727643966675, + 3.4558401107788086, + 0.8276746273040771, + 1.0009196996688843, + 1.8262240886688232, + 1.4334993362426758, + 1.2165818214416504, + 0.6880141496658325, + 1.535628318786621, + 2.1577844619750977, + 1.1765739917755127, + 1.50803542137146, + 0.6875573992729187, + 0.7675319910049438, + 1.07802152633667, + 2.0859181880950928, + 1.3635112047195435, + 1.8178761005401611, + 1.6484087705612183, + 0.6289695501327515, + 1.1417169570922852, + 1.1009031534194946, + 1.310752272605896, + 1.564943552017212, + 1.851792812347412, + 1.88722562789917, + 1.2785838842391968, + 0.3919602632522583, + 0.6825873851776123, + 0.3784511089324951, + 1.6347113847732544, + 1.420490026473999, + 0.4672729969024658, + 1.5788979530334473, + 0.6988112926483154, + 0.9903422594070435, + 1.238538384437561, + 0.3327326774597168 + ], + "off_dist": [ + 2.39933180809021, + 2.5969948768615723, + 2.6492018699645996, + 4.6350202560424805, + 2.7708566188812256, + 2.2283711433410645, + 3.396292209625244, + 2.2706634998321533, + 2.3811566829681396, + 2.3727149963378906, + 2.677572250366211, + 3.3519866466522217, + 2.4394428730010986, + 2.424985408782959, + 1.6856186389923096, + 2.307025909423828, + 2.670833110809326, + 2.9495346546173096, + 1.9581575393676758, + 2.454347610473633, + 2.5950753688812256, + 2.011359691619873, + 2.492596387863159, + 2.3113772869110107, + 2.5546088218688965, + 2.737287998199463, + 2.9174764156341553, + 2.7791500091552734, + 2.6969008445739746, + 1.8114197254180908, + 3.357861280441284, + 1.9615777730941772, + 2.8303873538970947, + 2.7506284713745117, + 1.7608919143676758, + 2.243497610092163, + 2.1959426403045654, + 2.061514377593994, + 2.4434313774108887, + 1.8391005992889404 + ], + "clf": [] + }, + "n_targets": 40, + "random_top5": 0.002770083102493075 + }, + "PIVOT-guidance": { + "top1": 0.05, + "top5": 0.075, + "ndcg": 0.06076691395177405, + "func_top5": 0.675, + "func_ndcg": 0.28302129846659285, + "rank": 197.55, + "endpoint_dist": 1.3085480779409409, + "spec_margin": 1.1785019636154175, + "off_dist": 2.4870500385761263, + "clf": null, + "_per": { + "top1": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "top5": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "ndcg": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.43067655807296235, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.9999999999989999, + 0.9999999999989999, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "func_top5": [ + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.0, + 0.5490262036478528, + 0.17394295399333848, + 0.8196104814589247, + 0.6976820249692228, + 0.6536531021326187, + 0.8196104814589247, + 0.05690655723988347, + 0.052144264846539085, + 0.0, + 0.0, + 0.8196104814589247, + 0.24557383318562032, + 0.0, + 0.11381311447976694, + 0.3731885353501183, + 0.43146614992049087, + 0.1803895185407146, + 0.0, + 0.42886224866474637, + 0.06978419164316285, + 0.5548688818687583, + 0.5031180611983256, + 0.5838689104894559, + 0.1788350137295854, + 0.3934156554966727, + 0.0, + 0.2900442269536079, + 0.0, + 0.16918066159999412, + 0.7032101736292273, + 0.1298338018041706, + 0.4492728162919408, + 0.0, + 0.0, + 0.06978419164316285, + 0.0, + 0.24048715545699928, + 0.4015524750470583, + 0.16811577046390594 + ], + "rank": [ + 411, + 12, + 135, + 26, + 17, + 56, + 11, + 12, + 33, + 113, + 291, + 14, + 4, + 561, + 18, + 84, + 156, + 188, + 106, + 62, + 234, + 1, + 1, + 758, + 128, + 349, + 15, + 28, + 38, + 607, + 14, + 147, + 651, + 283, + 790, + 97, + 336, + 739, + 111, + 265 + ], + "endpoint_dist": [ + 0.7788619995117188, + 2.117431879043579, + 0.8981291055679321, + 1.1791802644729614, + 2.233966827392578, + 1.2716954946517944, + 2.434670925140381, + 1.1498734951019287, + 1.1645748615264893, + 1.684700846672058, + 1.1419439315795898, + 1.521671175956726, + 1.262868881225586, + 0.9169500470161438, + 1.0522229671478271, + 1.5394939184188843, + 1.5928115844726562, + 0.8636164665222168, + 0.5946463346481323, + 0.6364714503288269, + 0.9466665983200073, + 1.3372395038604736, + 1.2468385696411133, + 1.2104741334915161, + 1.4099704027175903, + 1.172344446182251, + 1.0656836032867432, + 0.8919244408607483, + 1.6904374361038208, + 1.4194594621658325, + 2.675273895263672, + 1.5831266641616821, + 1.1956759691238403, + 1.3301384449005127, + 1.29361891746521, + 0.6645997166633606, + 1.390235424041748, + 1.0711721181869507, + 1.2048929929733276, + 1.5063679218292236 + ], + "spec_margin": [ + 1.6204698085784912, + -0.10165095329284668, + 1.7510727643966675, + 3.4558401107788086, + -0.023256778717041016, + 1.0417529344558716, + 0.268704891204834, + 1.0524473190307617, + 1.2165818214416504, + 0.6880141496658325, + 1.535628318786621, + 2.293555736541748, + 1.1765739917755127, + 1.50803542137146, + 0.6877552270889282, + 0.7675319910049438, + 1.07802152633667, + 2.0859181880950928, + 1.3635112047195435, + 1.8178761005401611, + 1.6484087705612183, + 0.7179131507873535, + 1.3804106712341309, + 1.1009031534194946, + 0.9676400423049927, + 1.564943552017212, + 1.851792812347412, + 1.88722562789917, + 0.8419169187545776, + 0.3919602632522583, + 0.6825873851776123, + 0.3784511089324951, + 1.6347113847732544, + 1.420490026473999, + 0.4672729969024658, + 1.5788979530334473, + 0.7785556316375732, + 0.9903422594070435, + 1.238538384437561, + 0.3327326774597168 + ], + "off_dist": [ + 2.39933180809021, + 2.0157809257507324, + 2.6492018699645996, + 4.6350202560424805, + 2.210710048675537, + 2.313448429107666, + 2.703375816345215, + 2.2023208141326904, + 2.3811566829681396, + 2.3727149963378906, + 2.677572250366211, + 3.8152270317077637, + 2.4394428730010986, + 2.424985408782959, + 1.7399781942367554, + 2.307025909423828, + 2.670833110809326, + 2.9495346546173096, + 1.9581575393676758, + 2.454347610473633, + 2.5950753688812256, + 2.055152654647827, + 2.627249240875244, + 2.3113772869110107, + 2.377610445022583, + 2.737287998199463, + 2.9174764156341553, + 2.7791500091552734, + 2.5323543548583984, + 1.8114197254180908, + 3.357861280441284, + 1.9615777730941772, + 2.8303873538970947, + 2.7506284713745117, + 1.7608919143676758, + 2.243497610092163, + 2.1687910556793213, + 2.061514377593994, + 2.4434313774108887, + 1.8391005992889404 + ], + "clf": [] + }, + "n_targets": 40, + "random_top5": 0.002770083102493075 + }, + "Additive+ranking": { + "top1": 0.95, + "top5": 0.975, + "ndcg": 0.9657732438383206, + "func_top5": 1.0, + "func_ndcg": 0.6652048246826138, + "rank": 1.675, + "endpoint_dist": 0.8411503508687019, + "spec_margin": 2.1007207214832304, + "off_dist": 2.9418710708618163, + "clf": null, + "_per": { + "top1": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "top5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "ndcg": [ + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.6309297535708266, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.0, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999 + ], + "func_top5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.3607790370814292, + 0.9999999999996392, + 0.4129233019279683, + 0.9999999999996392, + 0.9999999999996392, + 0.9999999999996392, + 0.9999999999996392, + 0.9456973440155002, + 0.4305632287245921, + 0.4745921515611962, + 0.5426118416977634, + 0.9999999999996392, + 0.4305632287245921, + 0.4646120878678205, + 0.5570272730063581, + 0.873309251116593, + 0.687334748339199, + 0.562183009575809, + 0.3607790370814292, + 0.5082527656822236, + 0.11381311447976694, + 0.6146996865129016, + 0.8654039058021243, + 0.7236207412564216, + 0.759496136636826, + 0.8909491779132166, + 0.4927712300231997, + 0.3607790370814292, + 0.7878767842395802, + 0.9223104630420079, + 0.9999999999996392, + 0.6553540821924644, + 0.5299596986814235, + 0.6734442887418696, + 0.3607790370814292, + 0.3607790370814292, + 0.5721363965748284, + 0.8455491977461236, + 0.9357439570164808, + 0.5314987088010797 + ], + "rank": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 27, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "endpoint_dist": [ + 0.7173737287521362, + 0.7461453080177307, + 0.785219132900238, + 0.538507878780365, + 1.097732424736023, + 0.6946150660514832, + 0.7801607251167297, + 0.7369425296783447, + 1.015824794769287, + 1.3046245574951172, + 1.0173161029815674, + 0.9735252857208252, + 1.092432975769043, + 0.8244563341140747, + 0.674659013748169, + 0.9909878373146057, + 0.9689846038818359, + 0.732428252696991, + 0.5732553005218506, + 0.6204644441604614, + 0.9914161562919617, + 0.7275379300117493, + 0.7922949194908142, + 0.8339179158210754, + 0.9220225214958191, + 0.8180479407310486, + 0.8987788558006287, + 0.7535209059715271, + 0.7320271730422974, + 1.037440299987793, + 1.0151782035827637, + 0.7500125169754028, + 0.8864849805831909, + 0.7386977672576904, + 0.9256577491760254, + 0.8188286423683167, + 0.6080302000045776, + 0.6138859391212463, + 0.9211081266403198, + 0.9754689931869507 + ], + "spec_margin": [ + 1.9674495458602905, + 2.103719711303711, + 2.245748996734619, + 3.573702335357666, + 2.0890731811523438, + 1.7624471187591553, + 3.303356885910034, + 1.8128483295440674, + 2.0443527698516846, + 2.065767288208008, + 1.6571438312530518, + 2.6439476013183594, + 2.032092571258545, + 1.785965085029602, + 1.3168447017669678, + 2.013680934906006, + 2.1790592670440674, + 2.5351104736328125, + 1.5227258205413818, + 2.132510185241699, + 1.9073677062988281, + 1.880821704864502, + 2.3598074913024902, + 1.7640490531921387, + 1.7445905208587646, + 2.3762378692626953, + 2.4540412425994873, + 2.107139825820923, + 2.3813529014587402, + 1.7228913307189941, + 3.270918846130371, + 1.9885872602462769, + 2.2718491554260254, + 2.49912691116333, + 1.3993134498596191, + 1.4822916984558105, + 2.1268625259399414, + 1.731029987335205, + 2.133510112762451, + 1.6394926309585571 + ], + "off_dist": [ + 2.6848232746124268, + 2.849864959716797, + 3.030968189239502, + 4.112210273742676, + 3.1868057250976562, + 2.457062244415283, + 4.083517551422119, + 2.549790859222412, + 3.0601775646209717, + 3.370391845703125, + 2.674459934234619, + 3.6174728870391846, + 3.124525547027588, + 2.6104214191436768, + 1.9915037155151367, + 3.004668712615967, + 3.1480438709259033, + 3.2675387859344482, + 2.0959811210632324, + 2.752974510192871, + 2.8987839221954346, + 2.6083595752716064, + 3.152102470397949, + 2.5979669094085693, + 2.6666131019592285, + 3.1942858695983887, + 3.3528201580047607, + 2.8606607913970947, + 3.113379955291748, + 2.760331630706787, + 4.286097049713135, + 2.7385997772216797, + 3.158334255218506, + 3.2378246784210205, + 2.3249711990356445, + 2.3011202812194824, + 2.7348926067352295, + 2.3449158668518066, + 3.0546181201934814, + 2.614961624145508 + ], + "clf": [] + }, + "n_targets": 40, + "random_top5": 0.002770083102493075 + }, + "LinearResponse+ranking": { + "top1": 0.9, + "top5": 0.95, + "ndcg": 0.9193426403608076, + "func_top5": 0.975, + "func_ndcg": 0.6613311093652607, + "rank": 2.2, + "endpoint_dist": 1.0196396365761757, + "spec_margin": 1.4613386183977126, + "off_dist": 2.4809782683849333, + "clf": null, + "_per": { + "top1": [ + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "top5": [ + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "ndcg": [ + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.0, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.3868528072341547, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.3868528072341547, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.0, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999 + ], + "func_top5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.4874697859644756, + 0.9999999999996392, + 0.5286633333094181, + 0.8196104814589247, + 0.9999999999996392, + 0.9999999999996392, + 0.9999999999996392, + 0.8893946731020875, + 0.4209088765950008, + 0.4745921515611962, + 0.3607790370814292, + 0.9999999999996392, + 0.6947009419882879, + 0.17973649046676787, + 0.5394675755622982, + 0.8266931349300584, + 0.689545205418837, + 0.7122606394323479, + 0.5721363965748284, + 0.6038700937941731, + 0.5776645452348329, + 0.7489233686198631, + 0.8286609472620452, + 0.5082527656822236, + 0.873309251116593, + 0.4948192717077505, + 0.4384685740390607, + 0.3607790370814292, + 0.7054249548886042, + 0.7515272698756076, + 0.9999999999996392, + 0.5887931728229896, + 0.4209088765950008, + 0.5929193762925764, + 0.580887827508521, + 0.17868853848086888, + 0.7589569849714464, + 0.6821957196755621, + 0.9456973440155002, + 0.6165377315019861 + ], + "rank": [ + 1, + 1, + 1, + 29, + 1, + 1, + 1, + 5, + 1, + 1, + 1, + 1, + 1, + 5, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 13, + 1, + 1, + 1, + 1 + ], + "endpoint_dist": [ + 0.6616038680076599, + 1.2548247575759888, + 0.7813584804534912, + 1.158394694328308, + 1.5520241260528564, + 0.923303484916687, + 1.76686429977417, + 0.8745992183685303, + 1.070665955543518, + 1.2548863887786865, + 0.9371170401573181, + 1.1795976161956787, + 1.0225927829742432, + 0.7508274912834167, + 1.0226606130599976, + 1.131973385810852, + 1.378308653831482, + 0.7616466879844666, + 0.7265413999557495, + 0.8877161741256714, + 0.6296344995498657, + 1.0646913051605225, + 1.0984606742858887, + 0.8197231292724609, + 1.1387754678726196, + 0.7523069381713867, + 0.8117942214012146, + 1.0146288871765137, + 1.2389315366744995, + 0.7988754510879517, + 1.6972215175628662, + 1.050825834274292, + 1.1287428140640259, + 1.0291059017181396, + 0.812923789024353, + 1.0367337465286255, + 0.8561542630195618, + 0.7039358019828796, + 0.7540229558944702, + 1.2505896091461182 + ], + "spec_margin": [ + 1.7379565238952637, + 0.9655307531356812, + 1.852024793624878, + 2.293393135070801, + 0.8486089706420898, + 1.3431891202926636, + 1.0528581142425537, + 1.4652132987976074, + 1.4513288736343384, + 1.3505439758300781, + 1.6941022872924805, + 1.6614453792572021, + 1.5319406986236572, + 1.5675017833709717, + 0.7009553909301758, + 1.4009870290756226, + 1.2738226652145386, + 2.199557304382324, + 1.1724765300750732, + 1.6043356657028198, + 2.161515235900879, + 1.0891995429992676, + 1.498476505279541, + 1.4074206352233887, + 1.2639492750167847, + 2.045914888381958, + 2.110194206237793, + 1.5556697845458984, + 1.3904625177383423, + 1.3211067914962769, + 1.302931785583496, + 0.9997358322143555, + 1.7146793603897095, + 1.830432653427124, + 1.124277949333191, + 1.097406268119812, + 1.4047913551330566, + 1.422738790512085, + 1.796676754951477, + 0.748192310333252 + ], + "off_dist": [ + 2.3995604515075684, + 2.22035551071167, + 2.633383274078369, + 3.4517879486083984, + 2.4006330966949463, + 2.2664926052093506, + 2.8197224140167236, + 2.3398125171661377, + 2.5219948291778564, + 2.6054303646087646, + 2.6312193870544434, + 2.841042995452881, + 2.5545334815979004, + 2.318329334259033, + 1.7236160039901733, + 2.5329604148864746, + 2.6521313190460205, + 2.9612040519714355, + 1.8990179300308228, + 2.492051839828491, + 2.791149854660034, + 2.15389084815979, + 2.5969371795654297, + 2.2271437644958496, + 2.4027247428894043, + 2.7982218265533447, + 2.9219884872436523, + 2.570298671722412, + 2.629394054412842, + 2.1199822425842285, + 3.0001533031463623, + 2.0505616664886475, + 2.8434221744537354, + 2.8595385551452637, + 1.937201738357544, + 2.1341400146484375, + 2.2609455585479736, + 2.1266746520996094, + 2.5506997108459473, + 1.9987819194793701 + ], + "clf": [] + }, + "n_targets": 40, + "random_top5": 0.002770083102493075 + }, + "NearestPerturbationCentroid+ranking": { + "top1": 0.95, + "top5": 0.975, + "ndcg": 0.9657732438383206, + "func_top5": 1.0, + "func_ndcg": 0.6652048246826138, + "rank": 1.675, + "endpoint_dist": 0.8411503508687019, + "spec_margin": 2.1007207214832304, + "off_dist": 2.9418710708618163, + "clf": null, + "_per": { + "top1": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "top5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "ndcg": [ + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.6309297535708266, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.0, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999 + ], + "func_top5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.3607790370814292, + 0.9999999999996392, + 0.4129233019279683, + 0.9999999999996392, + 0.9999999999996392, + 0.9999999999996392, + 0.9999999999996392, + 0.9456973440155002, + 0.4305632287245921, + 0.4745921515611962, + 0.5426118416977634, + 0.9999999999996392, + 0.4305632287245921, + 0.4646120878678205, + 0.5570272730063581, + 0.873309251116593, + 0.687334748339199, + 0.562183009575809, + 0.3607790370814292, + 0.5082527656822236, + 0.11381311447976694, + 0.6146996865129016, + 0.8654039058021243, + 0.7236207412564216, + 0.759496136636826, + 0.8909491779132166, + 0.4927712300231997, + 0.3607790370814292, + 0.7878767842395802, + 0.9223104630420079, + 0.9999999999996392, + 0.6553540821924644, + 0.5299596986814235, + 0.6734442887418696, + 0.3607790370814292, + 0.3607790370814292, + 0.5721363965748284, + 0.8455491977461236, + 0.9357439570164808, + 0.5314987088010797 + ], + "rank": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 27, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "endpoint_dist": [ + 0.7173737287521362, + 0.7461453080177307, + 0.785219132900238, + 0.538507878780365, + 1.097732424736023, + 0.6946150660514832, + 0.7801607251167297, + 0.7369425296783447, + 1.015824794769287, + 1.3046245574951172, + 1.0173161029815674, + 0.9735252857208252, + 1.092432975769043, + 0.8244563341140747, + 0.674659013748169, + 0.9909878373146057, + 0.9689846038818359, + 0.732428252696991, + 0.5732553005218506, + 0.6204644441604614, + 0.9914161562919617, + 0.7275379300117493, + 0.7922949194908142, + 0.8339179158210754, + 0.9220225214958191, + 0.8180479407310486, + 0.8987788558006287, + 0.7535209059715271, + 0.7320271730422974, + 1.037440299987793, + 1.0151782035827637, + 0.7500125169754028, + 0.8864849805831909, + 0.7386977672576904, + 0.9256577491760254, + 0.8188286423683167, + 0.6080302000045776, + 0.6138859391212463, + 0.9211081266403198, + 0.9754689931869507 + ], + "spec_margin": [ + 1.9674495458602905, + 2.103719711303711, + 2.245748996734619, + 3.573702335357666, + 2.0890731811523438, + 1.7624471187591553, + 3.303356885910034, + 1.8128483295440674, + 2.0443527698516846, + 2.065767288208008, + 1.6571438312530518, + 2.6439476013183594, + 2.032092571258545, + 1.785965085029602, + 1.3168447017669678, + 2.013680934906006, + 2.1790592670440674, + 2.5351104736328125, + 1.5227258205413818, + 2.132510185241699, + 1.9073677062988281, + 1.880821704864502, + 2.3598074913024902, + 1.7640490531921387, + 1.7445905208587646, + 2.3762378692626953, + 2.4540412425994873, + 2.107139825820923, + 2.3813529014587402, + 1.7228913307189941, + 3.270918846130371, + 1.9885872602462769, + 2.2718491554260254, + 2.49912691116333, + 1.3993134498596191, + 1.4822916984558105, + 2.1268625259399414, + 1.731029987335205, + 2.133510112762451, + 1.6394926309585571 + ], + "off_dist": [ + 2.6848232746124268, + 2.849864959716797, + 3.030968189239502, + 4.112210273742676, + 3.1868057250976562, + 2.457062244415283, + 4.083517551422119, + 2.549790859222412, + 3.0601775646209717, + 3.370391845703125, + 2.674459934234619, + 3.6174728870391846, + 3.124525547027588, + 2.6104214191436768, + 1.9915037155151367, + 3.004668712615967, + 3.1480438709259033, + 3.2675387859344482, + 2.0959811210632324, + 2.752974510192871, + 2.8987839221954346, + 2.6083595752716064, + 3.152102470397949, + 2.5979669094085693, + 2.6666131019592285, + 3.1942858695983887, + 3.3528201580047607, + 2.8606607913970947, + 3.113379955291748, + 2.760331630706787, + 4.286097049713135, + 2.7385997772216797, + 3.158334255218506, + 3.2378246784210205, + 2.3249711990356445, + 2.3011202812194824, + 2.7348926067352295, + 2.3449158668518066, + 3.0546181201934814, + 2.614961624145508 + ], + "clf": [] + }, + "n_targets": 40, + "random_top5": 0.002770083102493075 + }, + "EndpointMLP+ranking": { + "top1": 0.95, + "top5": 0.975, + "ndcg": 0.9657732438383206, + "func_top5": 1.0, + "func_ndcg": 0.6643281414589618, + "rank": 1.6, + "endpoint_dist": 0.8406605675816536, + "spec_margin": 2.099722996354103, + "off_dist": 2.9403835624456405, + "clf": null, + "_per": { + "top1": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "top5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "ndcg": [ + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.6309297535708266, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.0, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999, + 0.9999999999989999 + ], + "func_top5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.3607790370814292, + 0.9999999999996392, + 0.4129233019279683, + 0.9999999999996392, + 0.9999999999996392, + 0.9999999999996392, + 0.9999999999996392, + 0.9456973440155002, + 0.4305632287245921, + 0.4745921515611962, + 0.5426118416977634, + 0.9999999999996392, + 0.4305632287245921, + 0.4646120878678205, + 0.5570272730063581, + 0.873309251116593, + 0.691460951808786, + 0.5654062918494971, + 0.3607790370814292, + 0.5082527656822236, + 0.11381311447976694, + 0.5548688818687583, + 0.8621806235284363, + 0.7277469447260085, + 0.7650242852968305, + 0.8909491779132166, + 0.50788035359167, + 0.3607790370814292, + 0.7878767842395802, + 0.9223104630420079, + 0.9999999999996392, + 0.6553540821924644, + 0.5299596986814235, + 0.6693180852722828, + 0.3607790370814292, + 0.3607790370814292, + 0.5721363965748284, + 0.8455491977461236, + 0.9357439570164808, + 0.5314987088010797 + ], + "rank": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 24, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "endpoint_dist": [ + 0.7115392684936523, + 0.7434130311012268, + 0.782884955406189, + 0.5349298119544983, + 1.1008118391036987, + 0.6904085278511047, + 0.7770178914070129, + 0.7384058237075806, + 1.0147595405578613, + 1.302067756652832, + 1.015263557434082, + 0.9738177061080933, + 1.0930898189544678, + 0.8314464092254639, + 0.6720605492591858, + 0.9890942573547363, + 0.9730018973350525, + 0.7305974960327148, + 0.57706218957901, + 0.6231805682182312, + 0.989425539970398, + 0.7258337736129761, + 0.789780855178833, + 0.8334861993789673, + 0.9197162985801697, + 0.8218328356742859, + 0.9016136527061462, + 0.7549859285354614, + 0.7306838631629944, + 1.038749098777771, + 1.0174500942230225, + 0.74358731508255, + 0.8858193159103394, + 0.7361005544662476, + 0.9223771095275879, + 0.824489951133728, + 0.6082383394241333, + 0.6169683933258057, + 0.9182528257369995, + 0.9721778631210327 + ], + "spec_margin": [ + 1.9711594581604004, + 2.0922412872314453, + 2.2475204467773438, + 3.5705361366271973, + 2.0842247009277344, + 1.7688353061676025, + 3.3032357692718506, + 1.8071125745773315, + 2.0441155433654785, + 2.0665674209594727, + 1.657548427581787, + 2.640410900115967, + 2.03220272064209, + 1.7836847305297852, + 1.3169708251953125, + 2.016561508178711, + 2.173065185546875, + 2.5372564792633057, + 1.520702600479126, + 2.132035732269287, + 1.905267596244812, + 1.8801466226577759, + 2.3641176223754883, + 1.7644177675247192, + 1.7468843460083008, + 2.3724145889282227, + 2.451648235321045, + 2.107424259185791, + 2.3798720836639404, + 1.7198468446731567, + 3.2619473934173584, + 1.9915785789489746, + 2.274409770965576, + 2.5012764930725098, + 1.3998422622680664, + 1.4798988103866577, + 2.1273646354675293, + 1.7228996753692627, + 2.1340837478637695, + 1.637590765953064 + ], + "off_dist": [ + 2.6826987266540527, + 2.8356542587280273, + 3.030405282974243, + 4.105465888977051, + 3.1850366592407227, + 2.4592437744140625, + 4.080253601074219, + 2.545518398284912, + 3.05887508392334, + 3.3686351776123047, + 2.672811985015869, + 3.6142284870147705, + 3.1252925395965576, + 2.615131139755249, + 1.989031434059143, + 3.0056557655334473, + 3.1460671424865723, + 3.2678539752960205, + 2.097764730453491, + 2.755216360092163, + 2.89469313621521, + 2.605980396270752, + 3.1538984775543213, + 2.5979039669036865, + 2.6666007041931152, + 3.1942474842071533, + 3.353261947631836, + 2.862410068511963, + 3.11055588722229, + 2.7585959434509277, + 4.279397487640381, + 2.73516583442688, + 3.160229206085205, + 3.237377166748047, + 2.3222193717956543, + 2.3043887615203857, + 2.735602855682373, + 2.3398680686950684, + 3.0523366928100586, + 2.6097686290740967 + ], + "clf": [] + }, + "n_targets": 40, + "random_top5": 0.002770083102493075 + }, + "Random+ranking": { + "top1": 0.0, + "top5": 0.0, + "ndcg": 0.0, + "func_top5": 0.6, + "func_ndcg": 0.31792464420168004, + "rank": 820.225, + "endpoint_dist": 3.069971165060997, + "spec_margin": 0.3528890937566757, + "off_dist": 3.4228602588176726, + "clf": null, + "_per": { + "top1": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "top5": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "ndcg": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "func_top5": [ + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 0.0, + 1.0, + 1.0, + 0.0, + 1.0, + 0.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 0.0 + ], + "func_ndcg": [ + 0.0, + 0.7419209445012933, + 0.16595737932630603, + 0.18682058839661783, + 0.6409219429780558, + 0.6433793689850839, + 0.5679779905079548, + 0.6817911049877218, + 0.0, + 0.0, + 0.0, + 0.4637557067110293, + 0.0, + 0.0, + 0.0, + 0.44771831148081165, + 0.7653078254747857, + 0.06978419164316285, + 0.0, + 0.672509164948684, + 0.0, + 0.4524003745215921, + 0.5817105193518559, + 0.5790911234046386, + 0.6488908097392743, + 0.6409219429780558, + 0.0, + 0.6929197325758784, + 0.0, + 0.6850143872614097, + 0.3901923732229845, + 0.0, + 0.6772714573420283, + 0.05430265598413898, + 0.07768953695763149, + 0.0, + 0.0, + 0.5606285047646798, + 0.6281078300215263, + 0.0 + ], + "rank": [ + 1484, + 86, + 12, + 728, + 1567, + 428, + 72, + 141, + 744, + 1494, + 1327, + 147, + 994, + 366, + 1309, + 215, + 65, + 372, + 1661, + 1425, + 759, + 62, + 241, + 1139, + 702, + 1572, + 727, + 790, + 1133, + 1070, + 991, + 811, + 156, + 1589, + 927, + 578, + 1158, + 910, + 1428, + 1429 + ], + "endpoint_dist": [ + 1.2471455335617065, + 3.780824661254883, + 2.3386223316192627, + 3.5947601795196533, + 3.435229778289795, + 5.462389945983887, + 4.583520412445068, + 1.6975603103637695, + 6.2813029289245605, + 3.2201132774353027, + 1.954548954963684, + 2.568264961242676, + 4.240303993225098, + 1.9021812677383423, + 1.581223964691162, + 1.8886135816574097, + 2.0643303394317627, + 3.607733726501465, + 2.0478649139404297, + 2.4884705543518066, + 4.466367721557617, + 1.8336158990859985, + 3.0728282928466797, + 2.878474712371826, + 4.979918003082275, + 2.1764323711395264, + 8.584672927856445, + 1.425441026687622, + 3.742549419403076, + 1.836504340171814, + 3.9608829021453857, + 2.16227388381958, + 2.613037586212158, + 2.0466930866241455, + 2.479177236557007, + 4.7175116539001465, + 2.2013516426086426, + 1.6452608108520508, + 1.8277415037155151, + 4.1631059646606445 + ], + "spec_margin": [ + 1.2742022275924683, + -0.42791128158569336, + 1.129060983657837, + -1.119408369064331, + -0.7760045528411865, + 0.183258056640625, + -1.3876416683197021, + 0.876413106918335, + -0.03194618225097656, + -0.21651339530944824, + 1.0298165082931519, + 0.21366572380065918, + 0.578819751739502, + 1.1407452821731567, + 0.3463408946990967, + 0.5529183149337769, + 0.9577670097351074, + 1.0641913414001465, + 0.35391855239868164, + 0.5692179203033447, + 0.011257171630859375, + 0.668890118598938, + 0.20898771286010742, + 0.37965893745422363, + -0.10495424270629883, + 1.282132863998413, + -0.4226036071777344, + 1.4416706562042236, + 0.32228803634643555, + 0.3516949415206909, + -1.1902787685394287, + 0.18331551551818848, + 0.9950697422027588, + 1.1751482486724854, + 0.2742455005645752, + -0.3204960823059082, + 0.6251316070556641, + 0.9052252769470215, + 1.1004918813705444, + -0.08222198486328125 + ], + "off_dist": [ + 2.521347761154175, + 3.3529133796691895, + 3.4676833152770996, + 2.4753518104553223, + 2.6592252254486084, + 5.645648002624512, + 3.195878744125366, + 2.5739734172821045, + 6.249356746673584, + 3.0035998821258545, + 2.984365463256836, + 2.781930685043335, + 4.8191237449646, + 3.042926549911499, + 1.9275648593902588, + 2.4415318965911865, + 3.02209734916687, + 4.671925067901611, + 2.4017834663391113, + 3.0576884746551514, + 4.477624893188477, + 2.5025060176849365, + 3.281816005706787, + 3.25813364982605, + 4.874963760375977, + 3.4585652351379395, + 8.162069320678711, + 2.8671116828918457, + 4.064837455749512, + 2.188199281692505, + 2.770604133605957, + 2.3455893993377686, + 3.608107328414917, + 3.221841335296631, + 2.753422737121582, + 4.397015571594238, + 2.8264832496643066, + 2.5504860877990723, + 2.9282333850860596, + 4.080883979797363 + ], + "clf": [] + }, + "n_targets": 40, + "random_top5": 0.002770083102493075 + } + } +} \ No newline at end of file diff --git a/experiments/results/replogle_k562_nom_improved.json b/experiments/results/replogle_k562_nom_improved.json new file mode 100644 index 0000000000000000000000000000000000000000..bf87e3b300c0a0cefde1f65e452f82171577532a --- /dev/null +++ b/experiments/results/replogle_k562_nom_improved.json @@ -0,0 +1,981 @@ +{ + "dataset": "replogle_k562", + "split": "perturbation", + "reward": "cosine", + "n_candidates": 1805, + "n_targets": 120, + "random_top5": 0.002770083102493075, + "methods": { + "PIVOT-ranking": { + "top1": 0.0, + "top5": 0.0, + "ndcg": 0.0, + "func_top5": 0.7416666666666667, + "func_ndcg": 0.5503647310194125, + "top5_ci": [ + 0.0, + 0.0, + 0.0 + ], + "_top5_per": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "med_rank": 857.0 + }, + "PIVOT-guidance(random-init)": { + "top1": 0.0, + "top5": 0.0, + "ndcg": 0.0, + "func_top5": 0.7916666666666666, + "func_ndcg": 0.5487322808181787, + "top5_ci": [ + 0.0, + 0.0, + 0.0 + ], + "_top5_per": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "med_rank": 866.0 + }, + "EndpointMLP+ranking": { + "top1": 0.0, + "top5": 0.0, + "ndcg": 0.0, + "func_top5": 0.7416666666666667, + "func_ndcg": 0.5599157820889673, + "top5_ci": [ + 0.0, + 0.0, + 0.0 + ], + "_top5_per": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "med_rank": 761.0 + }, + "LinearResponse+ranking": { + "top1": 0.0, + "top5": 0.0, + "ndcg": 0.0, + "func_top5": 0.7416666666666667, + "func_ndcg": 0.558927740279396, + "top5_ci": [ + 0.0, + 0.0, + 0.0 + ], + "_top5_per": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "med_rank": 839.0 + }, + "Additive+ranking": { + "top1": 0.0, + "top5": 0.0, + "ndcg": 0.0, + "func_top5": 0.7416666666666667, + "func_ndcg": 0.5599157820889673, + "top5_ci": [ + 0.0, + 0.0, + 0.0 + ], + "_top5_per": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "med_rank": 844.5 + }, + "NearestPerturbationCentroid+ranking": { + "top1": 0.0, + "top5": 0.0, + "ndcg": 0.0, + "func_top5": 0.7416666666666667, + "func_ndcg": 0.5599157820889673, + "top5_ci": [ + 0.0, + 0.0, + 0.0 + ], + "_top5_per": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "med_rank": 790.0 + }, + "Random+ranking": { + "top1": 0.0, + "top5": 0.0, + "ndcg": 0.0, + "func_top5": 0.7083333333333334, + "func_ndcg": 0.3706553528381431, + "top5_ci": [ + 0.0, + 0.0, + 0.0 + ], + "_top5_per": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "med_rank": 955.0 + } + }, + "significance": { + "PIVOT-ranking_vs_PIVOT-guidance(random-init)": { + "p": 1.0, + "mean_diff_top5": 0.0 + }, + "PIVOT-ranking_vs_EndpointMLP+ranking": { + "p": 1.0, + "mean_diff_top5": 0.0 + }, + "PIVOT-ranking_vs_LinearResponse+ranking": { + "p": 1.0, + "mean_diff_top5": 0.0 + }, + "PIVOT-ranking_vs_Additive+ranking": { + "p": 1.0, + "mean_diff_top5": 0.0 + }, + "PIVOT-ranking_vs_NearestPerturbationCentroid+ranking": { + "p": 1.0, + "mean_diff_top5": 0.0 + }, + "PIVOT-ranking_vs_Random+ranking": { + "p": 1.0, + "mean_diff_top5": 0.0 + } + } +} \ No newline at end of file diff --git a/experiments/results/replogle_k562_nom_perturbation.json b/experiments/results/replogle_k562_nom_perturbation.json new file mode 100644 index 0000000000000000000000000000000000000000..38584f383f2c7c82a7cfdd169c0344989d281be3 --- /dev/null +++ b/experiments/results/replogle_k562_nom_perturbation.json @@ -0,0 +1,2773 @@ +{ + "split": "perturbation", + "reward": "centroid", + "n_candidates": 1805, + "n_targets": 40, + "methods": { + "PIVOT-ranking": { + "top1": 0.0, + "top5": 0.0, + "ndcg": 0.0, + "func_top5": 0.75, + "func_ndcg": 0.43398773723290346, + "rank": 477.075, + "endpoint_dist": 1.6277449652552605, + "spec_margin": 0.9393299654126167, + "off_dist": 2.5670749247074127, + "clf": null, + "_per": { + "top1": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "top5": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "ndcg": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "func_top5": [ + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.0, + 0.8196104814589247, + 0.25898197669424483, + 0.25807905549834614, + 0.7084012682349022, + 0.4144301094503377, + 0.39174687803411373, + 0.439764939411066, + 0.10905082208642256, + 0.8196104814589247, + 0.11443249549771052, + 0.5247884674204997, + 0.2974259152941696, + 0.3821659031256479, + 0.0, + 0.20400787375012425, + 0.8196104814589247, + 0.5221845661647552, + 0.8196104814589247, + 0.22479085346787228, + 0.8196104814589247, + 0.05690655723988347, + 0.07768953695763149, + 0.8196104814589247, + 0.06012983951357154, + 0.8196104814589247, + 0.8196104814589247, + 0.8196104814589247, + 0.8196104814589247, + 0.05430265598413898, + 0.5513465647256421, + 0.8196104814589247, + 0.0, + 0.29420263302048155, + 0.21688550815340366, + 0.8196104814589247, + 0.11703639675345502, + 0.0, + 0.6896964503021904, + 0.555736445028433 + ], + "rank": [ + 606, + 798, + 925, + 529, + 357, + 169, + 692, + 726, + 354, + 511, + 163, + 226, + 243, + 107, + 209, + 101, + 862, + 52, + 504, + 606, + 620, + 950, + 845, + 567, + 516, + 783, + 330, + 393, + 329, + 442, + 306, + 441, + 982, + 522, + 145, + 1016, + 656, + 370, + 92, + 38 + ], + "endpoint_dist": [ + 0.7434885501861572, + 2.4635772705078125, + 1.6023441553115845, + 0.8692498803138733, + 1.1865061521530151, + 1.5722774267196655, + 0.7362974286079407, + 0.9789136052131653, + 0.9780240654945374, + 6.6848225593566895, + 0.9765823483467102, + 1.9768712520599365, + 1.1871503591537476, + 1.01445472240448, + 1.8242679834365845, + 0.8309136033058167, + 2.4996466636657715, + 1.4547226428985596, + 1.2289146184921265, + 0.8584837317466736, + 2.255495071411133, + 0.9234660267829895, + 0.626628041267395, + 2.0583951473236084, + 0.983538806438446, + 1.82752525806427, + 1.7855417728424072, + 2.05389404296875, + 1.4342339038848877, + 1.1439824104309082, + 1.2706406116485596, + 2.8937065601348877, + 0.7645078301429749, + 1.3121113777160645, + 1.0582448244094849, + 5.560652256011963, + 0.9708915948867798, + 0.6884087920188904, + 1.975685715675354, + 1.8547395467758179 + ], + "spec_margin": [ + 1.672365427017212, + 0.7733314037322998, + 1.019189715385437, + 1.3633439540863037, + 0.8521960973739624, + 0.6960502862930298, + 1.4837749004364014, + 1.2703609466552734, + 1.397662878036499, + -2.9303510189056396, + 1.6186654567718506, + 0.47718214988708496, + 1.130790114402771, + 1.2825955152511597, + -0.15734422206878662, + 1.480198860168457, + 1.9295220375061035, + 1.5434458255767822, + 0.9457141160964966, + 1.6729440689086914, + 0.6493873596191406, + 0.8838562369346619, + 1.9253515005111694, + 0.3431985378265381, + 1.3820078372955322, + 0.7749754190444946, + 1.9326670169830322, + 0.5550122261047363, + 1.195478916168213, + 0.6238242387771606, + 1.2171826362609863, + 0.5463707447052002, + 1.9847569465637207, + 1.4454312324523926, + 0.9364702701568604, + -1.686753273010254, + 1.1488052606582642, + 1.5929045677185059, + 0.35722815990448, + 0.24340426921844482 + ], + "off_dist": [ + 2.415853977203369, + 3.2369086742401123, + 2.6215338706970215, + 2.2325937747955322, + 2.0387022495269775, + 2.2683277130126953, + 2.2200722694396973, + 2.249274492263794, + 2.3756868839263916, + 3.75447154045105, + 2.595247745513916, + 2.4540534019470215, + 2.3179404735565186, + 2.2970502376556396, + 1.6669237613677979, + 2.311112403869629, + 4.429168701171875, + 2.998168468475342, + 2.174628734588623, + 2.5314278602600098, + 2.9048824310302734, + 1.8073222637176514, + 2.5519795417785645, + 2.4015936851501465, + 2.365546703338623, + 2.6025006771087646, + 3.7182087898254395, + 2.6089062690734863, + 2.6297128200531006, + 1.7678066492080688, + 2.487823247909546, + 3.440077304840088, + 2.749264717102051, + 2.757542610168457, + 1.9947150945663452, + 3.873898983001709, + 2.119696855545044, + 2.281313419342041, + 2.332913875579834, + 2.0981438159942627 + ], + "clf": [] + }, + "n_targets": 40, + "random_top5": 0.002770083102493075 + }, + "PIVOT-guidance": { + "top1": 0.0, + "top5": 0.0, + "ndcg": 0.0, + "func_top5": 0.85, + "func_ndcg": 0.346193945477374, + "rank": 479.3, + "endpoint_dist": 1.6929722726345062, + "spec_margin": 0.8205313935875893, + "off_dist": 2.513503661751747, + "clf": null, + "_per": { + "top1": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "top5": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "ndcg": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "func_top5": [ + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.06012983951357154, + 0.7594806419453533, + 0.1423390241168964, + 0.1803895185407146, + 0.4371610381553215, + 0.6392209629182101, + 0.1803895185407146, + 0.18996364131774213, + 0.2557586944205568, + 0.8196104814589247, + 0.1298338018041706, + 0.3673613518206858, + 0.1803895185407146, + 0.2372960757805981, + 0.0, + 0.052144264846539085, + 0.8196104814589247, + 0.24364691628393753, + 0.4482741710520053, + 0.33392190490685864, + 0.8196104814589247, + 0.0, + 0.07768953695763149, + 0.46531021152215846, + 0.20659506710005476, + 0.6721367528581305, + 0.5759635651749873, + 0.8196104814589247, + 0.6229688013578893, + 0.12991403115673436, + 0.2302134223094644, + 0.7553544384757664, + 0.18421668714087336, + 0.1803895185407146, + 0.3130586958365468, + 0.8196104814589247, + 0.0, + 0.0, + 0.2346921745248536, + 0.26350162433993823 + ], + "rank": [ + 608, + 800, + 926, + 531, + 360, + 169, + 694, + 727, + 357, + 511, + 166, + 234, + 245, + 111, + 212, + 102, + 862, + 57, + 509, + 606, + 621, + 950, + 845, + 569, + 518, + 783, + 331, + 397, + 334, + 443, + 307, + 448, + 982, + 522, + 149, + 1016, + 657, + 370, + 99, + 44 + ], + "endpoint_dist": [ + 0.7434885501861572, + 2.4635772705078125, + 1.6023441553115845, + 0.8692498803138733, + 1.1865061521530151, + 1.5722774267196655, + 0.7362974286079407, + 1.1201238632202148, + 0.9780240654945374, + 6.6848225593566895, + 0.9765823483467102, + 1.9768712520599365, + 1.1871503591537476, + 1.01445472240448, + 1.8242679834365845, + 0.8309136033058167, + 2.4996466636657715, + 1.4547226428985596, + 1.7815905809402466, + 0.8584837317466736, + 2.255495071411133, + 0.9234660267829895, + 0.626628041267395, + 2.4905507564544678, + 0.983538806438446, + 1.82752525806427, + 2.1815366744995117, + 2.05389404296875, + 1.4342339038848877, + 1.1439824104309082, + 1.4812977313995361, + 3.3611063957214355, + 0.7645078301429749, + 1.4217694997787476, + 1.0582448244094849, + 5.560652256011963, + 0.9708915948867798, + 0.6884087920188904, + 1.9726204872131348, + 2.1571452617645264 + ], + "spec_margin": [ + 1.672365427017212, + 0.7733314037322998, + 1.019189715385437, + 1.3633439540863037, + 0.8521960973739624, + 0.6960502862930298, + 1.4837749004364014, + 1.1176819801330566, + 1.397662878036499, + -2.9303510189056396, + 1.6186654567718506, + 0.47718214988708496, + 1.130790114402771, + 1.2825955152511597, + -0.15734422206878662, + 1.480198860168457, + 1.9295220375061035, + 1.5434458255767822, + 0.0906752347946167, + 1.6729440689086914, + 0.6493873596191406, + 0.8838562369346619, + 1.9253515005111694, + -0.4390888214111328, + 1.3820078372955322, + 0.7749754190444946, + 1.1829278469085693, + 0.5550122261047363, + 1.195478916168213, + 0.6238242387771606, + 0.9619626998901367, + -0.876856803894043, + 1.9847569465637207, + 1.38202965259552, + 0.9364702701568604, + -1.686753273010254, + 1.1488052606582642, + 1.5929045677185059, + 0.423114538192749, + -0.29283154010772705 + ], + "off_dist": [ + 2.415853977203369, + 3.2369086742401123, + 2.6215338706970215, + 2.2325937747955322, + 2.0387022495269775, + 2.2683277130126953, + 2.2200722694396973, + 2.2378058433532715, + 2.3756868839263916, + 3.75447154045105, + 2.595247745513916, + 2.4540534019470215, + 2.3179404735565186, + 2.2970502376556396, + 1.6669237613677979, + 2.311112403869629, + 4.429168701171875, + 2.998168468475342, + 1.8722658157348633, + 2.5314278602600098, + 2.9048824310302734, + 1.8073222637176514, + 2.5519795417785645, + 2.051461935043335, + 2.365546703338623, + 2.6025006771087646, + 3.364464521408081, + 2.6089062690734863, + 2.6297128200531006, + 1.7678066492080688, + 2.443260431289673, + 2.4842495918273926, + 2.749264717102051, + 2.8037991523742676, + 1.9947150945663452, + 3.873898983001709, + 2.119696855545044, + 2.281313419342041, + 2.395735025405884, + 1.8643137216567993 + ], + "clf": [] + }, + "n_targets": 40, + "random_top5": 0.002770083102493075 + }, + "Additive+ranking": { + "top1": 0.0, + "top5": 0.0, + "ndcg": 0.0, + "func_top5": 0.925, + "func_ndcg": 0.5289588692976046, + "rank": 449.3, + "endpoint_dist": 1.4835666730999946, + "spec_margin": 1.5093856036663056, + "off_dist": 2.9929522782564164, + "clf": null, + "_per": { + "top1": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "top5": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "ndcg": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "func_top5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.18618449947286028, + 0.8196104814589247, + 0.5093871611140395, + 0.11381311447976694, + 0.6896964503021904, + 0.499202299879103, + 0.05430265598413898, + 0.8196104814589247, + 0.17623111247384093, + 0.8196104814589247, + 0.6149832266908569, + 0.7294157221885674, + 0.501433789044294, + 0.5791233260019254, + 0.25807905549834614, + 0.0, + 0.8196104814589247, + 0.7419209445012933, + 0.7594806419453533, + 0.6053288745612656, + 0.8196104814589247, + 0.07768953695763149, + 0.29420263302048155, + 0.7627039242190412, + 0.4883727071603744, + 0.8196104814589247, + 0.8196104814589247, + 0.8196104814589247, + 0.8196104814589247, + 0.3102233203448852, + 0.6353937943180514, + 0.8196104814589247, + 0.4889118588257541, + 0.16811577046390594, + 0.1841364577883096, + 0.8196104814589247, + 0.05430265598413898, + 0.07768953695763149, + 0.8196104814589247, + 0.7627039242190412 + ], + "rank": [ + 464, + 436, + 562, + 558, + 459, + 559, + 230, + 487, + 547, + 610, + 471, + 436, + 277, + 225, + 343, + 327, + 609, + 382, + 505, + 798, + 192, + 419, + 423, + 571, + 474, + 338, + 357, + 314, + 355, + 454, + 362, + 375, + 649, + 794, + 404, + 681, + 300, + 307, + 556, + 362 + ], + "endpoint_dist": [ + 0.7910816073417664, + 2.325538396835327, + 1.4062681198120117, + 0.8322772979736328, + 1.2257609367370605, + 1.4010910987854004, + 0.7616842985153198, + 0.8932631611824036, + 0.8577340245246887, + 4.744374752044678, + 0.7894836068153381, + 1.8425379991531372, + 1.1874315738677979, + 0.978216826915741, + 1.4649451971054077, + 0.8955826163291931, + 2.891011953353882, + 1.1721434593200684, + 1.2098002433776855, + 0.8302912712097168, + 1.8107973337173462, + 0.8613649606704712, + 0.878169596195221, + 1.9043508768081665, + 0.9688758850097656, + 1.9366815090179443, + 2.2544405460357666, + 1.667983889579773, + 1.1924028396606445, + 1.242406964302063, + 1.270032525062561, + 2.7028005123138428, + 0.6736534237861633, + 1.0901234149932861, + 0.9097232222557068, + 3.980757474899292, + 0.8556053638458252, + 0.8449692726135254, + 1.8862968683242798, + 1.9107120037078857 + ], + "spec_margin": [ + 1.7232837677001953, + 1.9759433269500732, + 1.599470615386963, + 1.453108549118042, + 1.0649609565734863, + 1.2389452457427979, + 1.5532299280166626, + 1.4933724403381348, + 1.6919171810150146, + 1.1277027130126953, + 1.839672565460205, + 0.97779381275177, + 1.3402292728424072, + 1.3308873176574707, + 0.6002246141433716, + 1.4666364192962646, + 1.1187450885772705, + 1.9536728858947754, + 1.3608942031860352, + 1.896925926208496, + 2.1419949531555176, + 1.0486434698104858, + 1.7888805866241455, + 0.994531512260437, + 1.54184889793396, + 2.2699358463287354, + 1.7643601894378662, + 1.9719387292861938, + 1.732537031173706, + 0.6122345924377441, + 1.593953013420105, + 2.815749406814575, + 2.2328741550445557, + 1.8885915279388428, + 1.112678050994873, + 1.611879587173462, + 1.4536590576171875, + 1.4404144287109375, + 1.1916362047195435, + 0.3594660758972168 + ], + "off_dist": [ + 2.5143654346466064, + 4.3014817237854, + 3.0057387351989746, + 2.285385847091675, + 2.290721893310547, + 2.6400363445281982, + 2.3149142265319824, + 2.3866355419158936, + 2.5496511459350586, + 5.872077465057373, + 2.6291561126708984, + 2.8203318119049072, + 2.527660846710205, + 2.3091042041778564, + 2.0651698112487793, + 2.3622190952301025, + 4.009757041931152, + 3.1258163452148438, + 2.5706944465637207, + 2.727217197418213, + 3.9527924060821533, + 1.910008430480957, + 2.6670501232147217, + 2.8988823890686035, + 2.5107247829437256, + 4.20661735534668, + 4.018800735473633, + 3.639922618865967, + 2.9249398708343506, + 1.8546415567398071, + 2.863985538482666, + 5.518549919128418, + 2.906527519226074, + 2.978714942932129, + 2.0224013328552246, + 5.592637062072754, + 2.3092644214630127, + 2.285383701324463, + 3.0779330730438232, + 2.2701780796051025 + ], + "clf": [] + }, + "n_targets": 40, + "random_top5": 0.002770083102493075 + }, + "LinearResponse+ranking": { + "top1": 0.0, + "top5": 0.0, + "ndcg": 0.0, + "func_top5": 0.925, + "func_ndcg": 0.5275197617991798, + "rank": 594.75, + "endpoint_dist": 1.5074330180883408, + "spec_margin": 1.0903448686003685, + "off_dist": 2.597777882218361, + "clf": null, + "_per": { + "top1": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "top5": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "ndcg": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "func_top5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.4361482129612715, + 0.8196104814589247, + 0.5051807282918888, + 0.23253378338725372, + 0.7594806419453533, + 0.5967465448927352, + 0.32749083505095544, + 0.5749649199350518, + 0.16335347807056153, + 0.8196104814589247, + 0.17394295399333848, + 0.8196104814589247, + 0.23497571470280879, + 0.5222167687620419, + 0.24464556152387298, + 0.0, + 0.8196104814589247, + 0.7627039242190412, + 0.8196104814589247, + 0.5586960504689169, + 0.8196104814589247, + 0.24464556152387298, + 0.22608721883987756, + 0.8196104814589247, + 0.11443249549771052, + 0.8196104814589247, + 0.8196104814589247, + 0.8196104814589247, + 0.8196104814589247, + 0.496278052474843, + 0.6850143872614097, + 0.7553544384757664, + 0.3225665727174216, + 0.2372960757805981, + 0.1707196717196504, + 0.8196104814589247, + 0.11381311447976694, + 0.24464556152387298, + 0.8196104814589247, + 0.7419209445012933 + ], + "rank": [ + 819, + 613, + 620, + 584, + 560, + 880, + 718, + 315, + 622, + 761, + 684, + 471, + 801, + 768, + 740, + 776, + 526, + 228, + 111, + 798, + 497, + 970, + 893, + 548, + 626, + 196, + 270, + 534, + 586, + 457, + 264, + 361, + 782, + 819, + 443, + 553, + 767, + 905, + 357, + 567 + ], + "endpoint_dist": [ + 0.7317129373550415, + 1.920383334159851, + 1.4073760509490967, + 0.7570603489875793, + 1.17207670211792, + 1.4544312953948975, + 0.6235144734382629, + 0.8932536244392395, + 0.8369724750518799, + 6.552571773529053, + 0.6789464950561523, + 1.666611671447754, + 1.142364263534546, + 0.9602575898170471, + 1.5338935852050781, + 0.9362837076187134, + 3.332012891769409, + 1.1075212955474854, + 1.127482533454895, + 1.1178308725357056, + 2.0792980194091797, + 0.9287006258964539, + 0.8524771332740784, + 2.000643253326416, + 0.7920212149620056, + 1.2070013284683228, + 2.0071728229522705, + 1.4477404356002808, + 1.1261117458343506, + 1.2170510292053223, + 1.068366527557373, + 2.0998857021331787, + 0.9043663740158081, + 1.2781434059143066, + 0.8842405676841736, + 4.939247131347656, + 0.9376744031906128, + 0.9116864204406738, + 1.769436240196228, + 1.893498420715332 + ], + "spec_margin": [ + 1.6089192628860474, + 1.5517617464065552, + 1.3225722312927246, + 1.4850592613220215, + 0.9671440124511719, + 0.9703366756439209, + 1.6035497188568115, + 1.4796998500823975, + 1.617544412612915, + -3.017054319381714, + 1.8496487140655518, + 0.9044928550720215, + 1.2502381801605225, + 1.2019782066345215, + 0.25848865509033203, + 1.374677062034607, + 0.3797588348388672, + 1.9601185321807861, + 0.9472290277481079, + 1.3767625093460083, + 0.8319146633148193, + 0.9684036374092102, + 1.6929290294647217, + 0.2231457233428955, + 1.6079893112182617, + 1.8344088792800903, + 1.7327463626861572, + 1.6003540754318237, + 1.855177640914917, + 0.546280026435852, + 1.5585684776306152, + 1.4500024318695068, + 1.7879267930984497, + 1.571855068206787, + 1.071916103363037, + -1.3561031818389893, + 1.2547610998153687, + 1.2583446502685547, + 0.7246888875961304, + 0.3055596351623535 + ], + "off_dist": [ + 2.340632200241089, + 3.4721450805664062, + 2.7299482822418213, + 2.242119550704956, + 2.139220714569092, + 2.4247679710388184, + 2.2270641326904297, + 2.372953414916992, + 2.454516887664795, + 3.535517454147339, + 2.528595209121704, + 2.5711045265197754, + 2.3926024436950684, + 2.162235736846924, + 1.7923822402954102, + 2.3109607696533203, + 3.7117717266082764, + 3.0676398277282715, + 2.074711561203003, + 2.494593381881714, + 2.911212682723999, + 1.897104263305664, + 2.5454061031341553, + 2.2237889766693115, + 2.400010585784912, + 3.041410207748413, + 3.7399191856384277, + 3.0480945110321045, + 2.9812893867492676, + 1.7633310556411743, + 2.6269350051879883, + 3.5498881340026855, + 2.692293167114258, + 2.8499984741210938, + 1.9561567306518555, + 3.583143949508667, + 2.1924355030059814, + 2.1700310707092285, + 2.4941251277923584, + 2.1990580558776855 + ], + "clf": [] + }, + "n_targets": 40, + "random_top5": 0.002770083102493075 + }, + "NearestPerturbationCentroid+ranking": { + "top1": 0.0, + "top5": 0.0, + "ndcg": 0.0, + "func_top5": 0.925, + "func_ndcg": 0.526194541190098, + "rank": 1014.875, + "endpoint_dist": 1.4835666730999946, + "spec_margin": 1.5093856036663056, + "off_dist": 2.9929522782564164, + "clf": null, + "_per": { + "top1": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "top5": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "ndcg": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "func_top5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.18618449947286028, + 0.8196104814589247, + 0.5093871611140395, + 0.11381311447976694, + 0.6896964503021904, + 0.499202299879103, + 0.05430265598413898, + 0.6489075176450882, + 0.17623111247384093, + 0.8196104814589247, + 0.6149832266908569, + 0.7294157221885674, + 0.501433789044294, + 0.5791233260019254, + 0.25807905549834614, + 0.0, + 0.8196104814589247, + 0.7419209445012933, + 0.8196104814589247, + 0.6053288745612656, + 0.8196104814589247, + 0.07768953695763149, + 0.29420263302048155, + 0.7627039242190412, + 0.4883727071603744, + 0.8196104814589247, + 0.8196104814589247, + 0.8196104814589247, + 0.8196104814589247, + 0.3102233203448852, + 0.6353937943180514, + 0.8196104814589247, + 0.4889118588257541, + 0.16811577046390594, + 0.1841364577883096, + 0.8196104814589247, + 0.05430265598413898, + 0.07768953695763149, + 0.8196104814589247, + 0.7627039242190412 + ], + "rank": [ + 955, + 433, + 951, + 1202, + 995, + 1204, + 1163, + 890, + 934, + 1064, + 1213, + 844, + 965, + 1303, + 1114, + 1310, + 453, + 1008, + 1363, + 1025, + 593, + 1043, + 902, + 1424, + 798, + 1000, + 1059, + 977, + 923, + 908, + 868, + 1331, + 829, + 975, + 925, + 1431, + 1012, + 1303, + 899, + 1006 + ], + "endpoint_dist": [ + 0.7910816073417664, + 2.325538396835327, + 1.4062681198120117, + 0.8322772979736328, + 1.2257609367370605, + 1.4010910987854004, + 0.7616842985153198, + 0.8932631611824036, + 0.8577340245246887, + 4.744374752044678, + 0.7894836068153381, + 1.8425379991531372, + 1.1874315738677979, + 0.978216826915741, + 1.4649451971054077, + 0.8955826163291931, + 2.891011953353882, + 1.1721434593200684, + 1.2098002433776855, + 0.8302912712097168, + 1.8107973337173462, + 0.8613649606704712, + 0.878169596195221, + 1.9043508768081665, + 0.9688758850097656, + 1.9366815090179443, + 2.2544405460357666, + 1.667983889579773, + 1.1924028396606445, + 1.242406964302063, + 1.270032525062561, + 2.7028005123138428, + 0.6736534237861633, + 1.0901234149932861, + 0.9097232222557068, + 3.980757474899292, + 0.8556053638458252, + 0.8449692726135254, + 1.8862968683242798, + 1.9107120037078857 + ], + "spec_margin": [ + 1.7232837677001953, + 1.9759433269500732, + 1.599470615386963, + 1.453108549118042, + 1.0649609565734863, + 1.2389452457427979, + 1.5532299280166626, + 1.4933724403381348, + 1.6919171810150146, + 1.1277027130126953, + 1.839672565460205, + 0.97779381275177, + 1.3402292728424072, + 1.3308873176574707, + 0.6002246141433716, + 1.4666364192962646, + 1.1187450885772705, + 1.9536728858947754, + 1.3608942031860352, + 1.896925926208496, + 2.1419949531555176, + 1.0486434698104858, + 1.7888805866241455, + 0.994531512260437, + 1.54184889793396, + 2.2699358463287354, + 1.7643601894378662, + 1.9719387292861938, + 1.732537031173706, + 0.6122345924377441, + 1.593953013420105, + 2.815749406814575, + 2.2328741550445557, + 1.8885915279388428, + 1.112678050994873, + 1.611879587173462, + 1.4536590576171875, + 1.4404144287109375, + 1.1916362047195435, + 0.3594660758972168 + ], + "off_dist": [ + 2.5143654346466064, + 4.3014817237854, + 3.0057387351989746, + 2.285385847091675, + 2.290721893310547, + 2.6400363445281982, + 2.3149142265319824, + 2.3866355419158936, + 2.5496511459350586, + 5.872077465057373, + 2.6291561126708984, + 2.8203318119049072, + 2.527660846710205, + 2.3091042041778564, + 2.0651698112487793, + 2.3622190952301025, + 4.009757041931152, + 3.1258163452148438, + 2.5706944465637207, + 2.727217197418213, + 3.9527924060821533, + 1.910008430480957, + 2.6670501232147217, + 2.8988823890686035, + 2.5107247829437256, + 4.20661735534668, + 4.018800735473633, + 3.639922618865967, + 2.9249398708343506, + 1.8546415567398071, + 2.863985538482666, + 5.518549919128418, + 2.906527519226074, + 2.978714942932129, + 2.0224013328552246, + 5.592637062072754, + 2.3092644214630127, + 2.285383701324463, + 3.0779330730438232, + 2.2701780796051025 + ], + "clf": [] + }, + "n_targets": 40, + "random_top5": 0.002770083102493075 + }, + "EndpointMLP+ranking": { + "top1": 0.0, + "top5": 0.0, + "ndcg": 0.0, + "func_top5": 0.925, + "func_ndcg": 0.5220999841830257, + "rank": 507.55, + "endpoint_dist": 1.4828851819038391, + "spec_margin": 1.5101026505231858, + "off_dist": 2.992987832427025, + "clf": null, + "_per": { + "top1": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "top5": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "ndcg": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "func_top5": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.19408984478732896, + 0.8196104814589247, + 0.5093871611140395, + 0.11381311447976694, + 0.6896964503021904, + 0.499202299879103, + 0.05430265598413898, + 0.6489075176450882, + 0.17623111247384093, + 0.8196104814589247, + 0.6149832266908569, + 0.7294157221885674, + 0.501433789044294, + 0.5791233260019254, + 0.25807905549834614, + 0.0, + 0.8196104814589247, + 0.7419209445012933, + 0.7032101736292273, + 0.6053288745612656, + 0.8196104814589247, + 0.07768953695763149, + 0.29420263302048155, + 0.7627039242190412, + 0.4883727071603744, + 0.8196104814589247, + 0.8196104814589247, + 0.8196104814589247, + 0.8196104814589247, + 0.3023179750304166, + 0.6375521854556513, + 0.8196104814589247, + 0.4889118588257541, + 0.16811577046390594, + 0.13199219294177048, + 0.8196104814589247, + 0.05430265598413898, + 0.07768953695763149, + 0.8196104814589247, + 0.7653078254747857 + ], + "rank": [ + 501, + 456, + 793, + 491, + 602, + 722, + 460, + 441, + 761, + 211, + 610, + 410, + 740, + 596, + 519, + 791, + 389, + 395, + 196, + 730, + 459, + 682, + 571, + 239, + 317, + 328, + 438, + 378, + 369, + 330, + 458, + 332, + 616, + 657, + 524, + 246, + 977, + 616, + 596, + 355 + ], + "endpoint_dist": [ + 0.788127601146698, + 2.3236305713653564, + 1.4040254354476929, + 0.8310549259185791, + 1.2236676216125488, + 1.3991789817810059, + 0.7625831365585327, + 0.8916113376617432, + 0.8571329712867737, + 4.743624210357666, + 0.7842857241630554, + 1.8409243822097778, + 1.1889394521713257, + 0.9782581925392151, + 1.4625096321105957, + 0.898481011390686, + 2.8906986713409424, + 1.1717466115951538, + 1.2101223468780518, + 0.8264373540878296, + 1.8072891235351562, + 0.8618491291999817, + 0.8796766400337219, + 1.9048603773117065, + 0.9663487076759338, + 1.9356915950775146, + 2.260815382003784, + 1.6731778383255005, + 1.1898635625839233, + 1.2420670986175537, + 1.268711805343628, + 2.697918176651001, + 0.6752347946166992, + 1.0929676294326782, + 0.906967282295227, + 3.9801554679870605, + 0.8546260595321655, + 0.846272885799408, + 1.8830167055130005, + 1.91085684299469 + ], + "spec_margin": [ + 1.7244186401367188, + 1.9791104793548584, + 1.6007493734359741, + 1.454641580581665, + 1.0647914409637451, + 1.241217851638794, + 1.5519427061080933, + 1.4942684173583984, + 1.6921191215515137, + 1.132720947265625, + 1.8430020809173584, + 0.9792376756668091, + 1.3373607397079468, + 1.3277947902679443, + 0.6020269393920898, + 1.4632021188735962, + 1.1198742389678955, + 1.9535547494888306, + 1.3624038696289062, + 1.8991485834121704, + 2.1501173973083496, + 1.0468807220458984, + 1.7869219779968262, + 0.991362452507019, + 1.5429887771606445, + 2.2720224857330322, + 1.7664763927459717, + 1.9695860147476196, + 1.7339993715286255, + 0.6108479499816895, + 1.5951225757598877, + 2.822641134262085, + 2.2305965423583984, + 1.8858641386032104, + 1.115355134010315, + 1.6147780418395996, + 1.453623652458191, + 1.439103603363037, + 1.1922818422317505, + 0.3599494695663452 + ], + "off_dist": [ + 2.5125463008880615, + 4.302741050720215, + 3.004774808883667, + 2.285696506500244, + 2.288459062576294, + 2.6403968334198, + 2.314525842666626, + 2.3858797550201416, + 2.5492520332336426, + 5.876345157623291, + 2.6272878646850586, + 2.820162057876587, + 2.5263001918792725, + 2.3060529232025146, + 2.0645365715026855, + 2.3616831302642822, + 4.010572910308838, + 3.1253013610839844, + 2.572526216506958, + 2.7255859375, + 3.957406520843506, + 1.908729910850525, + 2.6665985584259033, + 2.8962228298187256, + 2.5093374252319336, + 4.207714080810547, + 4.027291774749756, + 3.64276385307312, + 2.923862934112549, + 1.8529150485992432, + 2.8638343811035156, + 5.520559310913086, + 2.9058313369750977, + 2.9788317680358887, + 2.022322416305542, + 5.59493350982666, + 2.3082497119903564, + 2.28537654876709, + 3.075298547744751, + 2.270806312561035 + ], + "clf": [] + }, + "n_targets": 40, + "random_top5": 0.002770083102493075 + }, + "Random+ranking": { + "top1": 0.0, + "top5": 0.0, + "ndcg": 0.0, + "func_top5": 0.7, + "func_ndcg": 0.3796227424826798, + "rank": 835.075, + "endpoint_dist": 3.5514531686902044, + "spec_margin": 0.06944499611854553, + "off_dist": 3.6208981722593307, + "clf": null, + "_per": { + "top1": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "top5": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "ndcg": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "func_top5": [ + 0.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0, + 0.0, + 0.0, + 1.0, + 1.0 + ], + "func_ndcg": [ + 0.0, + 0.5410728315781073, + 0.7627039242190412, + 0.0, + 0.6415413239959994, + 0.1803895185407146, + 0.06425604298315837, + 0.5887609702257027, + 0.07768953695763149, + 0.7057973669791577, + 0.36068331303739254, + 0.6504298198589307, + 0.0, + 0.6156026077088004, + 0.05430265598413898, + 0.0, + 0.5874701422477776, + 0.7419209445012933, + 0.7294157221885674, + 0.1803895185407146, + 0.46527800892487164, + 0.0, + 0.1803895185407146, + 0.5935232626190471, + 0.4685012911985596, + 0.6897766796547542, + 0.8196104814589247, + 0.478419320161355, + 0.4713366666902212, + 0.06012983951357154, + 0.6149832266908569, + 0.4550042757773366, + 0.05430265598413898, + 0.0, + 0.4482741710520053, + 0.6692858826749959, + 0.0, + 0.0, + 0.6721367528581305, + 0.5615314259605786 + ], + "rank": [ + 101, + 1138, + 272, + 1527, + 577, + 379, + 563, + 341, + 295, + 982, + 752, + 625, + 525, + 126, + 1447, + 55, + 914, + 1022, + 396, + 1657, + 1275, + 636, + 1287, + 988, + 1025, + 1371, + 1451, + 1506, + 114, + 758, + 940, + 1330, + 1073, + 1569, + 1761, + 328, + 288, + 125, + 1620, + 264 + ], + "endpoint_dist": [ + 1.4414644241333008, + 6.693058013916016, + 2.2693819999694824, + 0.9652828574180603, + 3.2507967948913574, + 4.876590251922607, + 3.3306970596313477, + 1.2349841594696045, + 5.8425092697143555, + 9.255013465881348, + 1.2984212636947632, + 2.5836963653564453, + 5.906203269958496, + 3.9347445964813232, + 1.8527549505233765, + 2.458263397216797, + 7.6965436935424805, + 2.2596096992492676, + 2.4497601985931396, + 4.623591423034668, + 3.3757615089416504, + 2.8686976432800293, + 4.354642391204834, + 2.6507208347320557, + 1.2674821615219116, + 3.1807076930999756, + 4.321929454803467, + 2.25634765625, + 2.4679768085479736, + 1.242406964302063, + 1.650354266166687, + 4.492090225219727, + 5.940750598907471, + 1.6874139308929443, + 6.355512619018555, + 7.02701473236084, + 1.2605292797088623, + 5.716012001037598, + 2.3085837364196777, + 3.409825086593628 + ], + "spec_margin": [ + 1.4017744064331055, + -0.9852714538574219, + 0.5474181175231934, + 1.404078483581543, + 0.287736177444458, + -0.2198953628540039, + 0.43722963333129883, + 1.1993567943572998, + 0.1627063751220703, + -6.782273292541504, + 1.448604702949524, + 0.40788698196411133, + -0.2827153205871582, + 0.13470673561096191, + -0.07403314113616943, + 0.5791707038879395, + -2.201404571533203, + 1.170691967010498, + -0.0077054500579833984, + -0.23633718490600586, + -0.5291824340820312, + 0.1898641586303711, + 0.7707839012145996, + 0.1075143814086914, + 1.2684568166732788, + 0.2617514133453369, + -0.09739160537719727, + 0.4169282913208008, + 0.5450108051300049, + 0.6122345924377441, + 1.0239275693893433, + -0.9050800800323486, + 0.17412042617797852, + 1.30082106590271, + 0.1179203987121582, + -2.387010097503662, + 1.2541511058807373, + -0.34055328369140625, + 0.47608160972595215, + 0.12572550773620605 + ], + "off_dist": [ + 2.8432388305664062, + 5.707786560058594, + 2.816800117492676, + 2.369361400604248, + 3.5385329723358154, + 4.6566948890686035, + 3.7679266929626465, + 2.4343409538269043, + 6.005215644836426, + 2.472740411758423, + 2.747025966644287, + 2.9915833473205566, + 5.623487949371338, + 4.069451332092285, + 1.778721809387207, + 3.0374341011047363, + 5.495139122009277, + 3.4303016662597656, + 2.4420547485351562, + 4.387254238128662, + 2.846579074859619, + 3.0585618019104004, + 5.125426292419434, + 2.758235216140747, + 2.5359389781951904, + 3.4424591064453125, + 4.2245378494262695, + 2.673275947570801, + 3.0129876136779785, + 1.8546415567398071, + 2.6742818355560303, + 3.587010145187378, + 6.114871025085449, + 2.9882349967956543, + 6.473433017730713, + 4.640004634857178, + 2.5146803855895996, + 5.375458717346191, + 2.78466534614563, + 3.535550594329834 + ], + "clf": [] + }, + "n_targets": 40, + "random_top5": 0.002770083102493075 + } + } +} \ No newline at end of file diff --git a/models/norman/config.json b/models/norman/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f136954a69a168e2b6cee537262067a523d02b7d --- /dev/null +++ b/models/norman/config.json @@ -0,0 +1,30 @@ +{ + "dataset": "norman", + "embedding": "pca", + "split": "cell", + "rep_mode": "gene_op", + "match": "batch", + "d_pert": 64, + "hidden": 512, + "depth": 4, + "dropout": 0.0, + "lr": 0.001, + "weight_decay": 1e-05, + "batch_size": 1024, + "epochs": 60, + "lam_tan": 1.0, + "lam_semi": 0.5, + "lam_reg": 0.0001, + "grad_clip": 5.0, + "train_frac": 1.0, + "lam_dist": 0.0, + "n_dist_perts": 4, + "dist_n": 64, + "seed": 0, + "device_index": 3, + "components": [ + "map", + "tan", + "semi" + ] +} \ No newline at end of file diff --git a/models/norman/model.ptw b/models/norman/model.ptw new file mode 100644 index 0000000000000000000000000000000000000000..acf1a5604fbfb574e50c9f3fae223dd3fb2755ce --- /dev/null +++ b/models/norman/model.ptw @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66033fbe3d3755b025ca2cc9079f9d5796a6a0ad1e5174a917d3a8d29b01280b +size 3575254 diff --git a/models/norman/train_info.json b/models/norman/train_info.json new file mode 100644 index 0000000000000000000000000000000000000000..40e13d994f915b045096f8b202d6cb602f82488e --- /dev/null +++ b/models/norman/train_info.json @@ -0,0 +1,427 @@ +{ + "history": [ + { + "map": 9.090681012939005, + "tan": 51.53360254624311, + "semi": 0.024223575147729756, + "reg": 0.054827462553101426, + "total": 60.63640072766472 + }, + { + "map": 8.425921391038333, + "tan": 47.94027726790484, + "semi": 0.06671591771437842, + "reg": 0.08704450069104924, + "total": 56.39956412595861 + }, + { + "map": 8.010584641905393, + "tan": 45.6380573721493, + "semi": 0.14629346241845803, + "reg": 0.09322421428035288, + "total": 53.72179743822883 + }, + { + "map": 7.59055574501262, + "tan": 43.34580931944006, + "semi": 0.20786436011686044, + "reg": 0.10074490922338822, + "total": 51.04030850354363 + }, + { + "map": 7.467282842187321, + "tan": 42.08133489945356, + "semi": 0.229316008003319, + "reg": 0.10917095872847472, + "total": 49.66328727497774 + }, + { + "map": 7.1339511941461, + "tan": 41.18273690167595, + "semi": 0.23491062793661566, + "reg": 0.12083662180777859, + "total": 48.43415518367992 + }, + { + "map": 7.049157009405248, + "tan": 40.3212100758272, + "semi": 0.2424944858340656, + "reg": 0.1320137967739035, + "total": 47.49162696389591 + }, + { + "map": 6.835424668648663, + "tan": 39.61966542636647, + "semi": 0.24695228631882107, + "reg": 0.14088603240602157, + "total": 46.578581249012665 + }, + { + "map": 6.701632667990292, + "tan": 38.931431994718665, + "semi": 0.25987118329195413, + "reg": 0.14920588977196636, + "total": 45.763015522676355 + }, + { + "map": 6.650894136989818, + "tan": 38.54156505360323, + "semi": 0.2688319908783716, + "reg": 0.1569551400402013, + "total": 45.326890384449676 + }, + { + "map": 6.540102075127995, + "tan": 37.852282131419464, + "semi": 0.2698856874862138, + "reg": 0.16361256776487126, + "total": 44.527342571931726 + }, + { + "map": 6.474147845717037, + "tan": 37.53386289933149, + "semi": 0.2626304008504924, + "reg": 0.16940459477550843, + "total": 44.13934225194595 + }, + { + "map": 6.415744248558493, + "tan": 37.1947494394639, + "semi": 0.26695386714795055, + "reg": 0.17525369871188612, + "total": 43.743988541995776 + }, + { + "map": 6.372432673678679, + "tan": 36.79617848115809, + "semi": 0.2595589763539679, + "reg": 0.18128912391908028, + "total": 43.29840963027056 + }, + { + "map": 6.281425553209641, + "tan": 36.5531156764311, + "semi": 0.2646997798891628, + "reg": 0.18509905193658435, + "total": 42.966910137849695 + }, + { + "map": 6.194271781865289, + "tan": 36.4991086511051, + "semi": 0.25276901047019396, + "reg": 0.1893512685509289, + "total": 42.81978365954231 + }, + { + "map": 6.151554794872508, + "tan": 36.10174599815817, + "semi": 0.25931472944862705, + "reg": 0.1925594914047157, + "total": 42.3829771490658 + }, + { + "map": 6.139232712633469, + "tan": 35.99210352056167, + "semi": 0.255086947889889, + "reg": 0.19602525365703247, + "total": 42.25889907163732 + }, + { + "map": 6.0830746117760155, + "tan": 35.74959294936236, + "semi": 0.25373832552748565, + "reg": 0.19860927585293264, + "total": 41.95955607470344 + }, + { + "map": 6.0335288117913635, + "tan": 35.30244064331055, + "semi": 0.2590217774405199, + "reg": 0.20176466751624555, + "total": 41.465500102323645 + }, + { + "map": 6.110965315033408, + "tan": 35.13910877003389, + "semi": 0.2592454000430949, + "reg": 0.20424350789364645, + "total": 41.37971681707046 + }, + { + "map": 5.966023985077353, + "tan": 35.05921419929056, + "semi": 0.26047960142878923, + "reg": 0.2073487919919631, + "total": 41.155498280244714 + }, + { + "map": 5.955339403713451, + "tan": 34.779306804432586, + "semi": 0.26458901188829365, + "reg": 0.21020734945640845, + "total": 40.8669615352855 + }, + { + "map": 5.903381824493408, + "tan": 34.6496306587668, + "semi": 0.2582090713960283, + "reg": 0.2120374574819032, + "total": 40.682138835682586 + }, + { + "map": 5.896596165264354, + "tan": 34.57028377757353, + "semi": 0.26008522992624955, + "reg": 0.21361819347914526, + "total": 40.59694492115694 + }, + { + "map": 5.872959740021649, + "tan": 34.36479944341323, + "semi": 0.2692155930049279, + "reg": 0.2164839766919613, + "total": 40.372389400706574 + }, + { + "map": 5.916246035519769, + "tan": 34.28742004843319, + "semi": 0.2609161355477922, + "reg": 0.21840506331885562, + "total": 40.33414661183077 + }, + { + "map": 5.82154234717874, + "tan": 34.11426684435676, + "semi": 0.2631807318505119, + "reg": 0.21939034212161512, + "total": 40.06742224973791 + }, + { + "map": 5.7701695245855, + "tan": 33.894943573895624, + "semi": 0.25884204467429833, + "reg": 0.22040404467021718, + "total": 39.794556617736816 + }, + { + "map": 5.773050658843097, + "tan": 33.97260435889749, + "semi": 0.26208172738552094, + "reg": 0.22239502431715236, + "total": 39.876718521118164 + }, + { + "map": 5.795562414562001, + "tan": 33.92704879536348, + "semi": 0.25910024191526804, + "reg": 0.22363066476057558, + "total": 39.85218395906336 + }, + { + "map": 5.773452331038082, + "tan": 33.81521006191478, + "semi": 0.26146621125585895, + "reg": 0.22516642160275402, + "total": 39.719418357400336 + }, + { + "map": 5.737005745663362, + "tan": 33.67261639763327, + "semi": 0.25316396969206195, + "reg": 0.2269155650454409, + "total": 39.53622688966639 + }, + { + "map": 5.755465367261102, + "tan": 33.821473963120404, + "semi": 0.2626038178363267, + "reg": 0.22766746010850458, + "total": 39.708264126497156 + }, + { + "map": 5.697806498583625, + "tan": 33.55083689970129, + "semi": 0.2530074382529539, + "reg": 0.22914176521932378, + "total": 39.375170202816236 + }, + { + "map": 5.718713122255662, + "tan": 33.486163700328156, + "semi": 0.2511062082998893, + "reg": 0.23026478904135086, + "total": 39.33045269461239 + }, + { + "map": 5.680959217688617, + "tan": 33.43111181259155, + "semi": 0.25421492119922356, + "reg": 0.23129703542765448, + "total": 39.239201489616846 + }, + { + "map": 5.768596242455875, + "tan": 33.45481502308565, + "semi": 0.2551145647816798, + "reg": 0.23215885328895905, + "total": 39.35099158567541 + }, + { + "map": 5.68937446790583, + "tan": 33.34391178804285, + "semi": 0.24921904438558748, + "reg": 0.2329040185055312, + "total": 39.15791881785673 + }, + { + "map": 5.674329182680915, + "tan": 33.517153571633735, + "semi": 0.24444453637389577, + "reg": 0.23398998576928587, + "total": 39.31372793983011 + }, + { + "map": 5.64740593994365, + "tan": 33.40329338522518, + "semi": 0.24836034262005022, + "reg": 0.23468266953440273, + "total": 39.174902467166675 + }, + { + "map": 5.660573699895074, + "tan": 33.342162553001856, + "semi": 0.2492706146748627, + "reg": 0.23517600753728082, + "total": 39.12739445181454 + }, + { + "map": 5.709663426174837, + "tan": 33.25413148543414, + "semi": 0.24833782630808213, + "reg": 0.23567993259605238, + "total": 39.08798677781049 + }, + { + "map": 5.732073342098909, + "tan": 33.28247132020838, + "semi": 0.24573037137880044, + "reg": 0.23661140352487564, + "total": 39.137432659373566 + }, + { + "map": 5.699795814121471, + "tan": 33.2870241613949, + "semi": 0.24319663643836975, + "reg": 0.23693522328839584, + "total": 39.108441408942724 + }, + { + "map": 5.66639927555533, + "tan": 33.22080320470474, + "semi": 0.24251559464370503, + "reg": 0.2373233597506495, + "total": 39.008483325733856 + }, + { + "map": 5.605263008790858, + "tan": 33.22774385003483, + "semi": 0.24726183033164809, + "reg": 0.23782538315829108, + "total": 38.95666128046372 + }, + { + "map": 5.697606759912827, + "tan": 33.32277006261489, + "semi": 0.24908233061432838, + "reg": 0.23823434114456177, + "total": 39.1449411055621 + }, + { + "map": 5.6367628854863785, + "tan": 33.19991524079267, + "semi": 0.24523958914420185, + "reg": 0.2383270320646903, + "total": 38.95932102203369 + }, + { + "map": 5.643202080446131, + "tan": 33.072624010198254, + "semi": 0.24845830263460383, + "reg": 0.23823890024248293, + "total": 38.84007891486673 + }, + { + "map": 5.637564006973715, + "tan": 33.053748214946076, + "semi": 0.24669679955524557, + "reg": 0.23827644108849413, + "total": 38.814683970283056 + }, + { + "map": 5.623584515908185, + "tan": 32.98539599250345, + "semi": 0.24680674032253377, + "reg": 0.2385626555803944, + "total": 38.73240667230942 + }, + { + "map": 5.637006205670974, + "tan": 33.03702702241785, + "semi": 0.24454942257965312, + "reg": 0.2386796995997429, + "total": 38.796331125147205 + }, + { + "map": 5.6437254442888145, + "tan": 33.01595491521499, + "semi": 0.24437346265596502, + "reg": 0.2387647799709264, + "total": 38.781890252057245 + }, + { + "map": 5.622949656318216, + "tan": 33.08323285158943, + "semi": 0.2454144970020827, + "reg": 0.2388445695533472, + "total": 38.828912903280816 + }, + { + "map": 5.591019700555241, + "tan": 33.03575386720545, + "semi": 0.2427436273325892, + "reg": 0.23893670343300877, + "total": 38.748169001410986 + }, + { + "map": 5.5686474337297325, + "tan": 33.10028589473051, + "semi": 0.24532319868312163, + "reg": 0.2389937122078503, + "total": 38.79161789838005 + }, + { + "map": 5.666492995093851, + "tan": 33.08380558911492, + "semi": 0.24304543435573578, + "reg": 0.23905649868881002, + "total": 38.871844572179455 + }, + { + "map": 5.617153132663054, + "tan": 33.01807515761431, + "semi": 0.24422780490096876, + "reg": 0.2390376346952775, + "total": 38.75736561943503 + }, + { + "map": 5.6478853506200455, + "tan": 32.98348696091596, + "semi": 0.237635247190209, + "reg": 0.23903003269258669, + "total": 38.750213005963495 + } + ], + "duration_s": 132.3485288619995, + "device": "cuda:3", + "n_train_cells": 69601 +} \ No newline at end of file diff --git a/models/replogle_k562/config.json b/models/replogle_k562/config.json new file mode 100644 index 0000000000000000000000000000000000000000..ad92c98ea80fada50764cd1d29d1e46f51b0508e --- /dev/null +++ b/models/replogle_k562/config.json @@ -0,0 +1,30 @@ +{ + "dataset": "replogle_k562", + "embedding": "pca", + "split": "cell", + "rep_mode": "gene_op", + "match": "batch", + "d_pert": 64, + "hidden": 512, + "depth": 4, + "dropout": 0.0, + "lr": 0.001, + "weight_decay": 1e-05, + "batch_size": 1024, + "epochs": 60, + "lam_tan": 1.0, + "lam_semi": 0.5, + "lam_reg": 0.0001, + "grad_clip": 5.0, + "train_frac": 1.0, + "lam_dist": 0.0, + "n_dist_perts": 4, + "dist_n": 64, + "seed": 0, + "device_index": 3, + "components": [ + "map", + "tan", + "semi" + ] +} \ No newline at end of file diff --git a/models/replogle_k562/model.ptw b/models/replogle_k562/model.ptw new file mode 100644 index 0000000000000000000000000000000000000000..086377fd5f8a2a0d7a842710c9bb79e551475743 --- /dev/null +++ b/models/replogle_k562/model.ptw @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86c51f6874ae5ddd8a9db148b643c369bb6849341f1c2a11bd72af560ab92c96 +size 4010454 diff --git a/models/replogle_k562/train_info.json b/models/replogle_k562/train_info.json new file mode 100644 index 0000000000000000000000000000000000000000..7e7950683c0dc013ea6f9d5f615440089bcd6b23 --- /dev/null +++ b/models/replogle_k562/train_info.json @@ -0,0 +1,427 @@ +{ + "history": [ + { + "map": 17.031049418759036, + "tan": 100.7652035007229, + "semi": 0.0300869325780984, + "reg": 0.05529186458556683, + "total": 117.81130406763647 + }, + { + "map": 15.702098115698083, + "tan": 92.89242543802632, + "semi": 0.23400488679672216, + "reg": 0.06300994932845042, + "total": 108.71153378176999 + }, + { + "map": 14.478880721253233, + "tan": 86.36426633364195, + "semi": 0.44110062215235324, + "reg": 0.06825163805639589, + "total": 101.06370415625634 + }, + { + "map": 13.889420348328429, + "tan": 82.95308506953252, + "semi": 0.5040058114311912, + "reg": 0.07468520646745508, + "total": 97.09451611011059 + }, + { + "map": 13.466107021678578, + "tan": 80.20504859825233, + "semi": 0.543986253924184, + "reg": 0.08253169998333051, + "total": 93.94315684925427 + }, + { + "map": 12.98160643391795, + "tan": 78.17586279534675, + "semi": 0.5785175519330161, + "reg": 0.08983203342982701, + "total": 91.44673602611988 + }, + { + "map": 12.554369554891215, + "tan": 76.1268295684418, + "semi": 0.6103655777194283, + "reg": 0.0945982209273747, + "total": 88.98638935832234 + }, + { + "map": 12.333751034427, + "tan": 74.71545479514383, + "semi": 0.630091528613846, + "reg": 0.09966781087122954, + "total": 87.36425840699827 + }, + { + "map": 12.077223864468662, + "tan": 73.42789716844435, + "semi": 0.6364420017638763, + "reg": 0.10332031257740863, + "total": 85.82335018801999 + }, + { + "map": 11.925610579453506, + "tan": 72.46371103261973, + "semi": 0.6642577833943553, + "reg": 0.10804506952499414, + "total": 84.7214582368925 + }, + { + "map": 11.804282176030146, + "tan": 71.54561079941787, + "semi": 0.6641300312884442, + "reg": 0.11268213855755793, + "total": 83.68196769813439 + }, + { + "map": 11.561371233556178, + "tan": 70.57861020967557, + "semi": 0.6674197308428875, + "reg": 0.11685481599786064, + "total": 82.47370484587434 + }, + { + "map": 11.527189329073027, + "tan": 69.46824645996094, + "semi": 0.6522551836905541, + "reg": 0.12121122417511879, + "total": 81.3215787813261 + }, + { + "map": 11.307936358761477, + "tan": 68.95758423247895, + "semi": 0.6583475516988085, + "reg": 0.12639355069244063, + "total": 80.59471001563134 + }, + { + "map": 11.178683095164113, + "tan": 68.32847733931108, + "semi": 0.6504223911793201, + "reg": 0.13096818376284142, + "total": 79.83238745354987 + }, + { + "map": 11.003427319712452, + "tan": 67.71086586295785, + "semi": 0.6521954520956262, + "reg": 0.13524451967957732, + "total": 79.04040557068664 + }, + { + "map": 10.951560813111143, + "tan": 67.20007869175502, + "semi": 0.6550748394681262, + "reg": 0.13996704812947805, + "total": 78.47919226312018 + }, + { + "map": 10.97006556275603, + "tan": 66.65089327329164, + "semi": 0.6446052549721358, + "reg": 0.14411745791311387, + "total": 77.94327693790585 + }, + { + "map": 10.943851285166554, + "tan": 66.29637517557515, + "semi": 0.6255597565081212, + "reg": 0.14808494839575384, + "total": 77.55302161674994 + }, + { + "map": 10.829310615341385, + "tan": 65.82614502349458, + "semi": 0.6388694264672019, + "reg": 0.15234306628827926, + "total": 76.97490573239017 + }, + { + "map": 10.641024490455528, + "tan": 65.32731301443917, + "semi": 0.6232788764037095, + "reg": 0.15583418348392883, + "total": 76.27999204164976 + }, + { + "map": 10.647455141141817, + "tan": 64.99982744687563, + "semi": 0.6248106956481934, + "reg": 0.1595929733344487, + "total": 75.95970312341467 + }, + { + "map": 10.558299708676028, + "tan": 64.73885132430436, + "semi": 0.6366313130824597, + "reg": 0.16249608412965552, + "total": 75.61548237986379 + }, + { + "map": 10.54614951393821, + "tan": 64.54226055392971, + "semi": 0.6242870850996538, + "reg": 0.16610339148478073, + "total": 75.4005688753995 + }, + { + "map": 10.415968498626313, + "tan": 64.06486724259017, + "semi": 0.6104588338306972, + "reg": 0.16973760066094337, + "total": 74.78607970398743 + }, + { + "map": 10.342192488831358, + "tan": 63.679309597263085, + "semi": 0.6154440424659036, + "reg": 0.17334448517142953, + "total": 74.32923928793375 + }, + { + "map": 10.316441857969606, + "tan": 63.50818619170746, + "semi": 0.6117670683117656, + "reg": 0.175885058455653, + "total": 74.13052675321505 + }, + { + "map": 10.33138624414221, + "tan": 63.21885601885907, + "semi": 0.6094937672862759, + "reg": 0.17810528212553495, + "total": 73.85500434776405 + }, + { + "map": 10.27574026429808, + "tan": 62.94442481499214, + "semi": 0.6109191310870183, + "reg": 0.18016438457098874, + "total": 73.52563981886034 + }, + { + "map": 10.174346775203556, + "tan": 62.66180484325855, + "semi": 0.6059550039179913, + "reg": 0.1823571047612599, + "total": 73.13914450112875 + }, + { + "map": 10.115131217163878, + "tan": 62.483534082189784, + "semi": 0.593998660514881, + "reg": 0.1846313884893021, + "total": 72.8956800188337 + }, + { + "map": 10.109223378169073, + "tan": 62.23712921142578, + "semi": 0.6017299449289, + "reg": 0.18623309301865565, + "total": 72.64723294741148 + }, + { + "map": 10.1195849381484, + "tan": 62.454900221391156, + "semi": 0.5973637127257013, + "reg": 0.18793246014551682, + "total": 72.87318370868633 + }, + { + "map": 10.153793297804794, + "tan": 62.148185383189805, + "semi": 0.5928794482311646, + "reg": 0.18917821386417785, + "total": 72.59843682623529 + }, + { + "map": 10.018563270568848, + "tan": 62.01780002148121, + "semi": 0.5965188106933197, + "reg": 0.1902788954895812, + "total": 72.33464129559405 + }, + { + "map": 9.928655525306603, + "tan": 61.684608756721794, + "semi": 0.5881947108677456, + "reg": 0.19194930753150544, + "total": 71.907382122882 + }, + { + "map": 9.94748303797338, + "tan": 61.25752768578467, + "semi": 0.5875537790261306, + "reg": 0.19271386037399243, + "total": 71.4988080309583 + }, + { + "map": 10.026378891684793, + "tan": 61.36622505683403, + "semi": 0.592558935091093, + "reg": 0.1937573047040345, + "total": 71.68890479942421 + }, + { + "map": 10.008671971110555, + "tan": 61.23317356852742, + "semi": 0.5771013513788, + "reg": 0.1943636681739386, + "total": 71.53041770241477 + }, + { + "map": 9.882405454462225, + "tan": 61.013533703692545, + "semi": 0.5912098389167291, + "reg": 0.19557252777861311, + "total": 71.19156617003601 + }, + { + "map": 9.871191408727077, + "tan": 60.72403360342051, + "semi": 0.5780030558635663, + "reg": 0.19602004738597126, + "total": 70.88424890691584 + }, + { + "map": 9.842699013747177, + "tan": 60.829766062947066, + "semi": 0.5811261499856973, + "reg": 0.19663439207262806, + "total": 70.96305054503601 + }, + { + "map": 9.851649445372743, + "tan": 60.72009956062614, + "semi": 0.5874130930993464, + "reg": 0.1971096477725289, + "total": 70.86547792112673 + }, + { + "map": 9.752088125649985, + "tan": 60.520079773741884, + "semi": 0.5886614949672253, + "reg": 0.1976228573105552, + "total": 70.56652079000102 + }, + { + "map": 9.752064197094409, + "tan": 60.41498630077808, + "semi": 0.5917925590818579, + "reg": 0.19813017547130585, + "total": 70.4629694951045 + }, + { + "map": 9.708581379481725, + "tan": 60.503518339875455, + "semi": 0.5872518009953684, + "reg": 0.1987074689044581, + "total": 70.50574889740386 + }, + { + "map": 9.752153607157918, + "tan": 60.31089173354112, + "semi": 0.5835730073513923, + "reg": 0.19888550739783745, + "total": 70.3548547323648 + }, + { + "map": 9.624358759297953, + "tan": 60.314821961638216, + "semi": 0.5837421378532013, + "reg": 0.1992025768215006, + "total": 70.2310743455763 + }, + { + "map": 9.673671264153022, + "tan": 60.04714554625672, + "semi": 0.5863785871437618, + "reg": 0.19950627119510206, + "total": 70.01402956479555 + }, + { + "map": 9.601263975168203, + "tan": 59.98841005795962, + "semi": 0.582589351898664, + "reg": 0.19971654747987722, + "total": 69.88099145270013 + }, + { + "map": 9.659211703709193, + "tan": 60.230314973112826, + "semi": 0.5836410588258273, + "reg": 0.19984227941407787, + "total": 70.1813699301187 + }, + { + "map": 9.695229802812849, + "tan": 60.008484233509414, + "semi": 0.5898574017859125, + "reg": 0.199969258401301, + "total": 69.99866515320618 + }, + { + "map": 9.731371854806875, + "tan": 60.117312245554736, + "semi": 0.5865367480686733, + "reg": 0.20019410569946489, + "total": 70.14197520466594 + }, + { + "map": 9.704108560240114, + "tan": 60.16391328093293, + "semi": 0.5837481083808007, + "reg": 0.20012537322261117, + "total": 70.15991874793907 + }, + { + "map": 9.649337966720779, + "tan": 59.84723182777306, + "semi": 0.595232380675031, + "reg": 0.2003737132657658, + "total": 69.7942084770698 + }, + { + "map": 9.67079952784947, + "tan": 59.99238888629071, + "semi": 0.5785262170550111, + "reg": 0.20029796698650756, + "total": 69.95247412347175 + }, + { + "map": 9.637403655361819, + "tan": 59.95936525022829, + "semi": 0.5868523740149164, + "reg": 0.20019558710711344, + "total": 69.89021766959847 + }, + { + "map": 9.705964162752226, + "tan": 59.901117597307476, + "semi": 0.5755069797689264, + "reg": 0.20033776256945227, + "total": 69.8948581249683 + }, + { + "map": 9.650553046882926, + "tan": 60.12031802883396, + "semi": 0.5841203432578546, + "reg": 0.20030619087931398, + "total": 70.06295439484832 + }, + { + "map": 9.67817362252768, + "tan": 60.0385335947012, + "semi": 0.5809869928793474, + "reg": 0.20038042672268755, + "total": 70.00722394670758 + } + ], + "duration_s": 134.94324254989624, + "device": "cuda:3", + "n_train_cells": 78128 +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..fbe30a9b81de3e2ecdcec3ac14feaf41ff00d13a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +torch>=2.0 +numpy +scipy +scikit-learn +pandas +pyarrow +scanpy +anndata +matplotlib + +# gears is only needed for the head-to-head baseline and lives in a separate env +# (see scripts/setup_gears_env.sh) because it pins older torch/pyg versions. diff --git a/scripts/extra_ablations.py b/scripts/extra_ablations.py new file mode 100644 index 0000000000000000000000000000000000000000..5f69f0c4be13868ed8269faace68a193f3c7eb57 --- /dev/null +++ b/scripts/extra_ablations.py @@ -0,0 +1,110 @@ +"""extra experiments to fill in the runnable table cells. external methods we +can't reproduce stay unreported; only real computed values get emitted.""" +import sys, os, json, time +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) +import numpy as np, torch +from src.data.perturb_data import load_dataset +from src.data.splits import load_split +from src.training.train import TrainConfig, train +from src.experiments.predictors import PivotPredictor, BaselinePredictor +from src.experiments.forward_eval import evaluate_forward +from src.experiments.nomination_eval import evaluate_nomination +from src.evaluation.baselines import build_baseline +from src.utils.common import save_json + +gpu = int(os.environ.get("PIVOT_GPU", "3")) +data = load_dataset("norman") +out = {} + +# ===== core-ablation extras (centroid reward, held-out perturbation) ===== +sp = load_split(data.dir, "perturbation") +cands = [p for p in data.perturbations if len(data.parse(p)) == 1] +targets = [p for p in sp["test_perts"] if len(data.parse(p)) == 1 and p in cands][:30] +gc = data.functional_clusters(seed=0) + +def fwd_inv(model, reward="centroid"): + dev = next(model.parameters()).device + pred = PivotPredictor(model, data, dev) + f = evaluate_forward(pred, data, list(sp["test_perts"])[:50], data.control_idx, max_perts=50) + r = evaluate_nomination(pred, data, targets, cands, data.control_idx, reward_kind=reward, + method="ranking", gene_cluster=gc, device=dev) + g = evaluate_nomination(pred, data, targets, cands, data.control_idx, reward_kind=reward, + method="guidance", guidance_init="warm", rerank=False, + gene_cluster=gc, model=model, device=dev) + return f, r, g + +# velocity-only: train with only the tangent (velocity) loss +mv, _ = train(TrainConfig(dataset="norman", split="perturbation", epochs=60, device_index=gpu, + components=["tan"]), data=data, verbose=False) +fv, rv, _ = fwd_inv(mv) +out["velocity_only"] = {"mse": fv["mse"], "de_corr": fv["de_corr"], "mmd": fv["mmd"], + "endpoint_dist": rv["endpoint_dist"], "top5": rv["top5"], "ndcg": rv["ndcg"]} +print("velocity_only", out["velocity_only"], flush=True) + +# full model: ranking-only and guidance-without-reranking inverse rows +mf, info = train(TrainConfig(dataset="norman", split="perturbation", epochs=60, device_index=gpu), + data=data, verbose=False) +dev = next(mf.parameters()).device +ff, rf, gf = fwd_inv(mf) +out["ranking_only"] = {"mse": ff["mse"], "de_corr": ff["de_corr"], "mmd": ff["mmd"], + "endpoint_dist": rf["endpoint_dist"], "top5": rf["top5"], "ndcg": rf["ndcg"]} +out["guidance_no_rerank"] = {"mse": ff["mse"], "de_corr": ff["de_corr"], "mmd": ff["mmd"], + "endpoint_dist": gf["endpoint_dist"], "top5": gf["top5"], "ndcg": gf["ndcg"]} +print("ranking_only", out["ranking_only"], flush=True) +print("guidance_no_rerank", out["guidance_no_rerank"], flush=True) + +# ===== inverse-table baseline: average perturbation effect + ranking (cosine) ===== +bl = BaselinePredictor(build_baseline("AvgPerturbationEffect").fit(data, sp["train_perts"], sp["train_idx"])) +ra = evaluate_nomination(bl, data, targets, cands, data.control_idx, reward_kind="cosine", + method="ranking", gene_cluster=gc, device=dev) +out["avg_effect_ranking"] = {k: ra[k] for k in ["top1", "top5", "ndcg", "func_top5"]} +out["avg_effect_ranking"]["med_rank"] = float(np.median(ra["_per"]["rank"])) +print("avg_effect_ranking", out["avg_effect_ranking"], flush=True) + +# ===== gpu memory for compute table ===== +torch.cuda.reset_peak_memory_stats(dev) +c0 = torch.as_tensor(data.emb[data.control_idx[:256]], dtype=torch.float32, device=dev) +from src.evaluation import inference as inf +_ = inf.endpoint_ranking(mf, data, cands, c0, __import__("src.evaluation.rewards", fromlist=["Reward"]).Reward( + "centroid", target_c=data.emb[data.pert_to_idx[targets[0]]].mean(0), device=dev), device=dev) +out["gpu_mem_mb"] = round(torch.cuda.max_memory_allocated(dev) / 1e6, 1) +print("gpu_mem_mb", out["gpu_mem_mb"], flush=True) + +# ===== combination table: additive + random + pivot guidance (combination split) ===== +spc = load_split(data.dir, "combination") +combo_cands = data.combos +ctgt = [p for p in spc["test_perts"] if len(data.parse(p)) == 2][:26] +mc, _ = train(TrainConfig(dataset="norman", split="combination", epochs=60, device_index=gpu), + data=data, verbose=False) +devc = next(mc.parameters()).device +predc = PivotPredictor(mc, data, devc) +from src.experiments.nomination_eval import rank_candidates +import src.evaluation.metrics as M +rng = np.random.default_rng(0) +# additive baseline ranking over observed combos +addb = build_baseline("Additive").fit(data, spc["train_perts"], spc["train_idx"]) +addp = BaselinePredictor(addb) +def combo_eval(predictor): + e1=e5=ov=0.0 + for p in ctgt: + cstar = data.emb[data.pert_to_idx[p]].mean(0) + c0n = data.emb[rng.choice(data.control_idx, 128, replace=False)] + sk = dict(kind="centroid", c_star=cstar, target_sample=data.emb[data.pert_to_idx[p]], + device=devc, control_ref=data.emb[data.control_idx].mean(0)) + ranked,_ = rank_candidates(predictor, combo_cands, c0n, sk) + e1 += M.top_k_accuracy(ranked,p,1); e5 += M.top_k_accuracy(ranked,p,5) + ov += M.partial_overlap(data.parse(ranked[0]), set(data.parse(p))) + n=len(ctgt); return e1/n, e5/n, ov/n +ae1, ae5, aov = combo_eval(addp) +out["combo_additive"] = {"exact1": ae1, "exact5": ae5, "overlap": aov} +# random combos +r_e1=r_e5=r_ov=0.0 +for p in ctgt: + pick = list(rng.choice(combo_cands, 5, replace=False)) + r_e1 += float(p==pick[0]); r_e5 += float(p in pick) + r_ov += M.partial_overlap(data.parse(pick[0]), set(data.parse(p))) +n=len(ctgt); out["combo_random"]={"exact1":r_e1/n,"exact5":r_e5/n,"overlap":r_ov/n} +print("combo_additive", out["combo_additive"], "combo_random", out["combo_random"], flush=True) + +save_json(out, "experiments/results/norman_extra_ablations.json") +print("FILL_DONE", flush=True) diff --git a/scripts/figures.py b/scripts/figures.py new file mode 100644 index 0000000000000000000000000000000000000000..c920ed8d5b7a799fcf3d6450feaa8aabdfc66491 --- /dev/null +++ b/scripts/figures.py @@ -0,0 +1,172 @@ +"""figures for the paper, helvetica-ish font (nimbus sans). +figure 1: effect-axis embedding panels (control / observed / pivot-predicted). +figure 2: quantitative results (forward bars, gears head-to-head, dist-loss, reward).""" +import sys, os, glob, json +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) +import numpy as np +import matplotlib +matplotlib.use("Agg") +from matplotlib import font_manager as fm +import matplotlib.pyplot as plt +from matplotlib.patches import Patch +from matplotlib.lines import Line2D + +# helvetica-family font (nimbus sans = urw helvetica clone) +for patt in ["/usr/share/fonts/opentype/urw-base35/NimbusSans-*.otf", + "/usr/share/fonts/truetype/liberation2/LiberationSans-*.ttf"]: + for f in glob.glob(patt): + try: fm.fontManager.addfont(f) + except Exception: pass +plt.rcParams.update({ + "font.family": "sans-serif", + "font.sans-serif": ["Nimbus Sans", "Helvetica", "Arial", "Liberation Sans", "DejaVu Sans"], + "mathtext.fontset": "dejavusans", + "font.size": 11, "axes.labelsize": 12, "axes.titlesize": 12.5, + "xtick.labelsize": 10, "ytick.labelsize": 10, "legend.fontsize": 10, + "axes.linewidth": 0.9, "axes.edgecolor": "#444444", + "xtick.color": "#444444", "ytick.color": "#444444", + "axes.labelcolor": "#222222", "text.color": "#222222", + "figure.dpi": 150, "savefig.dpi": 320, "savefig.bbox": "tight", + "axes.spines.top": False, "axes.spines.right": False, + "axes.grid": True, "grid.color": "#E6E8EB", "grid.linewidth": 0.8, + "axes.axisbelow": True, "legend.frameon": False, +}) +# palette +C_CTRL, C_OBS, C_PRED, C_PIVOT, C_BASE, C_ACC = "#B9C2CC", "#2D6FB3", "#E4572E", "#1B9E77", "#9AA6B2", "#6A4C93" +FIG = "figures" +RES = "experiments/results" +os.makedirs(FIG, exist_ok=True) +def L(n): return json.load(open(f"{RES}/{n}.json")) +def save(fig, name): + fig.savefig(f"{FIG}/{name}.png"); fig.savefig(f"{FIG}/{name}.pdf"); plt.close(fig) + print("wrote", name) + + +def kde_contour(ax, X, color, levels=4): + try: + from scipy.stats import gaussian_kde + if len(X) < 10: return + k = gaussian_kde(X.T) + xmin, ymin = X.min(0); xmax, ymax = X.max(0) + xs, ys = np.mgrid[xmin:xmax:80j, ymin:ymax:80j] + z = k(np.vstack([xs.ravel(), ys.ravel()])).reshape(xs.shape) + ax.contour(xs, ys, z, levels=levels, colors=color, linewidths=1.0, alpha=0.55) + except Exception: + pass + + +def figure1(model, data, targets, device): + """effect-axis panels: x = projection on (target-control) direction, y = orthogonal pc.""" + from src.evaluation import inference as inf + import torch + rng = np.random.default_rng(0) + ctrl_idx = data.control_idx + cmean = data.emb[ctrl_idx].mean(0) + c0e = torch.as_tensor(data.emb[rng.choice(ctrl_idx, 400, replace=False)], dtype=torch.float32, device=device) + n = len(targets) + fig, axes = plt.subplots(1, n, figsize=(4.3 * n, 4.1)) + if n == 1: axes = [axes] + for ax, p in zip(axes, targets): + ti = data.pert_to_idx[p] + tmean = data.emb[ti].mean(0) + d = tmean - cmean; d = d / (np.linalg.norm(d) + 1e-9) # effect direction + # orthogonal axis = top pc of perturbed cells with effect-dir removed + Y = data.emb[ti] - data.emb[ti].mean(0) + Y = Y - np.outer(Y @ d, d) + u, s, vt = np.linalg.svd(Y, full_matrices=False); o = vt[0] + def proj(M): return np.c_[(M - cmean) @ d, (M - cmean) @ o] + ci = rng.choice(ctrl_idx, 500, replace=False) + Pc, Po = proj(data.emb[ci]), proj(data.emb[ti]) + e = inf.encode_label(model, data, p, device) + Ppred = proj(inf.forward_predict(model, c0e, e).cpu().numpy()) + ax.scatter(Pc[:, 0], Pc[:, 1], s=9, c=C_CTRL, alpha=0.7, linewidths=0, label="control", rasterized=True) + ax.scatter(Po[:, 0], Po[:, 1], s=11, c=C_OBS, alpha=0.55, linewidths=0, label="observed perturbed", rasterized=True) + ax.scatter(Ppred[:, 0], Ppred[:, 1], s=22, marker="X", c=C_PRED, alpha=0.9, + edgecolors="white", linewidths=0.4, label="PIVOT predicted") + kde_contour(ax, Po, C_OBS); kde_contour(ax, Ppred, C_PRED) + # arrow control-centroid -> observed-centroid (the transport) + cc, oc = proj(cmean[None])[0], proj(tmean[None])[0] + ax.annotate("", xy=(oc[0], oc[1]), xytext=(cc[0], cc[1]), + arrowprops=dict(arrowstyle="-|>", color="#333333", lw=1.6, alpha=0.8)) + ax.set_title(p.replace("_", "+"), fontweight="bold") + ax.set_xlabel("effect axis"); ax.set_ylabel("orthogonal axis") + ax.tick_params(length=0) + handles = [Line2D([], [], marker='o', ls='', mfc=C_CTRL, mec='none', ms=7, label='control'), + Line2D([], [], marker='o', ls='', mfc=C_OBS, mec='none', ms=7, label='observed perturbed'), + Line2D([], [], marker='X', ls='', mfc=C_PRED, mec='white', ms=8, label='PIVOT predicted')] + fig.legend(handles=handles, loc="lower center", ncol=3, bbox_to_anchor=(0.5, -0.04)) + fig.suptitle("Control cells transported toward the perturbed population", y=1.02, + fontsize=13, fontweight="bold") + fig.tight_layout() + save(fig, "fig1_embedding_panels") + + +def figure2_results(): + fig, ax = plt.subplots(2, 2, figsize=(11, 7.4)) + # (a) forward de-corr across methods (held-out perturbation), from benchmark + bf = L("norman_benchmark")["forward"] + order = ["PIVOT", "LinearResponse", "kNN-latent", "Additive", "NearestPerturbationCentroid", + "ConditionalMLP", "EndpointMLP", "AvgPerturbationEffect", "MeanControl", "Random"] + pretty = {"PIVOT": "PIVOT", "LinearResponse": "Linear", "kNN-latent": "kNN-latent", + "Additive": "Additive", "NearestPerturbationCentroid": "Nearest centroid", + "ConditionalMLP": "Conditional MLP", "EndpointMLP": "Endpoint MLP", + "AvgPerturbationEffect": "Avg. effect", "MeanControl": "Mean control", "Random": "Random"} + vals = [(pretty[m], bf[m]["de_corr"]) for m in order if m in bf] + vals.sort(key=lambda kv: kv[1]) + names = [v[0] for v in vals]; de = [v[1] for v in vals] + cols = [C_PIVOT if n == "PIVOT" else C_BASE for n in names] + a = ax[0, 0]; a.barh(names, de, color=cols, edgecolor="white", height=0.74) + for i, v in enumerate(de): a.text(v + 0.01, i, f"{v:.2f}", va="center", fontsize=9, color="#333") + a.set_xlim(0, 1.0); a.set_xlabel("DE correlation $\\uparrow$"); a.grid(axis="y", visible=False) + a.set_title("a Forward direction, held-out perturbations", loc="left", fontweight="bold") + + # (b) gears head-to-head + pg = L("pivot_vs_gears") + b = ax[0, 1] + bars = b.bar(["PIVOT", "GEARS"], [pg["pivot_pearson_de_expr"], pg["gears_pearson_de_expr"]], + color=[C_PIVOT, C_ACC], edgecolor="white", width=0.55) + for r, v in zip(bars, [pg["pivot_pearson_de_expr"], pg["gears_pearson_de_expr"]]): + b.text(r.get_x() + r.get_width()/2, v + 0.012, f"{v:.3f}", ha="center", fontsize=11, fontweight="bold") + b.set_ylim(0, 1.08); b.set_ylabel("Top-20 DE-gene Pearson $\\uparrow$"); b.grid(axis="x", visible=False) + b.set_title("b Head-to-head vs GEARS (matched perts)", loc="left", fontweight="bold") + + # (c) distributional loss: mmd down, de-corr preserved + dl = L("norman_distloss")["rows"] + lam = sorted(float(k) for k in dl); mmd = [dl[str(l) if str(l) in dl else f"{l:.1f}"]["mmd"] for l in lam] + de2 = [dl[str(l) if str(l) in dl else f"{l:.1f}"]["de_corr"] for l in lam] + c = ax[1, 0] + c.plot(lam, mmd, "o-", color=C_PRED, lw=2.2, ms=7, label="MMD $\\downarrow$") + c.set_xlabel("distributional-loss weight $\\lambda_{\\mathrm{dist}}$"); c.set_ylabel("population MMD $\\downarrow$", color=C_PRED) + c.tick_params(axis="y", colors=C_PRED); c.set_ylim(0, max(mmd)*1.15) + c2 = c.twinx(); c2.plot(lam, de2, "s--", color=C_OBS, lw=2.0, ms=6, label="DE-corr $\\uparrow$") + c2.set_ylabel("DE correlation $\\uparrow$", color=C_OBS); c2.tick_params(axis="y", colors=C_OBS) + c2.set_ylim(0.5, 0.95); c2.grid(False); c2.spines["top"].set_visible(False) + c.set_title("c Distributional flow loss: MMD $6\\times$ lower, direction kept", loc="left", fontweight="bold") + + # (d) reward ablation top-5 + rw = L("norman_ablation_reward")["rows"] + rmap = {"centroid": "Centroid", "nn_target": "NN-target", "mmd": "MMD", "wasserstein": "Wasserstein", "cosine": "Cosine"} + rn = [rmap.get(k, k) for k in rw]; t5 = [rw[k]["top5"] for k in rw] + cols2 = [C_PIVOT if rmap.get(k) == "Cosine" else C_BASE for k in rw] + d = ax[1, 1]; bars = d.bar(rn, t5, color=cols2, edgecolor="white", width=0.66) + for r, v in zip(bars, t5): d.text(r.get_x()+r.get_width()/2, v+0.006, f"{v:.2f}", ha="center", fontsize=9.5) + d.set_ylabel("nomination Top-5 $\\uparrow$"); d.set_ylim(0, max(t5)*1.25); d.grid(axis="x", visible=False) + d.tick_params(axis="x", rotation=18) + d.set_title("d Direction-aware reward wins at nomination", loc="left", fontweight="bold") + fig.tight_layout() + save(fig, "fig2_results") + + +if __name__ == "__main__": + from src.data.perturb_data import load_dataset + from src.training.train import TrainConfig, train + data = load_dataset("norman") + gpu = int(os.environ.get("PIVOT_GPU", "3")) + cfg = TrainConfig(dataset="norman", split="cell", epochs=60, device_index=gpu) + model, info = train(cfg, data=data, verbose=False) + dev = next(model.parameters()).device + singles = [p for p in data.perturbations if len(data.parse(p)) == 1] + combos = [p for p in data.perturbations if len(data.parse(p)) == 2] + figure1(model, data, [singles[0], singles[7], combos[0]], dev) + figure2_results() + print("FIGURES_V2_DONE") diff --git a/scripts/gears_compare.py b/scripts/gears_compare.py new file mode 100644 index 0000000000000000000000000000000000000000..8a30d03836d6c61a32d8a830dc5ae3f58def7d64 --- /dev/null +++ b/scripts/gears_compare.py @@ -0,0 +1,58 @@ +"""real gears head-to-head on norman, aligned to our held-out-perturbation test set. +runs in the isolated pivot_gears env (torch cu118 + pyg + cell-gears, gpu-capable).""" +import sys, os, json +import numpy as np +import torch + +# our held-out perturbation test labels (no src import; read the npz directly) +split = np.load("data/processed/norman/splits/perturbation.npz", allow_pickle=True) +my_test = [str(p) for p in split["test_perts"]] + +def to_gears(p): + g = [x for x in p.split("_") if x] + return g[0] + "+ctrl" if len(g) == 1 else g[0] + "+" + g[1] + +import pickle +from gears import PertData, GEARS +from gears.inference import evaluate, compute_metrics + +my_train = [str(p) for p in np.load("data/processed/norman/splits/perturbation.npz", + allow_pickle=True)["train_perts"]] +dev = "cuda" +pert_data = PertData("./gears_data") +pert_data.load(data_name="norman") +avail = set(map(str, pert_data.adata.obs["condition"].unique())) +print("example GEARS conditions:", list(avail)[:6], flush=True) +gears_test = sorted({to_gears(p) for p in my_test} & avail) +gears_train = sorted(({to_gears(p) for p in my_train} & avail) - set(gears_test)) +if "ctrl" in avail: + gears_train = sorted(set(gears_train) | {"ctrl"}) +# carve a small val set from train (gears requires non-empty val) +rng = np.random.default_rng(0) +val = sorted(rng.choice([c for c in gears_train if c != "ctrl"], + size=max(1, len(gears_train) // 10), replace=False).tolist()) +gears_train = sorted(set(gears_train) - set(val)) +split_dict = {"train": gears_train, "val": val, "test": gears_test} +print(f"custom split -> train {len(gears_train)} | val {len(val)} | test {len(gears_test)} " + f"(our test {len(my_test)})", flush=True) +os.makedirs("gears_data", exist_ok=True) +sd_path = "gears_data/pivot_custom_split.pkl" +pickle.dump(split_dict, open(sd_path, "wb")) +pert_data.prepare_split(split="custom", split_dict_path=sd_path) +pert_data.get_dataloader(batch_size=64, test_batch_size=128) + +model = GEARS(pert_data, device=dev) +model.model_initialize(hidden_size=64) +model.train(epochs=20) + +res = evaluate(pert_data.dataloader["test_loader"], model.model, False, dev) +out = compute_metrics(res) +metrics, per_pert = out if isinstance(out, tuple) else (out, {}) +keep = {k: float(v) for k, v in metrics.items() + if any(s in k for s in ["pearson", "mse", "r2", "spearman"])} +print("GEARS held-out-perturbation metrics:", flush=True) +for k, v in sorted(keep.items()): + print(f" {k:24s} {v:.4f}", flush=True) +json.dump({"n_test_perts": len(gears_test), "test_perts": gears_test, "metrics": keep}, + open("experiments/results/gears_norman.json", "w"), indent=2, default=float) +print("GEARS_RUN_DONE", flush=True) diff --git a/scripts/gears_ranking.py b/scripts/gears_ranking.py new file mode 100644 index 0000000000000000000000000000000000000000..0aef8d8904c74e9303aa377dbfec31564d5c65e8 --- /dev/null +++ b/scripts/gears_ranking.py @@ -0,0 +1,144 @@ +"""gears + ranking baseline for inverse nomination, done entirely in gears' native +gene space (fair: top-k recovery is representation-agnostic). trains gears on our +held-out-perturbation split, predicts post-perturbation expression for every +candidate, and ranks candidates by l2 distance to each held-out target's true mean +expression. emits top-1/5, ndcg@10, median rank for single-gene and combination +targets. runs in the isolated pivot_gears env.""" +import os, json, pickle +import numpy as np + +PROC = "data/processed/norman" +split = np.load(f"{PROC}/splits/perturbation.npz", allow_pickle=True) +splitC = np.load(f"{PROC}/splits/combination.npz", allow_pickle=True) +my_test = [str(p) for p in split["test_perts"]] +my_train = [str(p) for p in split["train_perts"]] +combo_test = [str(p) for p in splitC["test_perts"]] + + +def to_gears(p): + g = [x for x in p.split("_") if x] + return g[0] + "+ctrl" if len(g) == 1 else g[0] + "+" + g[1] + + +def ndcg_at_k(ranked, true, k=10): + rel = [1.0 if r == true else 0.0 for r in ranked[:k]] + dcg = sum(r / np.log2(i + 2) for i, r in enumerate(rel)) + return float(dcg) # ideal dcg = 1 (single relevant) -> ndcg = dcg + + +from gears import PertData, GEARS + +dev = "cuda" +pert_data = PertData("./gears_data") +pert_data.load(data_name="norman") +avail = set(map(str, pert_data.adata.obs["condition"].unique())) +gears_test = sorted({to_gears(p) for p in my_test} & avail) +gears_train = sorted(({to_gears(p) for p in my_train} & avail) - set(gears_test)) +if "ctrl" in avail: + gears_train = sorted(set(gears_train) | {"ctrl"}) +rng = np.random.default_rng(0) +val = sorted(rng.choice([c for c in gears_train if c != "ctrl"], + size=max(1, len(gears_train) // 10), replace=False).tolist()) +gears_train = sorted(set(gears_train) - set(val)) +split_dict = {"train": gears_train, "val": val, "test": gears_test} +os.makedirs("gears_data", exist_ok=True) +pickle.dump(split_dict, open("gears_data/pivot_custom_split.pkl", "wb")) +pert_data.prepare_split(split="custom", split_dict_path="gears_data/pivot_custom_split.pkl") +pert_data.get_dataloader(batch_size=64, test_batch_size=128) + +model = GEARS(pert_data, device=dev) +model.model_initialize(hidden_size=64) +model.train(epochs=20) +print("GEARS trained", flush=True) + +adata = pert_data.adata +genes = list(adata.var["gene_name"]) if "gene_name" in adata.var else list(adata.var_names) +cond = adata.obs["condition"].astype(str).values +X = adata.X +X = np.asarray(X.todense()) if hasattr(X, "todense") else np.asarray(X) + + +def true_mean(gcond): + m = cond == gcond + return X[m].mean(0) if m.sum() else None + + +# candidate single genes present in gears as "GENE+ctrl" +single_conds = sorted([c for c in avail if c.endswith("+ctrl") and c != "ctrl"]) +single_genes = [c.split("+")[0] for c in single_conds] +# predict every candidate's post-perturbation expression (gene space) +pred = {} +B = 64 +for i in range(0, len(single_genes), B): + chunk = single_genes[i:i + B] + try: + out = model.predict([[g] for g in chunk]) + except Exception as e: + print("predict chunk failed", chunk[:3], e, flush=True) + out = {} + for g in chunk: + for key in (g, g + "+ctrl", "_".join([g])): + if key in out: + pred[g] = np.asarray(out[key]); break +print(f"predicted {len(pred)}/{len(single_genes)} single candidates", flush=True) + +cand_genes = [g for g in single_genes if g in pred] +P = np.stack([pred[g] for g in cand_genes]) # [n_cand, n_genes] + +# single-gene targets (held-out), matched to pivot singles +single_targets = [t for t in my_test if "_" not in t and to_gears(t).split("+")[0] in pred] +res = {"single": {"top1": [], "top5": [], "ndcg": [], "rank": []}} +for t in single_targets: + tg = to_gears(t) + tm = true_mean(tg) + if tm is None: + continue + d = np.linalg.norm(P - tm[None], axis=1) + order = [cand_genes[i] for i in np.argsort(d)] + tgene = tg.split("+")[0] + res["single"]["top1"].append(float(order[0] == tgene)) + res["single"]["top5"].append(float(tgene in order[:5])) + res["single"]["ndcg"].append(ndcg_at_k(order, tgene, 10)) + res["single"]["rank"].append(int(order.index(tgene) + 1) if tgene in order else len(order)) + +# combination targets: predict combos and rank among observed combos +combo_conds = sorted([c for c in avail if c.count("+") == 1 and "ctrl" not in c]) +cpred = {} +for i in range(0, len(combo_conds), B): + chunk = combo_conds[i:i + B] + try: + out = model.predict([c.split("+") for c in chunk]) + except Exception as e: + print("combo predict failed", e, flush=True); out = {} + for c in chunk: + for key in (c, c.replace("+", "_"), "_".join(c.split("+"))): + if key in out: + cpred[c] = np.asarray(out[key]); break +print(f"predicted {len(cpred)}/{len(combo_conds)} combo candidates", flush=True) +cc = [c for c in combo_conds if c in cpred] +if cc: + PC = np.stack([cpred[c] for c in cc]) + combo_tg = [to_gears(t) for t in combo_test if to_gears(t) in cpred] + res["combo"] = {"top1": [], "top5": [], "overlap": [], "ndcg": []} + for tg in combo_tg: + tm = true_mean(tg) + if tm is None: + continue + d = np.linalg.norm(PC - tm[None], axis=1) + order = [cc[i] for i in np.argsort(d)] + res["combo"]["top1"].append(float(order[0] == tg)) + res["combo"]["top5"].append(float(tg in order[:5])) + res["combo"]["ndcg"].append(ndcg_at_k(order, tg, 10)) + s_true = set(tg.split("+")) + res["combo"]["overlap"].append(len(set(order[0].split("+")) & s_true) / len(s_true)) + +agg = {} +for grp, d in res.items(): + agg[grp] = {k: (float(np.mean(v)) if v else None) for k, v in d.items()} + agg[grp]["n"] = len(next(iter(d.values()))) + if "rank" in d and d["rank"]: + agg[grp]["med_rank"] = float(np.median(d["rank"])) +agg["n_candidates_single"] = len(cand_genes) +agg["n_candidates_combo"] = len(cc) if cc else 0 +json.dump(agg, open("experiments/results/gears_ranking.json", "w"), indent=2, default=float) +print("GEARS_RANK_DONE", json.dumps(agg), flush=True) diff --git a/scripts/setup_gears_env.sh b/scripts/setup_gears_env.sh new file mode 100644 index 0000000000000000000000000000000000000000..71d6e0b380155d906b349351b6ac2b5c1baa9621 --- /dev/null +++ b/scripts/setup_gears_env.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -e +source /home/bcheng/miniconda3/etc/profile.d/conda.sh +# fresh env, gears needs py3.10 +echo "=== create fresh env pivot_gears (py3.10) ===" +conda create -y -n pivot_gears python=3.10 >/tmp/g_env.log 2>&1 && echo "env created" +conda activate pivot_gears +# torch cu118 bundles its own cuda 11.8 runtime so it works on the old 470 driver +echo "=== torch 2.2.0 +cu118 WITH deps (bundles CUDA 11.8 runtime; works on 470 driver) ===" +pip install --no-input "numpy<2" >/tmp/g_np.log 2>&1 +pip install --no-input torch==2.2.0 torchvision==0.17.0 --index-url https://download.pytorch.org/whl/cu118 >/tmp/g_torch.log 2>&1 && echo "torch+cu118 installed" +python -c "import torch; print('TORCH', torch.__version__, 'CUDA avail:', torch.cuda.is_available(), 'devices:', torch.cuda.device_count())" +echo "=== torch_geometric + cell-gears ===" +pip install --no-input torch_geometric >/tmp/g_pyg.log 2>&1 && echo "pyg ok" +pip install --no-input cell-gears >/tmp/g_gears.log 2>&1 && echo "gears ok" +echo "=== final import + GPU test ===" +python - < max_cells: + rng = np.random.default_rng(seed) + idx = np.sort(rng.choice(adata.n_obs, size=max_cells, replace=False)) + adata = adata[idx].copy() + adata.layers["counts"] = adata.X.copy() + # normalize only to pick the same hvgs as preprocess + sc.pp.normalize_total(adata, target_sum=1e4) + sc.pp.log1p(adata) + sc.pp.highly_variable_genes(adata, n_top_genes=n_hvg, flavor="seurat") + adata = adata[:, adata.var["highly_variable"]].copy() + # min-cells filter identical to preprocess (sep/pert_col use preprocess defaults) + control_label = cfg["control_label"]; sep = cfg.get("sep", "_") + pert = adata.obs[cfg.get("pert_col", "perturbation")].astype(str).values + npg = np.array([len(parse_perturbation(p, control_label, sep)) for p in pert]) + is_ctrl = npg == 0 + import pandas as pd + vc = pd.Series(pert[~is_ctrl]).value_counts() + keep = set(vc[vc >= 20].index) + mask = is_ctrl | np.isin(pert, list(keep)) + adata = adata[mask].copy() + # restore raw counts into x + adata.X = adata.layers["counts"] + return adata + + +def build_scvi(dataset: str, n_latent: int = 50, max_epochs: int = 80, gpu: int = 0): + import scvi + import torch + + cache = os.path.join("data/processed", dataset) + meta = load_json(os.path.join(cache, "meta.json")) + adata = load_filtered_adata(dataset) + n_cached = np.load(os.path.join(cache, "pca_emb.npy")).shape[0] + assert adata.n_obs == n_cached, f"cell count mismatch {adata.n_obs} vs cached {n_cached}" + + scvi.settings.seed = 0 + scvi.model.SCVI.setup_anndata(adata, layer=None) + model = scvi.model.SCVI(adata, n_latent=n_latent) + model.to_device(gpu if torch.cuda.is_available() else "cpu") + model.train(max_epochs=max_epochs, early_stopping=True, + plan_kwargs={"lr": 1e-3}, enable_progress_bar=False) + z = model.get_latent_representation().astype(np.float32) + np.save(os.path.join(cache, "emb_scvi.npy"), z) + print(f"[{dataset}] scVI latent {z.shape} -> {cache}/emb_scvi.npy") + return z + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("dataset") + ap.add_argument("--gpu", type=int, default=0) + ap.add_argument("--n-latent", type=int, default=50) + ap.add_argument("--max-epochs", type=int, default=80) + args = ap.parse_args() + build_scvi(args.dataset, n_latent=args.n_latent, max_epochs=args.max_epochs, gpu=args.gpu) diff --git a/src/data/perturb_data.py b/src/data/perturb_data.py new file mode 100644 index 0000000000000000000000000000000000000000..cf00beffe921a1c6f3d850ebc91141d2b2db6256 --- /dev/null +++ b/src/data/perturb_data.py @@ -0,0 +1,223 @@ +"""perturbdata: in-memory representation of a preprocessed perturb-seq dataset. + +holds the data model, perturbation vocabulary, and configurable control-matching +(Table 18). +""" +from __future__ import annotations + +import os + +import numpy as np +import pandas as pd +import scipy.sparse as sp + +from src.utils.common import load_json + +MATCH_STRATEGIES = ( + "random", + "batch", + "celltype", + "batch_celltype", + "nearest", + "ot", # sinkhorn ot coupling control<->perturbed (distribution-preserving) +) + + +class PerturbData: + def __init__(self, cache_dir: str, embedding: str = "pca"): + self.dir = cache_dir + self.meta = load_json(os.path.join(cache_dir, "meta.json")) + self.obs = pd.read_parquet(os.path.join(cache_dir, "obs.parquet")) + self.genes = open(os.path.join(cache_dir, "genes_hvg.txt")).read().split("\n") + self.Xhvg = sp.load_npz(os.path.join(cache_dir, "Xhvg.npz")).tocsr() + self.pca_components = np.load(os.path.join(cache_dir, "pca_components.npy")) + self.pca_mean = np.load(os.path.join(cache_dir, "pca_mean.npy")) + pb = np.load(os.path.join(cache_dir, "pseudobulk.npz"), allow_pickle=True) + self.pb_labels = list(map(str, pb["labels"])) + self.pb_vecs = pb["vecs"].astype(np.float32) + self.control_mean = pb["control_mean"].astype(np.float32) + + self.embedding = embedding + self.emb = self._load_embedding(embedding) + self.d = self.emb.shape[1] + + self.sep = self.meta["sep"] + self.control_label = self.meta["control_label"] + self.operation = self.meta["operation"] + + self.is_control = self.obs["is_control"].values + self.control_idx = np.where(self.is_control)[0] + self.batch = self.obs["batch"].values + self.celltype = self.obs["celltype"].values + + # perturbation -> row indices + self.pert_to_idx: dict[str, np.ndarray] = { + p: sub.index.values + for p, sub in self.obs.groupby("perturbation") + if p != self.control_label + } + self.perturbations = sorted(self.pert_to_idx.keys()) + + # vocabulary: genes and operations + self.genes_vocab = sorted({g for p in self.perturbations for g in self.parse(p)}) + self.gene_to_id = {g: i for i, g in enumerate(self.genes_vocab)} + # operations: one modality per dataset, plus a 'control'/none slot id 0 + self.op_vocab = ["none", self.operation] + self.op_to_id = {o: i for i, o in enumerate(self.op_vocab)} + + self.singles = [p for p in self.perturbations if len(self.parse(p)) == 1] + self.combos = [p for p in self.perturbations if len(self.parse(p)) >= 2] + + self._pb_index = {p: i for i, p in enumerate(self.pb_labels)} + self._nn_control_cache: dict[str, np.ndarray] = {} + + # ---- embeddings ---- + def _load_embedding(self, embedding: str) -> np.ndarray: + path = os.path.join(self.dir, f"emb_{embedding}.npy") + if embedding == "pca": + path = os.path.join(self.dir, "pca_emb.npy") + if not os.path.exists(path): + raise FileNotFoundError( + f"embedding '{embedding}' not found ({path}); build it first" + ) + return np.load(path).astype(np.float32) + + def decode_to_genes(self, emb: np.ndarray) -> np.ndarray: + """decode embedding(s) back to hvg gene-space (only exact for pca).""" + if self.embedding != "pca": + raise NotImplementedError( + f"gene-space decode only defined for PCA, not '{self.embedding}'" + ) + return emb @ self.pca_components + self.pca_mean + + # ---- perturbation parsing / encoding ---- + def parse(self, label: str) -> list[str]: + if str(label) == self.control_label: + return [] + return [g for g in str(label).split(self.sep) if g and g != self.control_label] + + def pert_gene_op_ids(self, label: str): + """return (gene_ids, op_ids) arrays for a perturbation label.""" + genes = self.parse(label) + gids = np.array([self.gene_to_id[g] for g in genes if g in self.gene_to_id], dtype=np.int64) + oids = np.full(len(gids), self.op_to_id[self.operation], dtype=np.int64) + return gids, oids + + # ---- pseudobulk / effects (gene space, hvg-log) ---- + def effect_vector(self, label: str) -> np.ndarray: + """true perturbation effect = mean(perturbed) - mean(control), gene space.""" + return self.pb_vecs[self._pb_index[label]] - self.control_mean + + def all_effects(self) -> tuple[list[str], np.ndarray]: + return self.pb_labels, self.pb_vecs - self.control_mean[None, :] + + # ---- control matching (Table 18) ---- + def sample_controls(self, target_idx: np.ndarray, strategy: str, rng: np.random.Generator): + """for each perturbed cell in target_idx return a matched control row index.""" + if strategy not in MATCH_STRATEGIES: + raise ValueError(f"unknown matching strategy {strategy}") + cidx = self.control_idx + if strategy == "random": + return rng.choice(cidx, size=len(target_idx), replace=True) + + if strategy == "nearest": + return self._nearest_controls(target_idx) + + if strategy == "ot": + if not hasattr(self, "_ot_map"): + self.precompute_ot_matching() + return np.array([self._ot_map.get(int(i), self.control_idx[rng.integers(len(self.control_idx))]) + for i in target_idx], dtype=np.int64) + + # bucketed matching by batch / celltype / both + def key(i): + if strategy == "batch": + return self.batch[i] + if strategy == "celltype": + return self.celltype[i] + return (self.batch[i], self.celltype[i]) + + buckets: dict = {} + for i in cidx: + buckets.setdefault(key(i), []).append(i) + buckets = {k: np.asarray(v) for k, v in buckets.items()} + out = np.empty(len(target_idx), dtype=np.int64) + for j, i in enumerate(target_idx): + pool = buckets.get(key(i)) + if pool is None or len(pool) == 0: + pool = cidx # fall back to any control + out[j] = pool[rng.integers(len(pool))] + return out + + def _nearest_controls(self, target_idx: np.ndarray) -> np.ndarray: + from sklearn.neighbors import NearestNeighbors + + nn = NearestNeighbors(n_neighbors=1).fit(self.emb[self.control_idx]) + _, j = nn.kneighbors(self.emb[target_idx]) + return self.control_idx[j.ravel()] + + def precompute_ot_matching(self, max_ctrl: int = 800, max_pert: int = 1200, + eps: float = 0.05, iters: int = 150, seed: int = 0): + """for each perturbation, couple its cells to control cells via entropic ot + (sinkhorn) on embedding l2 cost, and assign each perturbed cell a control by + sampling its coupling row. distribution-preserving alternative to random + matching (cf. cellot / ot-cfm). caches self._ot_map (perturbed idx -> control idx).""" + import torch + + rng = np.random.default_rng(seed) + dev = "cuda" if torch.cuda.is_available() else "cpu" + cidx = self.control_idx + csamp = cidx if len(cidx) <= max_ctrl else cidx[rng.choice(len(cidx), max_ctrl, replace=False)] + C = torch.as_tensor(self.emb[csamp], dtype=torch.float32, device=dev) + self._ot_map = {} + for p, idx in self.pert_to_idx.items(): + t_idx = idx if len(idx) <= max_pert else idx[rng.choice(len(idx), max_pert, replace=False)] + T = torch.as_tensor(self.emb[t_idx], dtype=torch.float32, device=dev) + cost = torch.cdist(T, C).pow(2) + cost = cost / (cost.median() + 1e-8) + K = torch.exp(-cost / eps) + n, m = K.shape + u = torch.ones(n, device=dev) / n + v = torch.ones(m, device=dev) / m + a = torch.full((n,), 1.0 / n, device=dev) + b = torch.full((m,), 1.0 / m, device=dev) + for _ in range(iters): + u = a / (K @ v + 1e-8) + v = b / (K.t() @ u + 1e-8) + P = (u.unsqueeze(1) * K) * v.unsqueeze(0) # coupling (n, m) + P = P / (P.sum(1, keepdim=True) + 1e-12) + # sample a control per perturbed cell from its coupling row + choice = torch.multinomial(P, 1, generator=None).squeeze(1).cpu().numpy() + for c_local, cell in zip(choice, t_idx): + self._ot_map[int(cell)] = int(csamp[c_local]) + return self._ot_map + + # ---- functional clusters (for pathway/functional recovery) ---- + def functional_clusters(self, n_clusters: int = 15, seed: int = 0) -> dict[str, int]: + """cluster single-gene perturbations by effect-vector correlation. + + data-driven proxy for 'same pathway': perturbations with similar + transcriptional effects get grouped. used for functional top-k and + pathway-ndcg (Table 6).""" + from sklearn.cluster import AgglomerativeClustering + + labels = self.singles + E = np.stack([self.effect_vector(p) for p in labels]) + # correlation distance + En = E - E.mean(1, keepdims=True) + En = En / (np.linalg.norm(En, axis=1, keepdims=True) + 1e-8) + sim = np.clip(En @ En.T, -1, 1) + dist = 1 - sim + k = min(n_clusters, len(labels)) + cl = AgglomerativeClustering(n_clusters=k, metric="precomputed", linkage="average") + ids = cl.fit_predict(dist) + # map by single-gene name + out = {} + for p, c in zip(labels, ids): + g = self.parse(p)[0] + out[g] = int(c) + return out + + +def load_dataset(name: str, embedding: str = "pca", root: str = "data/processed") -> PerturbData: + return PerturbData(os.path.join(root, name), embedding=embedding) diff --git a/src/data/preprocess.py b/src/data/preprocess.py new file mode 100644 index 0000000000000000000000000000000000000000..c1bf39f6c79407265527f5a2a24d0f3d8559f4da --- /dev/null +++ b/src/data/preprocess.py @@ -0,0 +1,225 @@ +"""preprocess a raw perturb-seq .h5ad (scperturb format) into a compact, +reusable cache. + +does normalization, hvg selection, and the pca cell-state encoder (the default +invertible phi used for gene-space metrics). + +outputs, under data/processed//: + meta.json dataset summary (counts, operation, control label, ...) + genes_hvg.txt hvg gene symbols (defines gene-space for de metrics) + Xhvg.npz scipy csr, log1p(cp10k) on hvgs (n_cells x n_hvg) + obs.parquet per-cell metadata + parsed perturbation + pca_emb.npy pca cell-state embedding (n_cells x d) + pca_components.npy pca basis (d x n_hvg) + pca_mean.npy feature means (n_hvg,) + pseudobulk.npz per-perturbation mean hvg-log vectors + control mean + +the pca is fit on log1p(cp10k) hvg features (zero-centered, unscaled) so that +x_hat = emb @ components + mean reconstructs gene-space expression - this is what +lets us evaluate de-gene correlation on predictions made in embedding space. +""" +from __future__ import annotations + +import argparse +import os +import sys + +import numpy as np +import pandas as pd +import scipy.sparse as sp + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) +from src.utils.common import save_json, set_seed # noqa: E402 + + +def parse_perturbation(label: str, control_label: str, sep: str = "_"): + """return list of perturbed genes for a perturbation label ([] for control).""" + if str(label) == control_label: + return [] + return [g for g in str(label).split(sep) if g and g != control_label] + + +def preprocess( + raw_path: str, + out_dir: str, + name: str, + operation: str, + control_label: str = "control", + pert_col: str = "perturbation", + batch_col: str | None = "gemgroup", + celltype_col: str | None = "celltype", + sep: str = "_", + n_hvg: int = 2000, + n_pca: int = 50, + min_cells_per_pert: int = 20, + max_cells: int | None = None, + seed: int = 0, +): + """preprocess one dataset. operation is the crispr modality, e.g. + 'activation' (crispra, norman) or 'interference' (crispri, replogle).""" + import scanpy as sc + + set_seed(seed) + os.makedirs(out_dir, exist_ok=True) + print(f"[{name}] reading {raw_path}") + adata = sc.read_h5ad(raw_path) + + # optional subsample for tractability (deterministic) + if max_cells is not None and adata.n_obs > max_cells: + rng = np.random.default_rng(seed) + idx = np.sort(rng.choice(adata.n_obs, size=max_cells, replace=False)) + adata = adata[idx].copy() + print(f"[{name}] subsampled to {adata.n_obs} cells") + + # resolve columns with graceful fallback + if pert_col not in adata.obs: + raise KeyError(f"pert_col '{pert_col}' not in obs: {list(adata.obs.columns)}") + if batch_col is not None and batch_col not in adata.obs: + print(f"[{name}] batch_col '{batch_col}' missing -> single batch") + batch_col = None + if celltype_col is not None and celltype_col not in adata.obs: + celltype_col = None + + # --- normalization: cp10k + log1p --- + if not sp.issparse(adata.X): + adata.X = sp.csr_matrix(adata.X) + # guard: ensure raw-ish counts (integer). if already normalized, skip. + x0 = adata.X[:200].toarray() + looks_counts = np.allclose(x0, np.round(x0)) and x0.max() > 30 + if looks_counts: + sc.pp.normalize_total(adata, target_sum=1e4) + sc.pp.log1p(adata) + print(f"[{name}] applied CP10k + log1p") + else: + print(f"[{name}] data appears pre-normalized (max={x0.max():.2f}); skipping norm") + + # --- hvg selection (on all cells) --- + sc.pp.highly_variable_genes(adata, n_top_genes=n_hvg, flavor="seurat") + adata = adata[:, adata.var["highly_variable"]].copy() + genes = list(map(str, adata.var_names)) + Xhvg = adata.X.tocsr().astype(np.float32) + print(f"[{name}] HVG matrix: {Xhvg.shape}") + + # --- pca (centered, invertible) --- + from sklearn.decomposition import PCA + + Xd = Xhvg.toarray() + mean = Xd.mean(axis=0).astype(np.float32) + pca = PCA(n_components=n_pca, random_state=seed) + emb = pca.fit_transform(Xd - mean).astype(np.float32) + components = pca.components_.astype(np.float32) # (d, h) + evr = float(pca.explained_variance_ratio_.sum()) + print(f"[{name}] PCA d={n_pca} explains {evr:.3f} variance") + + # --- parse perturbations --- + pert = adata.obs[pert_col].astype(str).values + genes_per_cell = [parse_perturbation(p, control_label, sep) for p in pert] + nperts = np.array([len(g) for g in genes_per_cell]) + is_control = nperts == 0 + + obs = pd.DataFrame( + { + "perturbation": pert, + "n_pert_genes": nperts, + "is_control": is_control, + "pert_genes": [";".join(g) for g in genes_per_cell], + "batch": (adata.obs[batch_col].astype(str).values if batch_col else "0"), + "celltype": ( + adata.obs[celltype_col].astype(str).values if celltype_col else name + ), + } + ) + + # drop perturbations (non-control) with too few cells + vc = obs.loc[~obs.is_control, "perturbation"].value_counts() + keep_perts = set(vc[vc >= min_cells_per_pert].index) + keep_mask = obs.is_control.values | obs.perturbation.isin(keep_perts).values + if keep_mask.sum() < len(obs): + obs = obs.loc[keep_mask].reset_index(drop=True) + Xhvg = Xhvg[keep_mask] + emb = emb[keep_mask] + print(f"[{name}] dropped low-count perts -> {keep_mask.sum()} cells, " + f"{len(keep_perts)} perturbations") + + # --- pseudobulk per perturbation (gene space, hvg-log) --- + control_mean = np.asarray(Xhvg[obs.is_control.values].mean(axis=0)).ravel().astype(np.float32) + pb_labels, pb_vecs = [], [] + for p, sub in obs.groupby("perturbation"): + if p == control_label: + continue + idx = sub.index.values + pb_labels.append(p) + pb_vecs.append(np.asarray(Xhvg[idx].mean(axis=0)).ravel()) + pb_vecs = np.asarray(pb_vecs, dtype=np.float32) + + # --- write cache --- + sp.save_npz(os.path.join(out_dir, "Xhvg.npz"), Xhvg) + np.save(os.path.join(out_dir, "pca_emb.npy"), emb) + np.save(os.path.join(out_dir, "pca_components.npy"), components) + np.save(os.path.join(out_dir, "pca_mean.npy"), mean) + np.savez( + os.path.join(out_dir, "pseudobulk.npz"), + labels=np.array(pb_labels), + vecs=pb_vecs, + control_mean=control_mean, + ) + obs.to_parquet(os.path.join(out_dir, "obs.parquet")) + with open(os.path.join(out_dir, "genes_hvg.txt"), "w") as f: + f.write("\n".join(genes)) + + singles = [p for p in pb_labels if len(parse_perturbation(p, control_label, sep)) == 1] + combos = [p for p in pb_labels if len(parse_perturbation(p, control_label, sep)) >= 2] + all_genes = sorted({g for p in pb_labels for g in parse_perturbation(p, control_label, sep)}) + meta = { + "name": name, + "operation": operation, + "control_label": control_label, + "sep": sep, + "n_cells": int(obs.shape[0]), + "n_control": int(obs.is_control.sum()), + "n_hvg": len(genes), + "n_pca": n_pca, + "pca_explained_var": evr, + "n_perturbations": len(pb_labels), + "n_singles": len(singles), + "n_combos": len(combos), + "n_unique_target_genes": len(all_genes), + "n_batches": int(obs.batch.nunique()), + "n_celltypes": int(obs.celltype.nunique()), + } + save_json(meta, os.path.join(out_dir, "meta.json")) + print(f"[{name}] DONE -> {out_dir}") + print(meta) + return meta + + +DATASETS = { + "norman": dict( + raw="data/raw/NormanWeissman2019_filtered.h5ad", + name="norman", operation="activation", control_label="control", + batch_col="gemgroup", celltype_col="celltype", + ), + "replogle_k562": dict( + raw="data/raw/ReplogleWeissman2022_K562_essential.h5ad", + name="replogle_k562", operation="interference", control_label="control", + batch_col="gemgroup", celltype_col="celltype", max_cells=120000, + ), + "replogle_rpe1": dict( + raw="data/raw/ReplogleWeissman2022_rpe1.h5ad", + name="replogle_rpe1", operation="interference", control_label="control", + batch_col="gemgroup", celltype_col="celltype", max_cells=120000, + ), +} + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("dataset", choices=list(DATASETS.keys())) + ap.add_argument("--out-root", default="data/processed") + ap.add_argument("--n-hvg", type=int, default=2000) + ap.add_argument("--n-pca", type=int, default=50) + args = ap.parse_args() + cfg = dict(DATASETS[args.dataset]) + raw = cfg.pop("raw") + out = os.path.join(args.out_root, cfg["name"]) + preprocess(raw, out, n_hvg=args.n_hvg, n_pca=args.n_pca, **cfg) diff --git a/src/data/splits.py b/src/data/splits.py new file mode 100644 index 0000000000000000000000000000000000000000..05a495b055adf5d6d00568bfd04af478e4bee5f8 --- /dev/null +++ b/src/data/splits.py @@ -0,0 +1,147 @@ +"""deterministic, serialized data splits. + +six split regimes, each guaranteeing the stated generalization gap with no leakage: + cell held-out cells of seen perturbations (interpolation over cells) + perturbation held-out perturbation labels (cells never in train; genes may be seen elsewhere) + gene held-out target genes (gene id never appears in any train perturbation) + gene_family held-out functional families (effect-correlation clusters of genes) + combination held-out combos; train on singles + remaining combos + cell_type cross-dataset transfer (handled at the experiment level, marker only here) + +each split serializes train/val/test cell indices plus the train/test perturbation +label sets, so forward training and inverse-nomination eval read identical partitions. +control cells are always available in train (and proportionally in val/test for matching). +""" +from __future__ import annotations + +import argparse +import os + +import numpy as np + +from src.data.perturb_data import PerturbData, load_dataset + +SPLIT_TYPES = ("cell", "perturbation", "gene", "gene_family", "combination") + + +def _split_control(control_idx, rng, fracs=(0.7, 0.15, 0.15)): + perm = rng.permutation(control_idx) + n = len(perm) + a, b = int(fracs[0] * n), int((fracs[0] + fracs[1]) * n) + return perm[:a], perm[a:b], perm[b:] + + +def make_split(data: PerturbData, split_type: str, seed: int = 0, holdout_frac: float = 0.2): + rng = np.random.default_rng(seed) + perts = list(data.perturbations) + ctrl_tr, ctrl_va, ctrl_te = _split_control(data.control_idx, rng) + + def cells(pert_list): + if not pert_list: + return np.array([], dtype=np.int64) + return np.concatenate([data.pert_to_idx[p] for p in pert_list]) + + if split_type == "cell": + # all perturbations seen; split each perturbation's cells 70/15/15 + tr, va, te = [], [], [] + for p in perts: + idx = rng.permutation(data.pert_to_idx[p]) + n = len(idx) + a, b = int(0.7 * n), int(0.85 * n) + tr.append(idx[:a]); va.append(idx[a:b]); te.append(idx[b:]) + train_idx = np.concatenate(tr + [ctrl_tr]) + val_idx = np.concatenate(va + [ctrl_va]) + test_idx = np.concatenate(te + [ctrl_te]) + train_perts = test_perts = perts + + elif split_type == "perturbation": + held = set(rng.choice(perts, size=max(1, int(holdout_frac * len(perts))), replace=False)) + remaining = [p for p in perts if p not in held] + # carve a disjoint validation set of perturbations (no train/val cell overlap) + n_val = max(1, len(remaining) // 8) + val_perts = set(rng.choice(remaining, size=n_val, replace=False)) if remaining else set() + train_perts = [p for p in remaining if p not in val_perts] + test_perts = sorted(held) + train_idx = np.concatenate([cells(train_perts), ctrl_tr]) + val_idx = np.concatenate([cells(sorted(val_perts)), ctrl_va]) if val_perts else ctrl_va + test_idx = np.concatenate([cells(test_perts), ctrl_te]) + # leakage guard: val perturbations must be disjoint from train + assert not (set(val_perts) & set(train_perts)), "LEAKAGE: val/train perturbation overlap" + + elif split_type in ("gene", "gene_family"): + if split_type == "gene": + units = data.genes_vocab + unit_of = {g: g for g in units} + else: + fam = data.functional_clusters(seed=seed) # gene -> cluster id + unit_of = {g: f"fam{fam.get(g, -1)}" for g in data.genes_vocab} + units = sorted(set(unit_of.values())) + held_units = set(rng.choice(units, size=max(1, int(holdout_frac * len(units))), replace=False)) + held_genes = {g for g in data.genes_vocab if unit_of[g] in held_units} + test_perts = [p for p in perts if any(g in held_genes for g in data.parse(p))] + train_perts = [p for p in perts if p not in set(test_perts)] + train_idx = np.concatenate([cells(train_perts), ctrl_tr]) + val_idx = ctrl_va + test_idx = np.concatenate([cells(test_perts), ctrl_te]) if test_perts else ctrl_te + + elif split_type == "combination": + combos = data.combos + if not combos: + raise ValueError(f"{data.meta['name']} has no combinations") + held = set(rng.choice(combos, size=max(1, int(holdout_frac * len(combos))), replace=False)) + test_perts = sorted(held) + train_perts = [p for p in perts if p not in held] + train_idx = np.concatenate([cells(train_perts), ctrl_tr]) + val_idx = ctrl_va + test_idx = np.concatenate([cells(test_perts), ctrl_te]) + else: + raise ValueError(split_type) + + # leakage guard: test-only perturbations must not appear in train cells + if split_type != "cell": + train_label_set = set(np.unique(data.obs.loc[train_idx, "perturbation"].values)) + for p in test_perts: + assert p not in train_label_set or split_type == "cell", \ + f"LEAKAGE: held-out perturbation {p} present in train cells" + + return dict( + split_type=split_type, + seed=seed, + train_idx=train_idx.astype(np.int64), + val_idx=val_idx.astype(np.int64), + test_idx=test_idx.astype(np.int64), + train_perts=np.array(train_perts), + test_perts=np.array(test_perts), + ) + + +def save_split(d: dict, out_dir: str): + os.makedirs(out_dir, exist_ok=True) + path = os.path.join(out_dir, f"{d['split_type']}.npz") + np.savez(path, **{k: v for k, v in d.items() if isinstance(v, np.ndarray)}, + meta=np.array([d["split_type"], str(d["seed"])])) + return path + + +def load_split(cache_dir: str, split_type: str) -> dict: + z = np.load(os.path.join(cache_dir, "splits", f"{split_type}.npz"), allow_pickle=True) + return {k: z[k] for k in z.files} + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("dataset") + ap.add_argument("--seed", type=int, default=0) + ap.add_argument("--root", default="data/processed") + args = ap.parse_args() + data = load_dataset(args.dataset, root=args.root) + out = os.path.join(args.root, args.dataset, "splits") + for st in SPLIT_TYPES: + if st == "combination" and not data.combos: + print(f"skip {st} (no combos in {args.dataset})") + continue + d = make_split(data, st, seed=args.seed) + p = save_split(d, out) + print(f"{st:14s} train_cells={len(d['train_idx']):7d} " + f"val={len(d['val_idx']):6d} test={len(d['test_idx']):6d} " + f"train_perts={len(d['train_perts']):4d} test_perts={len(d['test_perts']):4d} -> {p}") diff --git a/src/evaluation/__init__.py b/src/evaluation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/evaluation/baselines.py b/src/evaluation/baselines.py new file mode 100644 index 0000000000000000000000000000000000000000..c0bc19ed9d9a1a4be3cf9fce9306dfaec5ed3006 --- /dev/null +++ b/src/evaluation/baselines.py @@ -0,0 +1,275 @@ +"""baseline forward models. + +all baselines are effect predictors in cell-state embedding space: +predict_effect(label) -> delta, and predict_endpoint(c0) = c0 + delta. +this unifies forward eval and the inverse "+ranking" wrappers (rank +candidates by the reward of their predicted endpoint). + +implemented: Random, MeanControl, GlobalAverageEffect, Additive, +LinearRidge, NearestCentroid, EndpointMLP. heavy/foundation baselines +are out of scope and handled as n/r rows by the runner, never faked. +""" +from __future__ import annotations + +import numpy as np + +from src.data.perturb_data import PerturbData + + +def training_effects(data: PerturbData, train_perts, train_idx) -> dict[str, np.ndarray]: + """embedding-space effect (pert mean - control mean) per training perturbation.""" + train_set = set(train_idx.tolist()) + ctrl = np.array([i for i in data.control_idx if i in train_set]) + cmean = data.emb[ctrl].mean(0) if len(ctrl) else data.emb[data.control_idx].mean(0) + eff = {} + for p in train_perts: + idx = np.array([i for i in data.pert_to_idx[p] if i in train_set]) + if len(idx) == 0: + idx = data.pert_to_idx[p] + eff[p] = data.emb[idx].mean(0) - cmean + return eff, cmean + + +class EffectPredictor: + name = "base" + + def fit(self, data, train_perts, train_idx): + self.data = data + self.eff, self.cmean = training_effects(data, train_perts, train_idx) + self.train_perts = list(train_perts) + self.d = data.d + self._fit_extra() + return self + + def _fit_extra(self): + pass + + def predict_effect(self, label) -> np.ndarray: + raise NotImplementedError + + def predict_endpoint(self, label, c0: np.ndarray) -> np.ndarray: + return c0 + self.predict_effect(label)[None, :] + + +class Random(EffectPredictor): + name = "Random" + + def _fit_extra(self): + self._rng = np.random.default_rng(0) + self._effs = np.stack(list(self.eff.values())) if self.eff else np.zeros((1, self.d)) + + def predict_effect(self, label): + return self._effs[self._rng.integers(len(self._effs))] + + +class MeanControl(EffectPredictor): + name = "MeanControl" + + def predict_effect(self, label): + return np.zeros(self.d, dtype=np.float32) + + +class GlobalAverageEffect(EffectPredictor): + name = "AvgPerturbationEffect" + + def _fit_extra(self): + self._mean = np.mean(list(self.eff.values()), axis=0) if self.eff else np.zeros(self.d) + + def predict_effect(self, label): + return self._mean + + +class Additive(EffectPredictor): + name = "Additive" + + def _fit_extra(self): + # single-gene effects from training singles + self._single = {} + for p, e in self.eff.items(): + g = self.data.parse(p) + if len(g) == 1: + self._single[g[0]] = e + self._fallback = np.mean(list(self.eff.values()), axis=0) if self.eff else np.zeros(self.d) + + def predict_effect(self, label): + genes = self.data.parse(label) + parts = [self._single[g] for g in genes if g in self._single] + return np.sum(parts, axis=0) if parts else self._fallback + + +class LinearRidge(EffectPredictor): + name = "LinearResponse" + + def _fit_extra(self): + from sklearn.linear_model import Ridge + + genes = self.data.genes_vocab + gid = {g: i for i, g in enumerate(genes)} + X = np.zeros((len(self.train_perts), len(genes)), dtype=np.float32) + Y = np.zeros((len(self.train_perts), self.d), dtype=np.float32) + for r, p in enumerate(self.train_perts): + for g in self.data.parse(p): + if g in gid: + X[r, gid[g]] = 1.0 + Y[r] = self.eff[p] + self._gid = gid + self._model = Ridge(alpha=1.0).fit(X, Y) + + def predict_effect(self, label): + x = np.zeros((1, len(self._gid)), dtype=np.float32) + for g in self.data.parse(label): + if g in self._gid: + x[0, self._gid[g]] = 1.0 + return self._model.predict(x)[0] + + +class NearestCentroid(EffectPredictor): + name = "NearestPerturbationCentroid" + + def _fit_extra(self): + self._sets = {p: set(self.data.parse(p)) for p in self.train_perts} + + def predict_effect(self, label): + gq = set(self.data.parse(label)) + best, best_j = None, -1.0 + for p, gs in self._sets.items(): + j = len(gq & gs) / max(len(gq | gs), 1) + if j > best_j: + best_j, best = j, p + return self.eff[best] if best is not None else np.zeros(self.d) + + +class EndpointMLP(EffectPredictor): + name = "EndpointMLP" + + def _fit_extra(self): + import torch + import torch.nn as nn + + genes = self.data.genes_vocab + gid = {g: i for i, g in enumerate(genes)} + X = np.zeros((len(self.train_perts), len(genes)), dtype=np.float32) + Y = np.zeros((len(self.train_perts), self.d), dtype=np.float32) + for r, p in enumerate(self.train_perts): + for g in self.data.parse(p): + if g in gid: + X[r, gid[g]] = 1.0 + Y[r] = self.eff[p] + self._gid = gid + dev = "cuda" if torch.cuda.is_available() else "cpu" + self._dev = dev + net = nn.Sequential(nn.Linear(len(genes), 256), nn.ReLU(), + nn.Linear(256, 256), nn.ReLU(), nn.Linear(256, self.d)).to(dev) + Xt = torch.as_tensor(X, device=dev); Yt = torch.as_tensor(Y, device=dev) + opt = torch.optim.Adam(net.parameters(), lr=1e-3, weight_decay=1e-5) + for _ in range(800): + opt.zero_grad() + loss = ((net(Xt) - Yt) ** 2).sum(-1).mean() + loss.backward(); opt.step() + net.eval() + self._net = net + + def predict_effect(self, label): + import torch + + x = np.zeros((1, len(self._gid)), dtype=np.float32) + for g in self.data.parse(label): + if g in self._gid: + x[0, self._gid[g]] = 1.0 + with torch.no_grad(): + return self._net(torch.as_tensor(x, device=self._dev)).cpu().numpy()[0] + + +class KNNLatent(EffectPredictor): + name = "kNN-latent" + K = 5 + + def _fit_extra(self): + self._sets = {p: set(self.data.parse(p)) for p in self.train_perts} + + def predict_effect(self, label): + gq = set(self.data.parse(label)) + sims = sorted(((len(gq & gs) / max(len(gq | gs), 1), p) for p, gs in self._sets.items()), + reverse=True)[: self.K] + if not sims or sims[0][0] == 0: + return np.mean(list(self.eff.values()), axis=0) if self.eff else np.zeros(self.d) + return np.mean([self.eff[p] for _, p in sims], axis=0) + + +class ConditionalMLP(EffectPredictor): + """c0-conditional endpoint predictor: mlp([c0, gene multi-hot]) -> c1 in embedding space. + a fair neural competitor to pivot (cell-state dependent, unlike the constant-effect mlp).""" + name = "ConditionalMLP" + + def _fit_extra(self): + import torch + import torch.nn as nn + + genes = self.data.genes_vocab + gid = {g: i for i, g in enumerate(genes)} + self._gid = gid + train_set = set(self._train_idx.tolist()) if hasattr(self, "_train_idx") else None + # build matched (c0, c1, multihot) training pairs + rng = np.random.default_rng(0) + rows_c0, rows_c1, rows_x = [], [], [] + for p in self.train_perts: + idx = self.data.pert_to_idx[p] + c1 = self.data.emb[idx] + ctrl = self.data.sample_controls(idx, "batch", rng) + c0 = self.data.emb[ctrl] + x = np.zeros((len(idx), len(genes)), dtype=np.float32) + for g in self.data.parse(p): + if g in gid: + x[:, gid[g]] = 1.0 + rows_c0.append(c0); rows_c1.append(c1); rows_x.append(x) + C0 = np.concatenate(rows_c0); C1 = np.concatenate(rows_c1); X = np.concatenate(rows_x) + dev = "cuda" if torch.cuda.is_available() else "cpu" + self._dev = dev + net = nn.Sequential(nn.Linear(self.d + len(genes), 512), nn.SiLU(), + nn.Linear(512, 512), nn.SiLU(), nn.Linear(512, self.d)).to(dev) + inp = torch.as_tensor(np.concatenate([C0, X], 1), device=dev) + tgt = torch.as_tensor(C1, device=dev) + opt = torch.optim.AdamW(net.parameters(), lr=1e-3, weight_decay=1e-5) + n = inp.shape[0] + for _ in range(40): + perm = torch.randperm(n, device=dev) + for b in range(0, n, 2048): + bi = perm[b:b + 2048] + opt.zero_grad() + loss = ((net(inp[bi]) - tgt[bi]) ** 2).sum(-1).mean() + loss.backward(); opt.step() + net.eval() + self._net = net + + def fit(self, data, train_perts, train_idx): + self._train_idx = train_idx + return super().fit(data, train_perts, train_idx) + + def predict_endpoint(self, label, c0): + import torch + x = np.zeros((1, len(self._gid)), dtype=np.float32) + for g in self.data.parse(label): + if g in self._gid: + x[0, self._gid[g]] = 1.0 + X = np.repeat(x, len(c0), axis=0) + inp = torch.as_tensor(np.concatenate([c0.astype(np.float32), X], 1), device=self._dev) + with torch.no_grad(): + return self._net(inp).cpu().numpy() + + def predict_effect(self, label): + # mean effect over a control sample (for forward effect-vector metrics) + c0 = self.data.emb[self.data.control_idx[:256]] + return self.predict_endpoint(label, c0).mean(0) - c0.mean(0) + + +BASELINES = { + b.name: b for b in [ + Random(), MeanControl(), GlobalAverageEffect(), Additive(), + LinearRidge(), NearestCentroid(), EndpointMLP(), KNNLatent(), ConditionalMLP(), + ] +} + + +def build_baseline(name: str) -> EffectPredictor: + cls = {b.name: type(b) for b in BASELINES.values()}[name] + return cls() diff --git a/src/evaluation/inference.py b/src/evaluation/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..a922a528159ed7db94a05be1c54caa9495aeea2b --- /dev/null +++ b/src/evaluation/inference.py @@ -0,0 +1,117 @@ +"""inference procedures, algorithms 2-6. + + Alg 2 forward_predict predict endpoint, aggregate population + Alg 3 endpoint_ranking score = reward of predicted endpoint, return topk + Alg 4 reward_guidance jacobian pull-back ascent in embedding space + Alg 5 project_and_rerank project e* to admissible candidates, rerank k-nn + Alg 6 greedy_combinatorial sequentially add the best gene (mean-pool embed) +""" +from __future__ import annotations + +import numpy as np +import torch + +from src.models.encoders import build_pert_tensors +from src.models.pivot import PIVOT, candidate_embeddings + + +def encode_label(model: PIVOT, data, label: str, device) -> torch.Tensor: + g, o, mask, pid = build_pert_tensors(data, [label], device=device) + return model.encode(g, o, mask, pid) # (1, m) + + +def encode_gene_set(model: PIVOT, data, genes: list[str], device) -> torch.Tensor: + """encode an arbitrary gene set via the dataset operation (mean-pool).""" + label = data.sep.join(genes) if genes else data.control_label + return encode_label(model, data, label, device) + + +@torch.no_grad() +def forward_predict(model: PIVOT, c0: torch.Tensor, e_u: torch.Tensor) -> torch.Tensor: + """Alg 2. c0: (N,d) control embeddings; e_u: (1,m) or (N,m). returns endpoint (N,d).""" + if e_u.shape[0] == 1 and c0.shape[0] > 1: + e_u = e_u.expand(c0.shape[0], -1) + return model.endpoint_from_e(c0, e_u) + + +@torch.no_grad() +def endpoint_ranking(model: PIVOT, data, candidate_labels, c0: torch.Tensor, reward, + topk: int | None = None, device="cpu", chunk: int = 256): + """Alg 3. score each candidate by reward of its predicted endpoint population. + + batched: all candidates x control cells go through the flow map in one + (chunked) forward pass, then per-row rewards are averaged per candidate. works + for any per-row reward (point/classifier). returns list of (label, score) desc. + """ + E = torch.as_tensor(candidate_embeddings(model, data, candidate_labels, device), device=device) + C, N = E.shape[0], c0.shape[0] + scores = np.empty(C, dtype=np.float64) + # chunk over candidates to bound memory (c*n rows per chunk) + cands_per_chunk = max(1, chunk * 1024 // max(N, 1)) + for s in range(0, C, cands_per_chunk): + e = E[s: s + cands_per_chunk] + c = e.shape[0] + c0_rep = c0.unsqueeze(0).expand(c, N, -1).reshape(c * N, -1) + e_rep = e.unsqueeze(1).expand(c, N, -1).reshape(c * N, -1) + chat = model.endpoint_from_e(c0_rep, e_rep) + r = reward(chat).view(c, N).mean(1) + scores[s: s + c] = r.detach().cpu().numpy() + order = np.argsort(-scores) + ranked = [(candidate_labels[i], float(scores[i])) for i in order] + return ranked if topk is None else ranked[:topk] + + +def reward_guidance(model: PIVOT, c0: torch.Tensor, reward, e_init: torch.Tensor, + steps: int = 25, step_size: float = 0.5, eps: float = 1e-8, + normalize: bool = True) -> torch.Tensor: + """Alg 4. gradient ascent on reward in embedding space via the flow-map jacobian. + + e_{l+1} = e_l + gamma * g_e/(norm(g_e)+eps), g_e from autograd.""" + e = e_init.detach().clone() + for _ in range(steps): + e.requires_grad_(True) + chat = model.endpoint_from_e(c0, e.expand(c0.shape[0], -1)) + r = reward(chat).mean() + (g_e,) = torch.autograd.grad(r, e) + with torch.no_grad(): + step = g_e / (g_e.norm() + eps) if normalize else g_e + e = e + step_size * step + e = e.detach() + return e + + +@torch.no_grad() +def project_and_rerank(model: PIVOT, data, candidate_labels, e_star: torch.Tensor, + c0: torch.Tensor, reward, k_nearest: int = 10, topk: int = 5, + device="cpu"): + """Alg 5. project e* to the k nearest admissible candidates, rerank by endpoint reward.""" + E = torch.as_tensor(candidate_embeddings(model, data, candidate_labels, device), device=device) + d = torch.cdist(e_star.view(1, -1), E).squeeze(0) + knn = torch.argsort(d)[:k_nearest].cpu().numpy() + sub = [candidate_labels[i] for i in knn] + ranked = endpoint_ranking(model, data, sub, c0, reward, device=device) + return ranked[:topk] + + +@torch.no_grad() +def greedy_combinatorial(model: PIVOT, data, gene_pool, c0: torch.Tensor, reward, + max_size: int = 2, min_gain: float = 1e-4, device="cpu"): + """Alg 6. greedily add the gene that most improves the reward of the mean-pooled endpoint.""" + chosen: list[str] = [] + best_score = -np.inf + history = [] + while len(chosen) < max_size: + remaining = [g for g in gene_pool if g not in chosen] + if not remaining: + break + # score all candidate gene-sets {chosen + g} in one batched pass + labels = [data.sep.join(chosen + [g]) for g in remaining] + ranked = endpoint_ranking(model, data, labels, c0, reward, device=device) + best_label, best_local = ranked[0] + best_gene = remaining[labels.index(best_label)] + if best_local - best_score < min_gain: + break + chosen.append(best_gene) + best_score = best_local + history.append((list(chosen), best_score)) + return chosen, best_score, history diff --git a/src/evaluation/metrics.py b/src/evaluation/metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..69d2ff2c02d46c085c90696aa64f8bd4ad2d46ce --- /dev/null +++ b/src/evaluation/metrics.py @@ -0,0 +1,156 @@ +"""shared metric library. built once, used by every table. + +pointwise/gene-level: mse, r2, pearson, spearman, de-gene correlation. +distributional: mmd (rbf), sliced-wasserstein. +nomination: top-k, ndcg@k (graded), true/median rank, functional (pathway) recovery, +partial overlap (combos), synergy correlation. +""" +from __future__ import annotations + +import numpy as np +from scipy.stats import pearsonr, spearmanr + + +# ---------- pointwise / gene-level ---------- +def mse(pred, true): + return float(np.mean((np.asarray(pred) - np.asarray(true)) ** 2)) + + +def r2(pred, true): + true = np.asarray(true); pred = np.asarray(pred) + ss_res = np.sum((true - pred) ** 2) + ss_tot = np.sum((true - true.mean()) ** 2) + 1e-12 + return float(1 - ss_res / ss_tot) + + +def pearson(pred, true): + pred, true = np.asarray(pred).ravel(), np.asarray(true).ravel() + if pred.std() < 1e-12 or true.std() < 1e-12: + return 0.0 + return float(pearsonr(pred, true)[0]) + + +def spearman(pred, true): + pred, true = np.asarray(pred).ravel(), np.asarray(true).ravel() + if pred.std() < 1e-12 or true.std() < 1e-12: + return 0.0 + return float(spearmanr(pred, true)[0]) + + +def de_gene_correlation(pred_effect, true_effect, k: int = 20): + """pearson r over the top-k differentially-expressed genes (by |true effect|).""" + true_effect = np.asarray(true_effect).ravel() + pred_effect = np.asarray(pred_effect).ravel() + de = np.argsort(-np.abs(true_effect))[:k] + return pearson(pred_effect[de], true_effect[de]) + + +# ---------- distributional ---------- +def mmd_rbf(X, Y, gamma: float | None = None, max_n: int = 500, seed: int = 0): + rng = np.random.default_rng(seed) + X = np.asarray(X); Y = np.asarray(Y) + if len(X) > max_n: + X = X[rng.choice(len(X), max_n, replace=False)] + if len(Y) > max_n: + Y = Y[rng.choice(len(Y), max_n, replace=False)] + if gamma is None: + from scipy.spatial.distance import cdist + med = np.median(cdist(X[:200], Y[:200]) ** 2) + 1e-9 + gamma = 1.0 / med + from scipy.spatial.distance import cdist + Kxx = np.exp(-gamma * cdist(X, X) ** 2) + Kyy = np.exp(-gamma * cdist(Y, Y) ** 2) + Kxy = np.exp(-gamma * cdist(X, Y) ** 2) + return float(Kxx.mean() + Kyy.mean() - 2 * Kxy.mean()) + + +def sliced_wasserstein(X, Y, n_proj: int = 50, seed: int = 0): + """sliced-wasserstein-1 distance (multivariate, standard approximation).""" + rng = np.random.default_rng(seed) + X = np.asarray(X); Y = np.asarray(Y) + d = X.shape[1] + dirs = rng.normal(size=(n_proj, d)) + dirs /= np.linalg.norm(dirs, axis=1, keepdims=True) + 1e-12 + tot = 0.0 + for w in dirs: + xp = np.sort(X @ w); yp = np.sort(Y @ w) + m = min(len(xp), len(yp)) + xq = np.quantile(xp, np.linspace(0, 1, m)) + yq = np.quantile(yp, np.linspace(0, 1, m)) + tot += np.mean(np.abs(xq - yq)) + return float(tot / n_proj) + + +# ---------- nomination ---------- +def top_k_accuracy(ranked_labels, true_label, k: int) -> float: + return float(true_label in list(ranked_labels)[:k]) + + +def rank_of_true(ranked_labels, true_label) -> int: + ranked = list(ranked_labels) + return ranked.index(true_label) + 1 if true_label in ranked else len(ranked) + 1 + + +def ndcg_at_k(ranked_labels, relevance: dict, k: int = 10) -> float: + """graded ndcg@k. relevance maps label -> gain (e.g. 1 exact, 0.5 same-pathway).""" + rl = list(ranked_labels)[:k] + dcg = sum(relevance.get(l, 0.0) / np.log2(i + 2) for i, l in enumerate(rl)) + ideal = sorted(relevance.values(), reverse=True)[:k] + idcg = sum(g / np.log2(i + 2) for i, g in enumerate(ideal)) + 1e-12 + return float(dcg / idcg) + + +def exact_relevance(true_label) -> dict: + return {true_label: 1.0} + + +def functional_relevance(true_label, parse_fn, gene_cluster: dict, candidate_labels) -> dict: + """graded relevance: 1.0 exact match, 0.5 if shares a functional cluster with the target.""" + tgt_genes = parse_fn(true_label) + tgt_clusters = {gene_cluster.get(g) for g in tgt_genes} + rel = {} + for l in candidate_labels: + if l == true_label: + rel[l] = 1.0 + else: + lg = parse_fn(l) + if any(gene_cluster.get(g) in tgt_clusters for g in lg): + rel[l] = max(rel.get(l, 0.0), 0.5) + return rel + + +# ---------- combinatorial ---------- +def partial_overlap(pred_genes, true_genes) -> float: + a, b = set(pred_genes), set(true_genes) + return len(a & b) / max(len(b), 1) + + +def synergy_correlation(pred_effects, additive_effects, true_effects): + """corr between predicted and true non-additive (synergy) components.""" + pred_syn = np.asarray(pred_effects) - np.asarray(additive_effects) + true_syn = np.asarray(true_effects) - np.asarray(additive_effects) + return pearson(pred_syn.ravel(), true_syn.ravel()) + + +# ---------- bootstrap CI / paired test ---------- +def bootstrap_ci(values, n_boot: int = 2000, alpha: float = 0.05, seed: int = 0): + rng = np.random.default_rng(seed) + values = np.asarray(values, dtype=float) + means = [rng.choice(values, len(values), replace=True).mean() for _ in range(n_boot)] + lo, hi = np.quantile(means, [alpha / 2, 1 - alpha / 2]) + return float(values.mean()), float(lo), float(hi) + + +def paired_bootstrap_pvalue(a, b, n_boot: int = 2000, seed: int = 0): + """two-sided p-value that mean(a) != mean(b) for paired samples a,b.""" + rng = np.random.default_rng(seed) + a = np.asarray(a, float); b = np.asarray(b, float) + diff = a - b + obs = diff.mean() + centered = diff - obs + count = 0 + for _ in range(n_boot): + s = rng.choice(centered, len(centered), replace=True).mean() + if abs(s) >= abs(obs): + count += 1 + return float((count + 1) / (n_boot + 1)), float(obs) diff --git a/src/evaluation/rewards.py b/src/evaluation/rewards.py new file mode 100644 index 0000000000000000000000000000000000000000..abf72ec761142ef070d2522134e26e287bd61a3f --- /dev/null +++ b/src/evaluation/rewards.py @@ -0,0 +1,128 @@ +"""reward definitions for desired-state nomination. + +all rewards take predicted endpoints chat (B, d) and return a per-row reward +(B,) that is differentiable in chat (so the flow-map jacobian can pull the +gradient back into perturbation-embedding space, Alg 4). higher = better. + +definitions: + target r = -dist(endpoint, c*)^2 (point target) + centroid r = -dist(endpoint, centroid)^2 (target-distribution centroid) + nn_target r = -min_j dist(endpoint, c*_j)^2 (nearest target sample) + mmd r = -mmd^2(pred batch, target) (distributional, batch-level) + wasserstein r = -sinkhorn(pred batch, target) (distributional, batch-level) + classifier r = log p(y* | endpoint) (target-state classifier) + combined r = alpha_dist*r_target + alpha_clf*r_classifier +""" +from __future__ import annotations + +import numpy as np +import torch +import torch.nn as nn + + +class TargetStateClassifier(nn.Module): + """small mlp classifying whether an embedding belongs to the target state. + + trained per target: positives = target-perturbation cells, negatives = + control + random other-perturbation cells. used for r_clf and the target-clf metric.""" + + def __init__(self, d: int, hidden: int = 128): + super().__init__() + self.net = nn.Sequential( + nn.Linear(d, hidden), nn.ReLU(), nn.Linear(hidden, hidden), nn.ReLU(), + nn.Linear(hidden, 1), + ) + + def forward(self, x): + return self.net(x).squeeze(-1) + + def fit(self, pos: np.ndarray, neg: np.ndarray, device, epochs: int = 200, lr: float = 1e-3): + X = torch.as_tensor(np.concatenate([pos, neg]), dtype=torch.float32, device=device) + y = torch.as_tensor(np.concatenate([np.ones(len(pos)), np.zeros(len(neg))]), + dtype=torch.float32, device=device) + opt = torch.optim.Adam(self.parameters(), lr=lr) + lossf = nn.BCEWithLogitsLoss() + self.train() + for _ in range(epochs): + opt.zero_grad() + loss = lossf(self(X), y) + loss.backward() + opt.step() + self.eval() + return self + + +def _rbf_mmd2(x, y, gamma=None): + """biased rbf mmd^2 between point sets x (n,d), y (m,d).""" + if gamma is None: + with torch.no_grad(): + d2 = torch.cdist(x[: min(200, len(x))], y[: min(200, len(y))]).pow(2) + med = d2.median().clamp(min=1e-6) + gamma = 1.0 / med + def k(a, b): + return torch.exp(-gamma * torch.cdist(a, b).pow(2)) + return k(x, x).mean() + k(y, y).mean() - 2 * k(x, y).mean() + + +def _sinkhorn(x, y, eps=0.1, iters=50): + """entropic ot (sinkhorn) cost between empirical x (n,d), y (m,d).""" + C = torch.cdist(x, y).pow(2) + n, m = C.shape + a = torch.full((n,), 1.0 / n, device=x.device) + b = torch.full((m,), 1.0 / m, device=x.device) + K = torch.exp(-C / eps) + u = torch.ones_like(a) + for _ in range(iters): + v = b / (K.t() @ u + 1e-8) + u = a / (K @ v + 1e-8) + P = torch.diag(u) @ K @ torch.diag(v) + return (P * C).sum() + + +class Reward: + """configurable reward.""" + + def __init__(self, kind: str = "centroid", target_c=None, target_sample=None, + classifier: TargetStateClassifier | None = None, + alpha_dist: float = 1.0, alpha_clf: float = 1.0, device="cpu", + control_ref=None): + self.kind = kind + self.device = device + self.alpha_dist = alpha_dist + self.alpha_clf = alpha_clf + self.classifier = classifier + self.target_c = (torch.as_tensor(target_c, dtype=torch.float32, device=device) + if target_c is not None else None) + self.target_sample = (torch.as_tensor(target_sample, dtype=torch.float32, device=device) + if target_sample is not None else None) + # control reference (mean control embedding) for direction-aware rewards + self.control_ref = (torch.as_tensor(control_ref, dtype=torch.float32, device=device) + if control_ref is not None else None) + + def __call__(self, chat: torch.Tensor) -> torch.Tensor: + k = self.kind + if k in ("target", "centroid"): + return -((chat - self.target_c) ** 2).sum(-1) + if k == "cosine": + # direction-aware: cosine between predicted effect and target effect + cref = self.control_ref if self.control_ref is not None else chat.mean(0, keepdim=True) + pe = chat - cref + te = (self.target_c - cref).view(1, -1) + pe = pe / (pe.norm(dim=-1, keepdim=True) + 1e-8) + te = te / (te.norm(dim=-1, keepdim=True) + 1e-8) + return (pe * te).sum(-1) + if k == "nn_target": + d2 = torch.cdist(chat, self.target_sample).pow(2) + return -d2.min(dim=1).values + if k == "mmd": + return -_rbf_mmd2(chat, self.target_sample).expand(chat.shape[0]) + if k == "wasserstein": + return -_sinkhorn(chat, self.target_sample).expand(chat.shape[0]) + if k == "classifier": + logit = self.classifier(chat) + return torch.nn.functional.logsigmoid(logit) + if k == "combined": + rt = -((chat - self.target_c) ** 2).sum(-1) + rc = torch.nn.functional.logsigmoid(self.classifier(chat)) + return self.alpha_dist * rt + self.alpha_clf * rc + raise ValueError(k) diff --git a/src/experiments/__init__.py b/src/experiments/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/experiments/combinatorial_eval.py b/src/experiments/combinatorial_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..88c60c0fb80f2e873688b58241a5d4ad99f79a6b --- /dev/null +++ b/src/experiments/combinatorial_eval.py @@ -0,0 +1,68 @@ +"""combinatorial nomination eval (tables 7, 19). + +target is a held-out multi-gene perturbation. methods: pivot greedy (alg 6), +pivot endpoint-ranking over the observed-combo vocab, and optionally guidance. +metrics: exact top-1/top-5 (gene-set match), partial overlap, endpoint distance, +ndcg@10, and synergy correlation vs the additive baseline. +""" +from __future__ import annotations + +import numpy as np + +from src.evaluation import inference as inf +from src.evaluation import metrics as M +from src.evaluation.rewards import Reward +from src.experiments.nomination_eval import numpy_score, rank_candidates + + +def evaluate_combinatorial(pivot_predictor, data, targets, combo_candidates, gene_pool, + control_pool, model, device, reward_kind="centroid", + n_ctrl=128, max_size=2, additive=None, seed=0): + import torch + + rng = np.random.default_rng(seed) + ctrl_all = data.emb[control_pool] + res = {k: [] for k in ["greedy_exact1", "greedy_exact5", "greedy_overlap", + "rank_exact1", "rank_exact5", "rank_ndcg", "rank_overlap", + "endpoint_dist", "synergy_corr"]} + for p in targets: + true_genes = set(data.parse(p)) + c_star = data.emb[data.pert_to_idx[p]].mean(0) + c0 = ctrl_all[rng.choice(len(ctrl_all), min(n_ctrl, len(ctrl_all)), replace=False)] + sk = dict(kind=reward_kind, c_star=c_star, target_sample=data.emb[data.pert_to_idx[p]], + device=device) + + # greedy (alg 6) + c0t = torch.as_tensor(c0, dtype=torch.float32, device=device) + rew = Reward(reward_kind, target_c=c_star, target_sample=data.emb[data.pert_to_idx[p]], + device=device) + chosen, _, _ = inf.greedy_combinatorial(model, data, gene_pool, c0t, rew, + max_size=max_size, device=device) + res["greedy_exact1"].append(float(set(chosen) == true_genes)) + res["greedy_overlap"].append(M.partial_overlap(chosen, true_genes)) + + # ranking over observed combos + ranked, _ = rank_candidates(pivot_predictor, combo_candidates, c0, sk) + res["rank_exact1"].append(M.top_k_accuracy(ranked, p, 1)) + res["rank_exact5"].append(M.top_k_accuracy(ranked, p, 5)) + res["rank_ndcg"].append(M.ndcg_at_k(ranked, M.exact_relevance(p), 10)) + # partial overlap of best-ranked combo's genes + res["rank_overlap"].append(M.partial_overlap(data.parse(ranked[0]), true_genes)) + # greedy only yields one set, so reuse exact1 as the top-5 proxy + res["greedy_exact5"].append(res["greedy_exact1"][-1]) + + top1_pop = pivot_predictor.population(ranked[0], c0) + res["endpoint_dist"].append(float(np.linalg.norm(top1_pop.mean(0) - c_star))) + + # synergy: predicted vs true non-additive component (gene space), if additive given + if additive is not None: + comp = data.pca_components + pred_eff = (pivot_predictor.population(p, c0).mean(0) - ctrl_all.mean(0)) @ comp + true_eff = (c_star - ctrl_all.mean(0)) @ comp + add_eff = additive.predict_effect(p) @ comp + res["synergy_corr"].append(M.synergy_correlation(pred_eff, add_eff, true_eff)) + + agg = {k: (float(np.mean(v)) if v else None) for k, v in res.items()} + agg["_per"] = res + agg["n_targets"] = len(targets) + return agg diff --git a/src/experiments/forward_eval.py b/src/experiments/forward_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..60e9bba68ee481e69ac90c53376ca1b60fc774e2 --- /dev/null +++ b/src/experiments/forward_eval.py @@ -0,0 +1,59 @@ +"""forward response prediction eval (tables 2,3,10,11). + +for each test perturbation: push a matched control sample through the predictor to +get a predicted post-perturbation population, then compare to the true one. +gene-space metrics use the pca decoder (affine, so mean-then-decode = +decode-then-mean). distributional metrics are computed in embedding space. +""" +from __future__ import annotations + +import numpy as np + +from src.evaluation import metrics as M + + +def _gene_effect(mean_emb, control_mean_emb, components): + """affine decode of an embedding-space shift to hvg gene-space effect.""" + return (mean_emb - control_mean_emb) @ components + + +def evaluate_forward(predictor, data, test_perts, control_idx_pool, n_ctrl=200, + de_k=20, seed=0, max_perts=None, dist_cap=300): + """aggregated forward metrics over test_perts. + + dist_cap subsamples populations for the quadratic mmd / wasserstein metrics.""" + rng = np.random.default_rng(seed) + comp = data.pca_components # (d, h) + ctrl_emb_all = data.emb[control_idx_pool] + ctrl_mean_emb = ctrl_emb_all.mean(0) + perts = list(test_perts) + if max_perts: + perts = perts[:max_perts] + + rows = {k: [] for k in ["mse", "r2", "pearson", "spearman", "de_corr", "mmd", "wasserstein"]} + for p in perts: + true_idx = data.pert_to_idx[p] + true_pop = data.emb[true_idx] + c0 = ctrl_emb_all[rng.choice(len(ctrl_emb_all), min(n_ctrl, len(ctrl_emb_all)), replace=False)] + pred_pop = predictor.population(p, c0) + + pred_eff = _gene_effect(pred_pop.mean(0), ctrl_mean_emb, comp) + true_eff = _gene_effect(true_pop.mean(0), ctrl_mean_emb, comp) + # decoded gene-space expression vectors for mse/r2 + pred_genes = pred_pop.mean(0) @ comp + data.pca_mean + true_genes = true_pop.mean(0) @ comp + data.pca_mean + + rows["mse"].append(M.mse(pred_genes, true_genes)) + rows["r2"].append(M.r2(pred_eff, true_eff)) + rows["pearson"].append(M.pearson(pred_eff, true_eff)) + rows["spearman"].append(M.spearman(pred_eff, true_eff)) + rows["de_corr"].append(M.de_gene_correlation(pred_eff, true_eff, k=de_k)) + tp = true_pop if len(true_pop) <= dist_cap else true_pop[rng.choice(len(true_pop), dist_cap, replace=False)] + pp = pred_pop if len(pred_pop) <= dist_cap else pred_pop[rng.choice(len(pred_pop), dist_cap, replace=False)] + rows["mmd"].append(M.mmd_rbf(pp, tp, seed=seed)) + rows["wasserstein"].append(M.sliced_wasserstein(pp, tp, seed=seed)) + + out = {k: float(np.mean(v)) for k, v in rows.items()} + out["_per_pert"] = {k: rows[k] for k in rows} # kept for bootstrap cis / paired tests + out["n_perts"] = len(perts) + return out diff --git a/src/experiments/make_figures.py b/src/experiments/make_figures.py new file mode 100644 index 0000000000000000000000000000000000000000..abd96e498661c25c6395b54d504922970efe98f5 --- /dev/null +++ b/src/experiments/make_figures.py @@ -0,0 +1,115 @@ +"""publication-quality figures. saves each one as both png and pdf to figures/. +reads only computed artifacts (result jsons, trained model). +""" +from __future__ import annotations + +import json +import os + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np + +plt.rcParams.update({ + "font.size": 12, "axes.labelsize": 14, "axes.titlesize": 14, + "xtick.labelsize": 11, "ytick.labelsize": 11, "legend.fontsize": 10, + "figure.figsize": (8, 6), "figure.dpi": 150, "savefig.dpi": 300, + "savefig.bbox": "tight", "axes.spines.top": False, "axes.spines.right": False, +}) +FIG = "figures" +RES = "experiments/results" + + +def _save(fig, name): + os.makedirs(FIG, exist_ok=True) + fig.savefig(os.path.join(FIG, f"{name}.png")) + fig.savefig(os.path.join(FIG, f"{name}.pdf")) + plt.close(fig) + print(f" figure -> {FIG}/{name}.{{png,pdf}}") + + +def _load(name): + with open(os.path.join(RES, f"{name}.json")) as fh: + return json.load(fh) + + +def fig_embedding_panels(model, data, targets, device, name="fig1_embedding_panels"): + """figure 1: control / target / pivot-predicted-endpoint / observed cells in 2d.""" + import torch + from src.evaluation import inference as inf + + # 2d projection: first 2 pcs of the cell-state embedding + proj = data.emb[:, :2] + fig, axes = plt.subplots(1, len(targets), figsize=(5 * len(targets), 4.5)) + if len(targets) == 1: + axes = [axes] + rng = np.random.default_rng(0) + ctrl = data.control_idx + c0e = torch.as_tensor(data.emb[rng.choice(ctrl, 300, replace=False)], dtype=torch.float32, device=device) + for ax, p in zip(axes, targets): + ci = rng.choice(ctrl, 400, replace=False) + ti = data.pert_to_idx[p] + ax.scatter(proj[ci, 0], proj[ci, 1], s=6, c="#bbbbbb", alpha=0.5, label="control") + ax.scatter(proj[ti, 0], proj[ti, 1], s=8, c="#1f77b4", alpha=0.6, label=f"observed {p}") + # pivot predicted endpoints for the true perturbation + e = inf.encode_label(model, data, p, device) + chat = inf.forward_predict(model, c0e, e).cpu().numpy() + ax.scatter(chat[:, 0], chat[:, 1], s=10, c="#d62728", alpha=0.7, marker="x", label="PIVOT predicted") + ax.set_title(p) + ax.set_xlabel("PC1"); ax.set_ylabel("PC2") + ax.legend(loc="best", markerscale=1.5) + fig.suptitle("Control, observed perturbed, and PIVOT-predicted endpoints (PC space)") + _save(fig, name) + + +def fig_guidance_steps(dataset="norman", name="fig_guidance_steps"): + obj = _load(f"{dataset}_ablation_guidance_steps") + steps = [int(k) for k in obj["rows"]] + top5 = [obj["rows"][str(s)]["top5"] for s in steps] + ndcg = [obj["rows"][str(s)]["ndcg"] for s in steps] + fig, ax = plt.subplots() + ax.plot(steps, top5, "o-", label="Top-5") + ax.plot(steps, ndcg, "s--", label="nDCG@10") + ax.set_xlabel("Reward-guidance steps L"); ax.set_ylabel("Recovery metric") + ax.set_title("Effect of reward-guidance steps") + ax.legend() + _save(fig, name) + + +def fig_datascale(dataset="norman", name="fig_datascale"): + obj = _load(f"{dataset}_ablation_datascale") + fr = sorted(float(k) for k in obj["rows"]) + de = [obj["rows"][str(f)]["forward"]["de_corr"] for f in fr] + t5 = [obj["rows"][str(f)]["inverse"]["top5"] for f in fr] + fig, ax = plt.subplots() + ax.plot([f * 100 for f in fr], de, "o-", label="Forward DE-corr") + ax.plot([f * 100 for f in fr], t5, "s--", label="Inverse Top-5") + ax.set_xlabel("Training data (%)"); ax.set_ylabel("Metric") + ax.set_title("Data scaling") + ax.legend() + _save(fig, name) + + +def fig_components(dataset="norman", name="fig_components"): + obj = _load(f"{dataset}_ablation_components") + rows = list(obj["rows"]) + t5 = [obj["rows"][r]["inverse"]["top5"] for r in rows] + de = [obj["rows"][r]["forward"]["de_corr"] for r in rows] + x = np.arange(len(rows)); w = 0.38 + fig, ax = plt.subplots(figsize=(9, 5)) + ax.bar(x - w / 2, de, w, label="Forward DE-corr") + ax.bar(x + w / 2, t5, w, label="Inverse Top-5") + ax.set_xticks(x); ax.set_xticklabels(rows, rotation=20, ha="right") + ax.set_ylabel("Metric"); ax.set_title("Component ablation") + ax.legend() + _save(fig, name) + + +def fig_loss_curves(history, name="fig_loss_curves"): + fig, ax = plt.subplots() + for k in ["total", "map", "tan", "semi"]: + ax.plot([h[k] for h in history], label=k) + ax.set_xlabel("Epoch"); ax.set_ylabel("Loss"); ax.set_yscale("log") + ax.set_title("PIVOT training losses"); ax.legend() + _save(fig, name) diff --git a/src/experiments/nomination_eval.py b/src/experiments/nomination_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..3ca38fc63518ec08b40e81c378643bc6ea0def1f --- /dev/null +++ b/src/experiments/nomination_eval.py @@ -0,0 +1,161 @@ +"""desired-state nomination eval (tables 4-7,13,14). + +a target state is induced by a held-out perturbation p: c* = centroid of p's cells. +given c*, a method nominates perturbations; we score exact/functional recovery, +ranking quality, endpoint distance, and specificity. + +ranking uses a numpy reward scorer (works for any predictor). pivot reward guidance +(Alg 4) also uses the differentiable torch reward and the model. +""" +from __future__ import annotations + +import numpy as np +import torch + +from src.evaluation import inference as inf +from src.evaluation import metrics as M +from src.evaluation.rewards import Reward + + +# numpy reward scorers (for ranking; mirror rewards.py) +def numpy_score(pop: np.ndarray, kind: str, c_star=None, target_sample=None, classifier=None, + device="cpu", control_ref=None): + if kind in ("target", "centroid", "combined"): + s = -np.mean(np.sum((pop - c_star) ** 2, axis=1)) + if kind != "combined": + return float(s) + if kind == "cosine": + cref = control_ref if control_ref is not None else pop.mean(0) + pe = pop - cref + te = c_star - cref + pe = pe / (np.linalg.norm(pe, axis=1, keepdims=True) + 1e-8) + te = te / (np.linalg.norm(te) + 1e-8) + return float(np.mean(pe @ te)) + if kind == "nn_target": + from scipy.spatial.distance import cdist + return float(-np.mean(cdist(pop, target_sample).min(axis=1) ** 2)) + if kind == "mmd": + return float(-M.mmd_rbf(pop, target_sample)) + if kind == "wasserstein": + return float(-M.sliced_wasserstein(pop, target_sample)) + if kind in ("classifier", "combined"): + with torch.no_grad(): + logit = classifier(torch.as_tensor(pop, dtype=torch.float32, device=device)) + rc = torch.nn.functional.logsigmoid(logit).mean().item() + if kind == "classifier": + return float(rc) + return float(s + rc) # combined + raise ValueError(kind) + + +_PER_ROW_REWARDS = ("centroid", "target", "nn_target", "classifier", "combined", "cosine") + + +def rank_candidates(predictor, candidates, c0, score_kwargs): + """rank candidates by reward of predicted endpoint. uses the batched gpu path for + pivot with per-row rewards; numpy per-candidate otherwise (baselines, mmd/wass).""" + from src.experiments.predictors import PivotPredictor + from src.evaluation import inference as inf + + kind = score_kwargs["kind"] + if isinstance(predictor, PivotPredictor) and kind in _PER_ROW_REWARDS: + dev = score_kwargs.get("device", "cpu") + rew = Reward(kind, target_c=score_kwargs.get("c_star"), + target_sample=score_kwargs.get("target_sample"), + classifier=score_kwargs.get("classifier"), device=dev, + control_ref=score_kwargs.get("control_ref")) + c0t = torch.as_tensor(c0, dtype=torch.float32, device=dev) + ranked = inf.endpoint_ranking(predictor.model, predictor.data, candidates, c0t, rew, device=dev) + return [l for l, _ in ranked], np.array([s for _, s in ranked]) + scores = np.array([numpy_score(predictor.population(c, c0), **score_kwargs) for c in candidates]) + order = np.argsort(-scores) + return [candidates[i] for i in order], scores[order] + + +def build_target(data, p, control_mean_emb): + return data.emb[data.pert_to_idx[p]].mean(0), data.emb[data.pert_to_idx[p]] + + +def evaluate_nomination(predictor, data, targets, candidates, control_pool, + reward_kind="centroid", method="ranking", n_ctrl=128, + gene_cluster=None, model=None, device="cpu", + guidance_steps=25, guidance_step=0.5, k_nearest=10, + classifier=None, seed=0, + guidance_init="warm", guidance_normalize=True, rerank=True): + """evaluate single-perturbation nomination. returns aggregated metric dict. + + method: 'ranking' (Alg 3) | 'guidance' (Alg 4+5, pivot only). + guidance_init: 'warm' (top-ranked) | 'random' | 'mean_top' (mean of top-k cand embeddings). + guidance_normalize: normalize the jacobian-pullback step (Alg 4). + rerank: if true, rerank k-nn by endpoint reward (Alg 5); else project to single nearest. + """ + rng = np.random.default_rng(seed) + ctrl_all = data.emb[control_pool] + ctrl_mean = ctrl_all.mean(0) + res = {k: [] for k in ["top1", "top5", "ndcg", "func_top5", "func_ndcg", + "rank", "endpoint_dist", "spec_margin", "off_dist", "clf"]} + for p in targets: + c_star, tgt_sample = build_target(data, p, ctrl_mean) + c0 = ctrl_all[rng.choice(len(ctrl_all), min(n_ctrl, len(ctrl_all)), replace=False)] + sk = dict(kind=reward_kind, c_star=c_star, target_sample=tgt_sample, + classifier=classifier, device=device, control_ref=ctrl_mean) + + ranked, _ = rank_candidates(predictor, candidates, c0, sk) + + if method == "guidance": + assert model is not None + c0t = torch.as_tensor(c0, dtype=torch.float32, device=device) + rew = Reward(reward_kind if reward_kind != "combined" else "centroid", + target_c=c_star, target_sample=tgt_sample, + classifier=classifier, device=device, control_ref=ctrl_mean) + if guidance_init == "random": + e_init = inf.encode_label(model, data, candidates[rng.integers(len(candidates))], device) + elif guidance_init == "mean_top": + from src.models.pivot import candidate_embeddings + E = candidate_embeddings(model, data, ranked[:k_nearest], device) + e_init = torch.as_tensor(E.mean(0, keepdims=True), device=device) + else: # warm + e_init = inf.encode_label(model, data, ranked[0], device) + e_star = inf.reward_guidance(model, c0t, rew, e_init, steps=guidance_steps, + step_size=guidance_step, normalize=guidance_normalize) + if rerank: + top = inf.project_and_rerank(model, data, candidates, e_star, c0t, rew, + k_nearest=k_nearest, topk=k_nearest, device=device) + guided = [l for l, _ in top] + else: + from src.models.pivot import candidate_embeddings + E = torch.as_tensor(candidate_embeddings(model, data, candidates, device), device=device) + nn = torch.argsort(torch.cdist(e_star.view(1, -1), E).squeeze(0)).cpu().numpy() + guided = [candidates[i] for i in nn[:k_nearest]] + ranked = guided + [r for r in ranked if r not in guided] # full order for rank metrics + + # metrics + res["top1"].append(M.top_k_accuracy(ranked, p, 1)) + res["top5"].append(M.top_k_accuracy(ranked, p, 5)) + res["ndcg"].append(M.ndcg_at_k(ranked, M.exact_relevance(p), 10)) + res["rank"].append(M.rank_of_true(ranked, p)) + if gene_cluster is not None: + frel = M.functional_relevance(p, data.parse, gene_cluster, candidates) + res["func_top5"].append(float(any(frel.get(l, 0) >= 0.5 for l in ranked[:5]))) + res["func_ndcg"].append(M.ndcg_at_k(ranked, frel, 10)) + # endpoint distance of nominated top-1 + specificity + top1_pop = predictor.population(ranked[0], c0) + res["endpoint_dist"].append(float(np.linalg.norm(top1_pop.mean(0) - c_star))) + # off-state distance: mean dist of top-1 endpoint to other perturbation centroids + others = rng.choice([q for q in candidates if q != p], size=min(20, len(candidates) - 1), + replace=False) + off = np.mean([np.linalg.norm(top1_pop.mean(0) - data.emb[data.pert_to_idx[q]].mean(0)) + for q in others]) + res["off_dist"].append(float(off)) + res["spec_margin"].append(float(off - np.linalg.norm(top1_pop.mean(0) - c_star))) + if classifier is not None: + with torch.no_grad(): + res["clf"].append(float(torch.sigmoid( + classifier(torch.as_tensor(top1_pop, dtype=torch.float32, device=device)) + ).mean().item())) + + agg = {k: (float(np.mean(v)) if v else None) for k, v in res.items()} + agg["_per"] = res + agg["n_targets"] = len(targets) + agg["random_top5"] = 5.0 / len(candidates) + return agg diff --git a/src/experiments/predictors.py b/src/experiments/predictors.py new file mode 100644 index 0000000000000000000000000000000000000000..c4e5f889b5e20067e289a21da3d9ad3443582b15 --- /dev/null +++ b/src/experiments/predictors.py @@ -0,0 +1,41 @@ +"""uniform predictor interface so pivot and every baseline run through the same +forward/inverse code. a predictor maps (perturbation label, control-cell embeddings +c0) -> predicted post-perturbation population ĉ1 in embedding space. +""" +from __future__ import annotations + +import numpy as np +import torch + +from src.evaluation import inference as inf + + +class PivotPredictor: + name = "PIVOT" + + def __init__(self, model, data, device): + self.model, self.data, self.device = model, data, device + model.eval() + + def population(self, label: str, c0: np.ndarray) -> np.ndarray: + c0t = torch.as_tensor(c0, dtype=torch.float32, device=self.device) + e = inf.encode_label(self.model, self.data, label, self.device) + return inf.forward_predict(self.model, c0t, e).cpu().numpy() + + def population_from_genes(self, genes: list[str], c0: np.ndarray) -> np.ndarray: + c0t = torch.as_tensor(c0, dtype=torch.float32, device=self.device) + e = inf.encode_gene_set(self.model, self.data, genes, self.device) + return inf.forward_predict(self.model, c0t, e).cpu().numpy() + + +class BaselinePredictor: + def __init__(self, baseline): + self.bl = baseline + self.name = baseline.name + + def population(self, label: str, c0: np.ndarray) -> np.ndarray: + return self.bl.predict_endpoint(label, c0) + + def population_from_genes(self, genes: list[str], c0: np.ndarray) -> np.ndarray: + label = self.bl.data.sep.join(genes) if genes else self.bl.data.control_label + return self.bl.predict_endpoint(label, c0) diff --git a/src/experiments/run_ablations.py b/src/experiments/run_ablations.py new file mode 100644 index 0000000000000000000000000000000000000000..7ac8e47ec7c3bae170afe8fc08dada9fbc92f75d --- /dev/null +++ b/src/experiments/run_ablations.py @@ -0,0 +1,216 @@ +"""ablation studies (tables 8, 12-18, 20). + +every ablation is a real training/eval run on real data. epoch budget (EPOCHS) is +held fixed across rows of a table for fair comparison. +""" +from __future__ import annotations + +import os +import time + +import numpy as np +import torch + +from src.data.perturb_data import PerturbData, load_dataset +from src.data.splits import load_split +from src.training.train import TrainConfig, train +from src.evaluation.baselines import build_baseline +from src.experiments.predictors import PivotPredictor, BaselinePredictor +from src.experiments.forward_eval import evaluate_forward +from src.experiments.nomination_eval import evaluate_nomination +from src.experiments.run_tables import (RESULTS_DIR, DEVICE_INDEX, candidates_singles, + save_table) +from src.models.encoders import REP_MODES +from src.utils.common import save_json + +EPOCHS = 45 + + +def _train(data, split, **kw): + cfg = TrainConfig(dataset=data.meta["name"], split=split, epochs=EPOCHS, + device_index=DEVICE_INDEX, **kw) + model, info = train(cfg, data=data, verbose=False) + return model, info, cfg + + +def _fwd_inv(data, model, split, max_perts=50, max_targets=30, reward="centroid"): + """compact forward + inverse metrics for one model (for ablation rows).""" + device = next(model.parameters()).device + sp = load_split(data.dir, split) + cands = candidates_singles(data) + test_perts = list(sp["test_perts"]) if split != "cell" else cands + targets = [p for p in test_perts if len(data.parse(p)) == 1 and p in cands][:max_targets] + control_pool = data.control_idx + gc = data.functional_clusters(seed=0) + pred = PivotPredictor(model, data, device) + fwd = evaluate_forward(pred, data, [p for p in test_perts][:max_perts], control_pool, max_perts=max_perts) + inv = evaluate_nomination(pred, data, targets, cands, control_pool, reward_kind=reward, + method="ranking", gene_cluster=gc, device=device) + return {"forward": {k: fwd[k] for k in ["mse", "de_corr", "mmd", "pearson"]}, + "inverse": {k: inv[k] for k in ["top1", "top5", "ndcg", "endpoint_dist"]}} + + +# table 8: component ablation +def ablation_components(data, split="perturbation"): + variants = { + "flow-map-only": ["map"], + "no-tangent": ["map", "semi"], + "no-semigroup": ["map", "tan"], + "PIVOT-full": ["map", "tan", "semi"], + } + out = {"split": split, "rows": {}} + for name, comps in variants.items(): + model, info, cfg = _train(data, split, components=comps) + out["rows"][name] = _fwd_inv(data, model, split) + # endpointmlp baseline (no flow map) + sp = load_split(data.dir, split) + bl = build_baseline("EndpointMLP").fit(data, sp["train_perts"], sp["train_idx"]) + device = torch.device(f"cuda:{DEVICE_INDEX}") + cands = candidates_singles(data) + targets = [p for p in sp["test_perts"] if len(data.parse(p)) == 1 and p in cands][:30] + gc = data.functional_clusters(seed=0) + fwd = evaluate_forward(BaselinePredictor(bl), data, list(sp["test_perts"])[:50], data.control_idx, max_perts=50) + inv = evaluate_nomination(BaselinePredictor(bl), data, targets, cands, data.control_idx, + method="ranking", gene_cluster=gc, device=device) + out["rows"]["EndpointMLP"] = {"forward": {k: fwd[k] for k in ["mse", "de_corr", "mmd", "pearson"]}, + "inverse": {k: inv[k] for k in ["top1", "top5", "ndcg", "endpoint_dist"]}} + return out + + +# table 12: perturbation representation +def ablation_representation(data, split="perturbation"): + out = {"split": split, "rows": {}} + for rep in REP_MODES: + model, info, cfg = _train(data, split, rep_mode=rep) + out["rows"][rep] = _fwd_inv(data, model, split) + return out + + +# table 13: inverse-search strategy (single model) +def ablation_inverse_search(data, split="perturbation"): + model, info, cfg = _train(data, split) + device = next(model.parameters()).device + sp = load_split(data.dir, split) + cands = candidates_singles(data) + targets = [p for p in sp["test_perts"] if len(data.parse(p)) == 1 and p in cands][:30] + gc = data.functional_clusters(seed=0) + pred = PivotPredictor(model, data, device) + common = dict(reward_kind="centroid", gene_cluster=gc, model=model, device=device) + strategies = { + "ranking-only": dict(method="ranking"), + "random-init-opt": dict(method="guidance", guidance_init="random", rerank=False), + "mean-top-init": dict(method="guidance", guidance_init="mean_top", rerank=False), + "guidance-no-norm": dict(method="guidance", guidance_init="warm", guidance_normalize=False, rerank=False), + "guidance-norm": dict(method="guidance", guidance_init="warm", guidance_normalize=True, rerank=False), + "guidance+rerank": dict(method="guidance", guidance_init="warm", guidance_normalize=True, rerank=True), + } + out = {"split": split, "rows": {}} + for name, kw in strategies.items(): + r = evaluate_nomination(pred, data, targets, cands, data.control_idx, **common, **kw) + out["rows"][name] = {k: r[k] for k in ["top1", "top5", "ndcg", "endpoint_dist"]} + return out + + +# table 14: guidance steps +def ablation_guidance_steps(data, split="perturbation"): + model, info, cfg = _train(data, split) + device = next(model.parameters()).device + sp = load_split(data.dir, split) + cands = candidates_singles(data) + targets = [p for p in sp["test_perts"] if len(data.parse(p)) == 1 and p in cands][:30] + gc = data.functional_clusters(seed=0) + pred = PivotPredictor(model, data, device) + out = {"split": split, "rows": {}} + for steps in [0, 5, 10, 25, 50, 100]: + if steps == 0: + r = evaluate_nomination(pred, data, targets, cands, data.control_idx, + method="ranking", gene_cluster=gc, device=device) + else: + r = evaluate_nomination(pred, data, targets, cands, data.control_idx, method="guidance", + gene_cluster=gc, model=model, device=device, + guidance_steps=steps, rerank=True) + out["rows"][str(steps)] = {k: r[k] for k in ["top1", "top5", "ndcg", "endpoint_dist"]} + return out + + +# table 15: reward definitions +def ablation_reward(data, split="perturbation"): + model, info, cfg = _train(data, split) + device = next(model.parameters()).device + sp = load_split(data.dir, split) + cands = candidates_singles(data) + targets = [p for p in sp["test_perts"] if len(data.parse(p)) == 1 and p in cands][:25] + gc = data.functional_clusters(seed=0) + pred = PivotPredictor(model, data, device) + out = {"split": split, "rows": {}} + for rk in ["centroid", "cosine", "nn_target", "mmd", "wasserstein"]: + r = evaluate_nomination(pred, data, targets, cands, data.control_idx, reward_kind=rk, + method="ranking", gene_cluster=gc, device=device) + out["rows"][rk] = {k: r[k] for k in ["top1", "top5", "ndcg", "endpoint_dist"]} + return out + + +# table 17: data scaling +def ablation_datascale(data, split="perturbation"): + out = {"split": split, "rows": {}} + for frac in [0.1, 0.25, 0.5, 0.75, 1.0]: + model, info, cfg = _train(data, split, train_frac=frac) + out["rows"][str(frac)] = _fwd_inv(data, model, split) + return out + + +# table 18: control matching +def ablation_matching(data, split="perturbation"): + from src.data.perturb_data import MATCH_STRATEGIES + out = {"split": split, "rows": {}} + for ms in MATCH_STRATEGIES: + model, info, cfg = _train(data, split, match=ms) + out["rows"][ms] = _fwd_inv(data, model, split) + return out + + +# table 20: compute cost +def ablation_cost(data, split="perturbation"): + model, info, cfg = _train(data, split) + device = next(model.parameters()).device + n_params = sum(p.numel() for p in model.parameters()) + cands = candidates_singles(data) + pred = PivotPredictor(model, data, device) + c0 = data.emb[data.control_idx[:128]] + # pivot endpoint ranking: time per query (one target, rank all candidates) + import time as _t + t0 = _t.time() + _ = [pred.population(c, c0) for c in cands] + rank_time = _t.time() - t0 + out = {"split": split, + "PIVOT": {"params": int(n_params), "train_time_s": info["duration_s"], + "rank_time_per_query_s": rank_time, "candidate_evals": len(cands)}} + return out + + +ABLATIONS = { + "components": ablation_components, # table 8 + "representation": ablation_representation, # table 12 + "inverse_search": ablation_inverse_search, # table 13 + "guidance_steps": ablation_guidance_steps, # table 14 + "reward": ablation_reward, # table 15 + "datascale": ablation_datascale, # table 17 + "matching": ablation_matching, # table 18 + "cost": ablation_cost, # table 20 +} + + +if __name__ == "__main__": + import argparse + ap = argparse.ArgumentParser() + ap.add_argument("--dataset", default="norman") + ap.add_argument("--ablations", nargs="+", default=list(ABLATIONS.keys())) + args = ap.parse_args() + data = load_dataset(args.dataset) + t0 = time.time() + for name in args.ablations: + print(f"=== ablation: {name} ===", flush=True) + res = ABLATIONS[name](data) + save_table(f"{args.dataset}_ablation_{name}", res) + print(f" done {name} ({time.time()-t0:.0f}s elapsed)", flush=True) + print(f"all ablations done in {time.time()-t0:.0f}s") diff --git a/src/experiments/run_tables.py b/src/experiments/run_tables.py new file mode 100644 index 0000000000000000000000000000000000000000..f8686293d54b5bf7b3bdae316513010c18b13407 --- /dev/null +++ b/src/experiments/run_tables.py @@ -0,0 +1,164 @@ +"""master experiment runner, trains pivot, runs baselines, assembles every results +table. each table function returns a json-serializable dict and is saved to +experiments/results/.json. nothing is hard-coded, every number comes from +these calls. +""" +from __future__ import annotations + +import json +import os +import time + +import numpy as np +import torch + +from src.data.perturb_data import load_dataset +from src.data.splits import load_split +from src.training.train import TrainConfig, train +from src.evaluation.baselines import BASELINES, build_baseline +from src.evaluation.rewards import TargetStateClassifier +from src.experiments.predictors import PivotPredictor, BaselinePredictor +from src.experiments.forward_eval import evaluate_forward +from src.experiments.nomination_eval import evaluate_nomination +from src.experiments.combinatorial_eval import evaluate_combinatorial +from src.utils.common import save_json + +RESULTS_DIR = "experiments/results" +DEVICE_INDEX = int(os.environ.get("PIVOT_GPU", "3")) + + +class ModelCache: + """train-once cache keyed by config tuple.""" + + def __init__(self, dataset, embedding="pca"): + self.data = load_dataset(dataset, embedding=embedding) + self.dataset = dataset + self.embedding = embedding + self._cache = {} + + def get(self, split, **kw): + key = (split, self.embedding, tuple(sorted(kw.items()))) + if key in self._cache: + return self._cache[key] + cfg = TrainConfig(dataset=self.dataset, embedding=self.embedding, split=split, + device_index=DEVICE_INDEX, **kw) + model, info = train(cfg, data=self.data, verbose=False) + self._cache[key] = (model, info, cfg) + return model, info, cfg + + +def candidates_singles(data): + return [p for p in data.perturbations if len(data.parse(p)) == 1] + + +def get_classifier(data, target_perts, control_pool, device, seed=0): + """train one target-state classifier: positives = union of target cells, + negatives = control + random other-perturbation cells (for r_clf / target-clf).""" + rng = np.random.default_rng(seed) + pos = np.concatenate([data.pert_to_idx[p] for p in target_perts]) + pos = pos[rng.choice(len(pos), min(4000, len(pos)), replace=False)] + neg_pool = np.concatenate([control_pool, + np.concatenate([data.pert_to_idx[p] for p in + rng.choice(list(set(data.perturbations) - set(target_perts)), + size=min(20, len(data.perturbations)), replace=False)])]) + neg = neg_pool[rng.choice(len(neg_pool), min(4000, len(neg_pool)), replace=False)] + clf = TargetStateClassifier(data.d).to(device) + clf.fit(data.emb[pos], data.emb[neg], device) + return clf + + +def save_table(name, obj): + os.makedirs(RESULTS_DIR, exist_ok=True) + save_json(obj, os.path.join(RESULTS_DIR, f"{name}.json")) + print(f" saved {RESULTS_DIR}/{name}.json") + + +# table 2 / 3 / 10 / 11: forward prediction +def table_forward(cache: ModelCache, split: str, max_perts=80, baseline_names=None): + data = cache.data + model, info, cfg = cache.get(split) + device = next(model.parameters()).device + sp = load_split(data.dir, split) + test_perts = list(sp["test_perts"]) if split != "cell" else candidates_singles(data) + control_pool = sp["test_idx"][data.is_control[sp["test_idx"]]] + if len(control_pool) < 50: + control_pool = data.control_idx + + out = {"split": split, "models": {}} + pred = PivotPredictor(model, data, device) + out["models"]["PIVOT"] = evaluate_forward(pred, data, test_perts, control_pool, max_perts=max_perts) + for bname in (baseline_names or ["MeanControl", "AvgPerturbationEffect", "Additive", + "LinearResponse", "NearestPerturbationCentroid", "EndpointMLP", "Random"]): + bl = build_baseline(bname).fit(data, sp["train_perts"], sp["train_idx"]) + out["models"][bname] = evaluate_forward(BaselinePredictor(bl), data, test_perts, + control_pool, max_perts=max_perts) + return out + + +# table 4 / 6: single-gene desired-state nomination & recovery +def table_nomination(cache: ModelCache, split: str, max_targets=40, reward_kind="centroid", + with_guidance=True, baseline_names=None): + data = cache.data + model, info, cfg = cache.get(split) + device = next(model.parameters()).device + sp = load_split(data.dir, split) + cands = candidates_singles(data) + targets = [p for p in (list(sp["test_perts"]) if split != "cell" else cands) + if len(data.parse(p)) == 1 and p in cands][:max_targets] + control_pool = data.control_idx + gene_cluster = data.functional_clusters(seed=cfg.seed) + + out = {"split": split, "reward": reward_kind, "n_candidates": len(cands), + "n_targets": len(targets), "methods": {}} + pred = PivotPredictor(model, data, device) + out["methods"]["PIVOT-ranking"] = evaluate_nomination( + pred, data, targets, cands, control_pool, reward_kind=reward_kind, method="ranking", + gene_cluster=gene_cluster, device=device) + if with_guidance: + out["methods"]["PIVOT-guidance"] = evaluate_nomination( + pred, data, targets, cands, control_pool, reward_kind=reward_kind, method="guidance", + gene_cluster=gene_cluster, model=model, device=device) + for bname in (baseline_names or ["Additive", "LinearResponse", "NearestPerturbationCentroid", + "EndpointMLP", "Random"]): + bl = build_baseline(bname).fit(data, sp["train_perts"], sp["train_idx"]) + out["methods"][f"{bname}+ranking"] = evaluate_nomination( + BaselinePredictor(bl), data, targets, cands, control_pool, reward_kind=reward_kind, + method="ranking", gene_cluster=gene_cluster, device=device) + return out + + +# table 7: combinatorial nomination +def table_combinatorial(cache: ModelCache, split="combination", max_targets=26): + data = cache.data + model, info, cfg = cache.get(split) + device = next(model.parameters()).device + sp = load_split(data.dir, split) + targets = [p for p in sp["test_perts"] if len(data.parse(p)) == 2][:max_targets] + combo_cands = data.combos + gene_pool = [data.parse(p)[0] for p in candidates_singles(data)] + control_pool = data.control_idx + additive = build_baseline("Additive").fit(data, sp["train_perts"], sp["train_idx"]) + pred = PivotPredictor(model, data, device) + out = {"split": split, "n_targets": len(targets), "n_combo_candidates": len(combo_cands)} + out["result"] = evaluate_combinatorial(pred, data, targets, combo_cands, gene_pool, + control_pool, model, device, additive=additive) + return out + + +if __name__ == "__main__": + import argparse + ap = argparse.ArgumentParser() + ap.add_argument("--dataset", default="norman") + ap.add_argument("--tables", nargs="+", default=["forward_cell"]) + args = ap.parse_args() + cache = ModelCache(args.dataset) + t0 = time.time() + for tname in args.tables: + print(f"=== {tname} ===") + if tname.startswith("forward_"): + save_table(f"{args.dataset}_{tname}", table_forward(cache, tname.split("_", 1)[1])) + elif tname.startswith("nom_"): + save_table(f"{args.dataset}_{tname}", table_nomination(cache, tname.split("_", 1)[1])) + elif tname == "combinatorial": + save_table(f"{args.dataset}_combinatorial", table_combinatorial(cache)) + print(f"done in {time.time()-t0:.1f}s") diff --git a/src/models/__init__.py b/src/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/models/encoders.py b/src/models/encoders.py new file mode 100644 index 0000000000000000000000000000000000000000..7793fc69853526357d5a2f73c5e77197a637d13a --- /dev/null +++ b/src/models/encoders.py @@ -0,0 +1,96 @@ +"""perturbation encoder ψ_η plus a builder that turns perturbation labels into +padded (gene_id, op_id, mask) tensors. + +ψ separates gene identity from operation and is permutation-invariant over the +gene set (mean pooling), so one encoder handles single-gene and combinatorial +perturbations: + + e_u = (1/M) Σ_j ( W_g[g_j] + W_o[o_j] ) + +rep modes: gene_op (default), op_only, gene_only, random_id, gene_pathway_op. +""" +from __future__ import annotations + +import numpy as np +import torch +import torch.nn as nn + +REP_MODES = ("gene_op", "op_only", "gene_only", "random_id", "gene_pathway_op") + + +class PerturbationEncoder(nn.Module): + def __init__( + self, + n_genes: int, + n_ops: int, + n_perts: int, + emb_dim: int = 64, + mode: str = "gene_op", + gene_pathway: np.ndarray | None = None, # (n_genes,) cluster id, for gene_pathway_op mode + n_pathways: int = 0, + ): + super().__init__() + assert mode in REP_MODES, mode + self.mode = mode + self.emb_dim = emb_dim + self.W_g = nn.Embedding(n_genes, emb_dim) + self.W_o = nn.Embedding(n_ops, emb_dim) + nn.init.normal_(self.W_g.weight, std=0.02) + nn.init.normal_(self.W_o.weight, std=0.02) + if mode == "random_id": + self.W_id = nn.Embedding(n_perts + 1, emb_dim) # +1 for control/unknown + nn.init.normal_(self.W_id.weight, std=0.02) + if mode == "gene_pathway_op": + assert gene_pathway is not None and n_pathways > 0 + self.register_buffer("gene_pathway", torch.as_tensor(gene_pathway, dtype=torch.long)) + self.W_path = nn.Embedding(n_pathways, emb_dim) + nn.init.normal_(self.W_path.weight, std=0.02) + + def forward(self, gene_ids: torch.Tensor, op_ids: torch.Tensor, mask: torch.Tensor, + pert_ids: torch.Tensor | None = None) -> torch.Tensor: + """gene_ids/op_ids/mask: (B, Lmax) padded; mask 1 for valid gene slots. + + returns e_u: (B, emb_dim). control rows (mask all zero) map to the zero vector. + """ + if self.mode == "random_id": + return self.W_id(pert_ids) + + m = mask.unsqueeze(-1).float() # (b, l, 1) + if self.mode == "op_only": + per_gene = self.W_o(op_ids) + elif self.mode == "gene_only": + per_gene = self.W_g(gene_ids) + elif self.mode == "gene_pathway_op": + path_ids = self.gene_pathway[gene_ids.clamp(min=0)] + per_gene = self.W_g(gene_ids) + self.W_o(op_ids) + self.W_path(path_ids) + else: # gene_op + per_gene = self.W_g(gene_ids) + self.W_o(op_ids) + per_gene = per_gene * m + denom = m.sum(dim=1).clamp(min=1.0) # (b,1); control -> 1 to avoid /0 (sum is 0 anyway) + return per_gene.sum(dim=1) / denom + + +def build_pert_tensors(data, labels, device="cpu"): + """convert a list of perturbation labels into padded (gene_ids, op_ids, mask, pert_ids). + + pert_ids index into ``data.perturbations`` (len = control/unknown sentinel) for random_id mode. + """ + pert_index = {p: i for i, p in enumerate(data.perturbations)} + L = max((len(data.parse(p)) for p in labels), default=1) + L = max(L, 1) + B = len(labels) + g = np.zeros((B, L), dtype=np.int64) + o = np.zeros((B, L), dtype=np.int64) + mask = np.zeros((B, L), dtype=np.float32) + pid = np.full(B, len(data.perturbations), dtype=np.int64) # sentinel + for i, p in enumerate(labels): + gids, oids = data.pert_gene_op_ids(p) + n = len(gids) + if n: + g[i, :n] = gids + o[i, :n] = oids + mask[i, :n] = 1.0 + if p in pert_index: + pid[i] = pert_index[p] + t = lambda a: torch.as_tensor(a, device=device) + return t(g), t(o), t(mask), t(pid) diff --git a/src/models/flow_map.py b/src/models/flow_map.py new file mode 100644 index 0000000000000000000000000000000000000000..f25c83479393936966dce9ee879b4323f5c7faf8 --- /dev/null +++ b/src/models/flow_map.py @@ -0,0 +1,68 @@ +"""perturbation-conditioned flow map X_θ. + + X_θ(s, t, c_s, e_u) = c_s + (t - s) · v_θ(s, t, c_s, e_u), 0 ≤ s ≤ t ≤ 1 + ĉ1(u) = X_θ(0, 1, c0, e_u) + +v_θ is an mlp over [γ(s), γ(t), c_s, e_u], γ is a fourier time embedding. +the map is differentiable in e_u, which is what enables the inverse-design +jacobian ∇_{e_u} X_{0,1} used by reward guidance (Alg 4). +""" +from __future__ import annotations + +import math + +import torch +import torch.nn as nn + + +class FourierTime(nn.Module): + """map a scalar t∈[0,1] to [t, sin(2πk t), cos(2πk t) ...].""" + + def __init__(self, n_freq: int = 6): + super().__init__() + self.register_buffer("freqs", 2 * math.pi * (2.0 ** torch.arange(n_freq)).float()) + + def forward(self, t: torch.Tensor) -> torch.Tensor: + t = t.view(-1, 1) + ang = t * self.freqs.view(1, -1) + return torch.cat([t, torch.sin(ang), torch.cos(ang)], dim=-1) + + +class FlowMap(nn.Module): + def __init__(self, d_state: int, d_pert: int, hidden: int = 512, depth: int = 4, + n_freq: int = 6, dropout: float = 0.0): + super().__init__() + self.d_state = d_state + self.d_pert = d_pert + self.time = FourierTime(n_freq) + t_dim = 1 + 2 * n_freq + in_dim = 2 * t_dim + d_state + d_pert + layers = [nn.Linear(in_dim, hidden), nn.SiLU()] + for _ in range(depth - 1): + layers += [nn.Linear(hidden, hidden), nn.SiLU()] + if dropout > 0: + layers += [nn.Dropout(dropout)] + layers += [nn.Linear(hidden, d_state)] + self.net = nn.Sequential(*layers) + # zero-init final layer => v≈0 at init => X≈identity (stable warm start) + nn.init.zeros_(self.net[-1].weight) + nn.init.zeros_(self.net[-1].bias) + + def velocity(self, s, t, c_s, e_u): + s = s.expand(c_s.shape[0]) if s.dim() == 0 else s + t = t.expand(c_s.shape[0]) if t.dim() == 0 else t + h = torch.cat([self.time(s), self.time(t), c_s, e_u], dim=-1) + return self.net(h) + + def forward(self, s, t, c_s, e_u): + """X_θ(s,t,c_s,e_u). s,t are scalars or (b,) tensors in [0,1].""" + v = self.velocity(s, t, c_s, e_u) + dt = (t - s) + dt = dt.view(-1, 1) if torch.is_tensor(dt) and dt.dim() >= 1 else dt + return c_s + dt * v + + def endpoint(self, c0, e_u): + """ĉ1 = X_θ(0,1,c0,e_u).""" + zeros = torch.zeros(c0.shape[0], device=c0.device) + ones = torch.ones(c0.shape[0], device=c0.device) + return self.forward(zeros, ones, c0, e_u) diff --git a/src/models/pivot.py b/src/models/pivot.py new file mode 100644 index 0000000000000000000000000000000000000000..b7bc2e87d8f27e0a106fe67894f13fa87af31eab --- /dev/null +++ b/src/models/pivot.py @@ -0,0 +1,54 @@ +"""the pivot model: perturbation encoder ψ_η + flow map X_θ, plus the +inverse-design jacobian used by reward guidance (Alg 4).""" +from __future__ import annotations + +import numpy as np +import torch +import torch.nn as nn + +from src.models.encoders import PerturbationEncoder, build_pert_tensors +from src.models.flow_map import FlowMap + + +class PIVOT(nn.Module): + def __init__(self, d_state: int, n_genes: int, n_ops: int, n_perts: int, + d_pert: int = 64, hidden: int = 512, depth: int = 4, + rep_mode: str = "gene_op", gene_pathway=None, n_pathways: int = 0, + dropout: float = 0.0): + super().__init__() + self.encoder = PerturbationEncoder( + n_genes, n_ops, n_perts, emb_dim=d_pert, mode=rep_mode, + gene_pathway=gene_pathway, n_pathways=n_pathways, + ) + self.flow = FlowMap(d_state, d_pert, hidden=hidden, depth=depth, dropout=dropout) + self.d_state, self.d_pert = d_state, d_pert + + def encode(self, g, o, mask, pid): + return self.encoder(g, o, mask, pid) + + def endpoint_from_e(self, c0, e): + return self.flow.endpoint(c0, e) + + def endpoint_from_pert(self, c0, g, o, mask, pid): + e = self.encode(g, o, mask, pid) + if e.shape[0] == 1 and c0.shape[0] > 1: + e = e.expand(c0.shape[0], -1) + return self.flow.endpoint(c0, e) + + # --- inverse-design jacobian (Alg 4 building block) --- + def endpoint_jacobian_e(self, c0_single, e_single): + """∇_e X_{0,1}(c0, e) for a single (c0, e). returns (d_state, d_pert).""" + e = e_single.detach().clone().requires_grad_(True) + c0 = c0_single.unsqueeze(0) + + def f(ev): + return self.flow.endpoint(c0, ev.unsqueeze(0)).squeeze(0) + + return torch.autograd.functional.jacobian(f, e, create_graph=False) + + +@torch.no_grad() +def candidate_embeddings(model: PIVOT, data, labels, device) -> np.ndarray: + """ψ(u) for every candidate perturbation label -> (n_cand, d_pert).""" + g, o, mask, pid = build_pert_tensors(data, labels, device=device) + return model.encode(g, o, mask, pid).cpu().numpy() diff --git a/src/training/__init__.py b/src/training/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/training/losses.py b/src/training/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..9815f30096d40ff6d68e0da331a49ce541da5e13 --- /dev/null +++ b/src/training/losses.py @@ -0,0 +1,70 @@ +"""pivot training losses: + + L_map = E‖X_θ(s,t,c_s,e_u) − c_t‖² + L_tan = E‖v_θ(τ,τ,c_τ,e_u) − (c1−c0)‖² + L_semi = E‖X_θ(s,t,c_s,e_u) − X_θ(r,t, X_θ(s,r,c_s,e_u), e_u)‖² + L_reg = ‖e_u‖² + L = L_map + λ_tan·L_tan + λ_semi·L_semi + λ_reg·L_reg + +c_s, c_t, c_τ are linear interpolants between matched control c0 and perturbed c1. +""" +from __future__ import annotations + +import torch + + +def _sorted_uniform(n, k, device): + """return k columns of sorted U(0,1) samples: shape (n, k), each row ascending.""" + u = torch.rand(n, k, device=device) + return torch.sort(u, dim=1).values + + +def interp(c0, c1, frac): + """(1-frac)·c0 + frac·c1, frac shape (b,).""" + f = frac.view(-1, 1) + return (1 - f) * c0 + f * c1 + + +def compute_losses(flow, e_u, c0, c1, lambdas: dict): + """return (total, dict_of_components). shapes: c0,c1,(b,d); e_u (b,m).""" + B = c0.shape[0] + dev = c0.device + + # map loss: 0<=s<=t<=1 + st = _sorted_uniform(B, 2, dev) + s, t = st[:, 0], st[:, 1] + c_s = interp(c0, c1, s) + c_t = interp(c0, c1, t) + pred_t = flow(s, t, c_s, e_u) + L_map = ((pred_t - c_t) ** 2).sum(-1).mean() + + # tangent loss: diagonal velocity matches interpolation velocity (c1-c0) + tau = torch.rand(B, device=dev) + c_tau = interp(c0, c1, tau) + v_diag = flow.velocity(tau, tau, c_tau, e_u) + L_tan = ((v_diag - (c1 - c0)) ** 2).sum(-1).mean() + + # semigroup loss: 0<=s<=r<=t<=1, direct vs composed + srt = _sorted_uniform(B, 3, dev) + s2, r2, t2 = srt[:, 0], srt[:, 1], srt[:, 2] + direct = flow(s2, t2, c_s, e_u) + mid = flow(s2, r2, c_s, e_u) + composed = flow(r2, t2, mid, e_u) + L_semi = ((direct - composed) ** 2).sum(-1).mean() + + # regularizer + L_reg = (e_u ** 2).sum(-1).mean() + + total = ( + lambdas.get("map", 1.0) * L_map + + lambdas.get("tan", 1.0) * L_tan + + lambdas.get("semi", 1.0) * L_semi + + lambdas.get("reg", 1e-4) * L_reg + ) + return total, { + "map": L_map.item(), + "tan": L_tan.item(), + "semi": L_semi.item(), + "reg": L_reg.item(), + "total": total.item(), + } diff --git a/src/training/train.py b/src/training/train.py new file mode 100644 index 0000000000000000000000000000000000000000..abdcb7b84c331832155d5461c3e7776b16c54340 --- /dev/null +++ b/src/training/train.py @@ -0,0 +1,191 @@ +"""pivot training loop, algorithm 1. + +per minibatch of matched (c0, c1) pairs with perturbation u: + embed perturbations -> sample (s,t) -> interpolants -> L_map; + sample τ -> L_tan; sample (s,r,t) -> L_semi; add L_reg; backprop on (θ, η). +""" +from __future__ import annotations + +import os +import time +from dataclasses import dataclass, field, asdict + +import numpy as np +import torch + +from src.data.perturb_data import PerturbData +from src.data.splits import load_split +from src.models.pivot import PIVOT +from src.training.losses import compute_losses +from src.utils.common import pick_device, save_json, set_seed + + +@dataclass +class TrainConfig: + dataset: str = "norman" + embedding: str = "pca" + split: str = "perturbation" + rep_mode: str = "gene_op" + match: str = "batch" # control-matching strategy + d_pert: int = 64 + hidden: int = 512 + depth: int = 4 + dropout: float = 0.0 + lr: float = 1e-3 + weight_decay: float = 1e-5 + batch_size: int = 1024 + epochs: int = 60 + lam_tan: float = 1.0 + lam_semi: float = 0.5 + lam_reg: float = 1e-4 + grad_clip: float = 5.0 + train_frac: float = 1.0 # data-scaling ablation + lam_dist: float = 0.0 # distributional flow loss (population mmd) weight + n_dist_perts: int = 4 # perturbations sampled per step for the dist loss + dist_n: int = 64 # cells per population in the dist loss + seed: int = 0 + device_index: int | None = None + components: list = field(default_factory=lambda: ["map", "tan", "semi"]) # ablate losses + + +def _ablate_lambdas(cfg: TrainConfig) -> dict: + lam = {"map": 1.0, "tan": cfg.lam_tan, "semi": cfg.lam_semi, "reg": cfg.lam_reg} + if "map" not in cfg.components: + lam["map"] = 0.0 + if "tan" not in cfg.components: + lam["tan"] = 0.0 + if "semi" not in cfg.components: + lam["semi"] = 0.0 + return lam + + +def make_model(data: PerturbData, cfg: TrainConfig, device) -> PIVOT: + gene_pathway, n_path = None, 0 + if cfg.rep_mode == "gene_pathway_op": + fc = data.functional_clusters(seed=cfg.seed) + gp = np.array([fc.get(g, 0) for g in data.genes_vocab], dtype=np.int64) + gene_pathway, n_path = gp, int(gp.max()) + 1 + return PIVOT( + d_state=data.d, n_genes=len(data.genes_vocab), n_ops=len(data.op_vocab), + n_perts=len(data.perturbations), d_pert=cfg.d_pert, hidden=cfg.hidden, + depth=cfg.depth, rep_mode=cfg.rep_mode, gene_pathway=gene_pathway, + n_pathways=n_path, dropout=cfg.dropout, + ).to(device) + + +def train(cfg: TrainConfig, data: PerturbData | None = None, verbose: bool = True, log_every: int = 10): + set_seed(cfg.seed) + device = pick_device(cfg.device_index) + if data is None: + data = PerturbData(os.path.join("data/processed", cfg.dataset), embedding=cfg.embedding) + split = load_split(data.dir, cfg.split) + rng = np.random.default_rng(cfg.seed) + + emb = torch.as_tensor(data.emb, device=device) + labels = data.obs["perturbation"].values + + # perturbed training cells (controls used only as matching pool) + train_idx = split["train_idx"] + is_ctrl = data.is_control[train_idx] + pert_train = train_idx[~is_ctrl] + if cfg.train_frac < 1.0: + n = max(1, int(cfg.train_frac * len(pert_train))) + pert_train = rng.choice(pert_train, size=n, replace=False) + if verbose: + print(f"train cells: {len(pert_train)} perturbed | device {device} | " + f"split {cfg.split} | match {cfg.match} | rep {cfg.rep_mode}") + + model = make_model(data, cfg, device) + opt = torch.optim.AdamW(model.parameters(), lr=cfg.lr, weight_decay=cfg.weight_decay) + sched = torch.optim.lr_scheduler.CosineAnnealingLR(opt, T_max=cfg.epochs) + lam = _ablate_lambdas(cfg) + + from src.models.encoders import build_pert_tensors + + # precompute per-perturbation train cell lists for the distributional loss + dist_cells = None + if cfg.lam_dist > 0: + tr_labels = labels[pert_train] + dist_cells = {p: pert_train[tr_labels == p] for p in np.unique(tr_labels)} + dist_cells = {p: v for p, v in dist_cells.items() if len(v) >= 8} + dist_pert_list = list(dist_cells.keys()) + + def _mmd2_torch(x, y): + with torch.no_grad(): + d = torch.cdist(x[: min(128, len(x))], y[: min(128, len(y))]) + gamma = 1.0 / (d.median() ** 2 + 1e-8) + k = lambda a, b: torch.exp(-gamma * torch.cdist(a, b) ** 2) + return k(x, x).mean() + k(y, y).mean() - 2 * k(x, y).mean() + + history = [] + t0 = time.time() + for epoch in range(cfg.epochs): + model.train() + perm = rng.permutation(pert_train) + ep_losses = [] + for b in range(0, len(perm), cfg.batch_size): + batch_idx = perm[b: b + cfg.batch_size] + ctrl_idx = data.sample_controls(batch_idx, cfg.match, rng) + c0 = emb[ctrl_idx] + c1 = emb[batch_idx] + blabels = labels[batch_idx] + g, o, mask, pid = build_pert_tensors(data, list(blabels), device=device) + e = model.encode(g, o, mask, pid) + total, comp = compute_losses(model.flow, e, c0, c1, lam) + if cfg.lam_dist > 0: + terms = [] + for p in rng.choice(dist_pert_list, size=min(cfg.n_dist_perts, len(dist_pert_list)), + replace=False): + pc = dist_cells[p] + c1d = emb[rng.choice(pc, min(cfg.dist_n, len(pc)), replace=False)] + c0d = emb[rng.choice(data.control_idx, c1d.shape[0], replace=True)] + gd, od, md, pidd = build_pert_tensors(data, [p], device=device) + ed = model.encode(gd, od, md, pidd).expand(c0d.shape[0], -1) + chat = model.flow.endpoint(c0d, ed) + terms.append(_mmd2_torch(chat, c1d)) + Ldist = torch.stack(terms).mean() + total = total + cfg.lam_dist * Ldist + comp["dist"] = Ldist.item() + opt.zero_grad() + total.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), cfg.grad_clip) + opt.step() + ep_losses.append(comp) + sched.step() + m = {k: float(np.mean([d[k] for d in ep_losses])) for k in ep_losses[0]} + history.append(m) + if verbose and (epoch % log_every == 0 or epoch == cfg.epochs - 1): + print(f" ep {epoch:3d} total={m['total']:.4f} map={m['map']:.4f} " + f"tan={m['tan']:.4f} semi={m['semi']:.4f}") + dur = time.time() - t0 + if verbose: + print(f"trained in {dur:.1f}s") + return model, {"history": history, "duration_s": dur, "device": str(device), + "n_train_cells": int(len(pert_train))} + + +def save_checkpoint(model, cfg: TrainConfig, info: dict, out_dir: str): + os.makedirs(out_dir, exist_ok=True) + torch.save(model.state_dict(), os.path.join(out_dir, "model.pt")) + save_json(asdict(cfg), os.path.join(out_dir, "config.json")) + save_json(info, os.path.join(out_dir, "train_info.json")) + + +if __name__ == "__main__": + import argparse, dataclasses + ap = argparse.ArgumentParser() + for f in dataclasses.fields(TrainConfig): + if f.name == "components": + ap.add_argument("--components", nargs="+", default=None) + elif f.type == "int | None" or f.name == "device_index": + ap.add_argument(f"--{f.name.replace('_','-')}", type=int, default=None) + else: + typ = type(f.default) + ap.add_argument(f"--{f.name.replace('_','-')}", type=typ, default=f.default) + ap.add_argument("--out", default="experiments/exp_001") + args = ap.parse_args() + kw = {k: v for k, v in vars(args).items() if k != "out" and v is not None} + cfg = TrainConfig(**kw) + model, info = train(cfg) + save_checkpoint(model, cfg, info, args.out) + print("saved ->", args.out) diff --git a/src/utils/__init__.py b/src/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/utils/common.py b/src/utils/common.py new file mode 100644 index 0000000000000000000000000000000000000000..8bf56c24d86e2f2cabdc107de2ec39772ca0693d --- /dev/null +++ b/src/utils/common.py @@ -0,0 +1,83 @@ +"""shared utils: seeding, device selection, timing, json i/o.""" +from __future__ import annotations + +import json +import os +import random +import time +from contextlib import contextmanager + +import numpy as np + + +def set_seed(seed: int = 0) -> None: + """seed python, numpy and torch (if available) for reproducibility.""" + random.seed(seed) + np.random.seed(seed) + os.environ["PYTHONHASHSEED"] = str(seed) + try: + import torch + + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + # determinism for reproducibility; benchmark off + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + except Exception: + pass + + +def pick_device(prefer_index: int | None = None): + """return a torch device, picking the gpu with the most free memory. + + on this shared node several gpus may be partly occupied, so we grab the + emptiest one unless ``prefer_index`` is given. + """ + import torch + + if not torch.cuda.is_available(): + return torch.device("cpu") + if prefer_index is not None: + return torch.device(f"cuda:{prefer_index}") + best, best_free = 0, -1 + for i in range(torch.cuda.device_count()): + free, _ = torch.cuda.mem_get_info(i) + if free > best_free: + best, best_free = i, free + return torch.device(f"cuda:{best}") + + +@contextmanager +def timer(name: str = "block"): + """context manager yielding wall-clock seconds via ``.t``.""" + + class _T: + t = 0.0 + + obj = _T() + start = time.perf_counter() + try: + yield obj + finally: + obj.t = time.perf_counter() - start + + +def save_json(obj, path: str) -> None: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as f: + json.dump(obj, f, indent=2, default=_json_default) + + +def load_json(path: str): + with open(path) as f: + return json.load(f) + + +def _json_default(o): + if isinstance(o, (np.integer,)): + return int(o) + if isinstance(o, (np.floating,)): + return float(o) + if isinstance(o, (np.ndarray,)): + return o.tolist() + return str(o)