diff --git a/README.md b/README.md index e2b66f71e4ea46b9f8eee388ec05fca842d9fda1..7e410b6417e8c74c107bd68ed92b3d02fd109bc8 100644 --- a/README.md +++ b/README.md @@ -27,4 +27,11 @@ Reference: pip install -e ".[dev]" # Or uv pip install -r pyproject.toml && uv pip install pdf2image -``` \ No newline at end of file +``` + +# Current Steps to get results: +1. Run https://github.com/autogluon/tabarena/blob/main/examples/plots/run_generate_website_artifacts.py +2. Run https://github.com/autogluon/tabarena/blob/main/scripts/run_generate_zip_for_website.py + * ake sure to update the relevant metadata for the LB in the `format_leaderboard` function and code! +3. Unzip the generated zip file into the `data` folder in this repo and run `data_pdfs_to_pngs.py` +4. Test the LB locally, otherwise you are done :) \ No newline at end of file diff --git a/data/imputation_no/splits_all/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index a10dc781e82927f8b05f2b1b3dd7103e393b84b4..93560599fecf62b10c4661a295845ab4612e956b 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1dd9ea2c826175640f8d2397b1335e732569371bbcb838c21696add572fef528 -size 434350 +oid sha256:def4bff07f14c36cc14480de76d68d2b9708718c7359b64d2eb49c6c87b51bb6 +size 446843 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_all/pareto_n_configs_imp.png.zip index 8bb9e45cab3893e0cc86f612eadaa9936410f4e1..fd8327981cf444e2888933e1e7bdd4ca076ddd69 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bad65cc0285c1450996cf99e3a2cb903a9c18154770b93e76067fe7afc3c6ef9 -size 968055 +oid sha256:8ea109b15e43b572c2048d230ade24248a53d6e191af98e58dccf9f04527fca0 +size 934905 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_all/tuning-impact-elo.png.zip index 8e98e8e2ef2c23e866239c0302b8fa7e4d2abfb0..1aa00f1ca3635d6678089d875c9a3a4697e8a24c 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c2789a139f2ed0e703246604cd3517c8cad11d2f18e5e058a6195ec2b4603eb3 -size 108478 +oid sha256:02c6255dccd921f27823c4c120c7d34c5443506e671c4a6df959a430c1d54c97 +size 115082 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_all/datasets_all/website_leaderboard.csv index cf673d8023d1873fa5cd219707769e44889d4319..95946d43f379171a2fb835f86836d0c61d59546c 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_all/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_all/datasets_all/website_leaderboard.csv @@ -1,51 +1,52 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1679.0,+64/-51,5.507,0.789,6.94,3.2,582.21,6.116,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1626.0,+89/-60,4.753,0.739,8.58,2.9,2040.22,8.92,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1580.0,+73/-54,5.726,0.68,10.27,3.97,2040.22,1.218,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1546.0,+72/-52,6.429,0.654,11.6,4.97,5.81,0.645,✔️,0.0,False,GPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1539.0,+55/-51,8.693,0.549,11.93,5.37,1735.72,2.559,✔️,0.0,False,CPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1536.0,+57/-48,8.739,0.556,12.03,6.93,2950.72,11.988,✔️,0.0,False,GPU -6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1479.0,+65/-53,8.827,0.523,14.61,4.53,4907.64,286.652,✔️,0.0,False,GPU -7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1466.0,+56/-43,10.223,0.428,15.26,8.29,3285.87,1.472,✔️,0.0,False,GPU -8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1450.0,+36/-31,11.452,0.378,16.06,11.15,416.98,2.639,✔️,0.0,False,CPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1445.0,+46/-48,10.227,0.379,16.28,8.65,2950.72,0.663,✔️,0.0,False,GPU -10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1432.0,+44/-43,10.958,0.36,16.98,10.14,1658.41,0.653,✔️,0.0,False,CPU -11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1420.0,+60/-58,10.229,0.419,17.58,6.04,4907.64,39.96,✔️,0.0,False,GPU -12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1419.0,+42/-44,11.142,0.345,17.6,9.69,1658.41,0.081,✔️,0.0,False,CPU -13,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1409.0,+82/-54,11.442,0.385,18.12,7.61,4621.67,8.149,✔️,0.0,False,GPU -14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1404.0,+56/-47,11.14,0.341,18.38,9.66,3285.87,0.173,✔️,0.0,False,GPU -15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1400.0,+31/-30,12.143,0.28,18.63,13.38,416.98,0.334,✔️,0.0,False,CPU -16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1389.0,+35/-36,12.22,0.296,19.21,12.62,693.49,1.69,✔️,0.0,False,CPU -17,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1378.0,+41/-42,12.023,0.315,19.76,11.31,6.83,0.08,✔️,0.0,False,CPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1371.0,+45/-40,11.984,0.243,20.17,10.17,4621.67,0.475,✔️,0.0,False,GPU -19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1366.0,+49/-41,12.584,0.267,20.45,11.91,846.88,2.552,✔️,0.0,False,GPU -20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1363.0,+31/-33,12.54,0.241,20.59,14.97,693.49,0.308,✔️,0.0,False,CPU -21,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1339.0,+70/-60,12.933,0.336,21.91,7.2,47.62,43.739,✔️,0.0,False,GPU -22,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1313.0,+46/-38,13.883,0.198,23.35,14.43,846.88,0.13,✔️,0.0,False,GPU -23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1305.0,+54/-47,13.655,0.236,23.79,14.16,10.49,0.132,✔️,0.0,False,GPU -24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1299.0,+50/-50,13.439,0.164,24.11,16.52,2874.67,1.952,✔️,0.0,False,CPU -25,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1282.0,+43/-40,15.191,0.171,25.07,16.13,2929.85,0.422,✔️,0.0,False,CPU -26,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1257.0,+44/-44,15.566,0.123,26.41,15.77,14.87,0.308,✔️,0.0,False,GPU -27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1243.0,+47/-43,15.868,0.121,27.22,17.59,2929.85,0.051,✔️,0.0,False,CPU -28,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1240.0,+36/-41,14.957,0.086,27.36,19.31,10.06,1.689,✔️,0.0,False,GPU -29,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1219.0,+39/-38,15.422,0.091,28.49,20.6,1.94,0.123,✔️,0.0,False,CPU -30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1219.0,+52/-61,16.343,0.134,28.51,17.29,593.24,4.467,✔️,0.0,False,CPU -31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1218.0,+41/-47,15.309,0.078,28.58,20.1,2874.67,0.131,✔️,0.0,False,CPU -32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1212.0,+46/-42,16.88,0.101,28.89,19.17,183.02,0.761,✔️,0.0,False,CPU -33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1208.0,+46/-48,17.03,0.115,29.06,19.52,7.32,0.05,✔️,0.0,False,CPU -34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1190.0,+35/-29,16.049,0.048,30.03,26.04,1.96,0.142,✔️,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1179.0,+55/-48,17.756,0.076,30.61,20.12,373.18,0.771,✔️,0.0,False,CPU -36,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1172.0,+47/-52,17.853,0.076,30.96,19.36,183.02,0.091,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1153.0,+52/-55,17.842,0.057,31.96,21.29,593.24,0.306,✔️,0.0,False,CPU -38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1141.0,+49/-46,18.528,0.052,32.55,19.95,373.18,0.085,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1073.0,+35/-48,19.694,0.012,35.84,28.74,9.99,0.126,✔️,0.0,False,CPU -40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1043.0,+62/-61,23.269,0.036,37.16,29.18,3.23,0.919,✔️,0.0,False,GPU -41,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1005.0,+59/-60,22.719,0.013,38.74,28.39,2.86,0.373,✔️,0.0,False,CPU -42,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+45/-50,23.485,0.006,38.93,34.17,0.43,0.054,✔️,0.0,False,CPU -43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),998.0,+63/-88,25.352,0.016,39.0,27.81,129.01,1.802,✔️,0.0,False,CPU -44,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),980.0,+59/-71,24.998,0.011,39.69,34.68,0.25,0.05,✔️,0.0,False,CPU -45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),970.0,+64/-97,31.269,0.032,40.07,30.33,237.58,0.424,✔️,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),940.0,+71/-111,31.88,0.023,41.15,31.63,237.58,0.085,✔️,0.0,False,CPU -47,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),888.0,+68/-108,30.599,0.003,42.8,32.9,129.01,0.18,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),853.0,+76/-133,34.618,0.014,43.79,34.99,1.19,0.12,✔️,0.0,False,CPU -49,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),636.0,+89/-113,44.269,0.0,47.95,43.9,0.19,0.036,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1674.0,+64/-51,0.779,7.11,3.25,5.696,582.21,6.116,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1622.0,+87/-59,0.733,8.76,2.95,4.989,2040.22,8.92,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1576.0,+73/-54,0.675,10.47,4.04,5.962,2040.22,1.218,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1544.0,+71/-51,0.643,11.79,5.2,6.68,5.81,0.645,✔️,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1534.0,+55/-51,0.542,12.23,5.48,8.82,1735.72,2.559,✔️,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1532.0,+57/-47,0.541,12.29,7.13,8.942,2950.72,11.988,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1476.0,+64/-52,0.51,14.92,4.58,8.97,4907.64,286.652,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1462.0,+55/-42,0.415,15.58,8.43,10.369,3285.87,1.472,✔️,0.0,False,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1446.0,+36/-31,0.372,16.4,11.34,11.57,416.98,2.639,✔️,0.0,False,CPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1442.0,+47/-47,0.371,16.62,8.87,10.411,2950.72,0.663,✔️,0.0,False,GPU +10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1429.0,+42/-42,0.351,17.3,10.3,11.136,1658.41,0.653,✔️,0.0,False,CPU +11,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1417.0,+41/-42,0.335,17.95,9.81,11.319,1658.41,0.081,✔️,0.0,False,CPU +12,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1416.0,+59/-58,0.418,17.96,6.14,10.351,4907.64,39.96,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1406.0,+82/-53,0.381,18.5,7.7,11.589,4621.67,8.149,✔️,0.0,False,GPU +14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1401.0,+55/-47,0.333,18.77,9.81,11.277,3285.87,0.173,✔️,0.0,False,GPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1396.0,+30/-29,0.275,19.03,13.61,12.262,416.98,0.334,✔️,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1386.0,+34/-34,0.289,19.61,12.83,12.354,693.49,1.69,✔️,0.0,False,CPU +17,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1376.0,+41/-43,0.302,20.17,11.47,12.131,6.83,0.08,✔️,0.0,False,CPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1368.0,+45/-40,0.241,20.58,10.45,12.119,4621.67,0.475,✔️,0.0,False,GPU +19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1363.0,+48/-40,0.259,20.89,12.21,12.749,846.88,2.552,✔️,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1361.0,+30/-32,0.233,21.0,15.22,12.678,693.49,0.308,✔️,0.0,False,CPU +21,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1336.0,+70/-59,0.335,22.37,7.33,13.017,47.62,43.739,✔️,0.0,False,GPU +22,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1311.0,+47/-37,0.189,23.79,14.73,14.035,846.88,0.13,✔️,0.0,False,GPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1303.0,+54/-47,0.226,24.27,14.7,13.768,10.49,0.132,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1297.0,+50/-49,0.159,24.61,16.79,13.542,2874.67,1.952,✔️,0.0,False,CPU +25,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1295.0,+58/-51,0.272,24.72,8.97,14.411,14.11,2.075,➖,0.0,False,GPU +26,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1280.0,+42/-39,0.162,25.57,16.61,15.31,2929.85,0.422,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1256.0,+44/-42,0.119,26.94,16.39,15.671,14.87,0.308,✔️,0.0,False,GPU +28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1241.0,+46/-44,0.11,27.77,17.92,15.99,2929.85,0.051,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1238.0,+37/-42,0.084,27.96,19.99,15.056,10.06,1.689,✔️,0.0,False,GPU +30,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1218.0,+39/-39,0.092,29.03,20.86,15.53,1.94,0.123,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1216.0,+42/-48,0.079,29.15,20.42,15.395,2874.67,0.131,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1216.0,+51/-60,0.129,29.15,17.72,16.432,593.24,4.467,✔️,0.0,False,CPU +33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1210.0,+46/-43,0.099,29.47,19.54,17.023,183.02,0.761,✔️,0.0,False,CPU +34,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1207.0,+46/-48,0.107,29.68,20.02,17.135,7.32,0.05,✔️,0.0,False,CPU +35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1189.0,+35/-30,0.048,30.63,26.49,16.14,1.96,0.142,✔️,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1179.0,+54/-48,0.073,31.2,20.58,17.919,373.18,0.771,✔️,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1171.0,+47/-52,0.075,31.59,19.83,17.986,183.02,0.091,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1151.0,+51/-55,0.056,32.68,21.94,17.928,593.24,0.306,✔️,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1141.0,+49/-45,0.05,33.17,21.01,18.69,373.18,0.085,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1072.0,+34/-49,0.009,36.57,29.41,19.793,9.99,0.126,✔️,0.0,False,CPU +41,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1043.0,+61/-62,0.03,37.89,30.27,23.406,3.23,0.919,✔️,0.0,False,GPU +42,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1004.0,+58/-60,0.01,39.56,29.17,22.789,2.86,0.373,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+45/-47,0.005,39.71,35.21,23.577,0.43,0.054,✔️,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),998.0,+62/-88,0.017,39.8,28.27,25.428,129.01,1.802,✔️,0.0,False,CPU +45,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),980.0,+58/-70,0.01,40.48,35.51,25.075,0.25,0.05,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),968.0,+65/-96,0.031,40.95,30.93,31.37,237.58,0.424,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),938.0,+71/-108,0.023,42.03,32.29,31.963,237.58,0.085,✔️,0.0,False,CPU +48,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),887.0,+67/-107,0.004,43.7,33.56,30.663,129.01,0.18,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),852.0,+77/-130,0.014,44.7,35.66,34.682,1.19,0.12,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),634.0,+89/-113,0.0,48.93,44.79,44.315,0.19,0.036,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_all/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_all/winrate_matrix.png.zip index f3f7aef273c47476fdd25857778e2bbc6184bbed..b33418aab0dd0fcf7373513f1d4d70ed7b093829 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d7f0256cccbc9d360e60c248ee0d2afee7b3fbb0eb13ce8ded31b2f523126680 -size 1739374 +oid sha256:b08addd3e815d85bed684d11a9abe993ef5c59d29a092c2ff8b71e377247b3e6 +size 1957507 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index e364386bc5c70bc6e6d9d6f9df5a510b93bec420..e3727fe9b037bfd0289e76db8fe69adc7d7f9f98 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2e82722ede5d45ca41b062fa94c487123bbf01997c66b06cd2cf06e4281142c5 -size 399354 +oid sha256:c6f2cf21459ad0b69f6902d50904f75268b00910ac1d7dcdb5b4d1efbca04e4b +size 416676 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip index 8889dbfb5ad66e2e185502ae38fc02ab6b0fffc8..0c572a3b3d219ce5b03418aace26b3ca6f6eb3f9 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b819ee8c30207b57868a68efcc7ec9b1d27ef0ff5aeeda722d05d4d59568bac1 -size 989200 +oid sha256:cc527d8877a52f15b9bd640a5538fa61cfd334e7211c78fb67a2ceec6f7f7b3d +size 933091 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_medium/tuning-impact-elo.png.zip index 9b9aea74c66d806917036138659a06b106202c62..7a3b3d760f284676285ea88f94c1a8b23caf1e24 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:84be5741ee4ed6304624f803d704de577234f3a7a9827a3673a42bee30f83d2f -size 112567 +oid sha256:6256b79d1e995b904310a0836d76b62e08fb2b967cdacd6fd4258ccf32f1fa6e +size 118151 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_all/datasets_medium/website_leaderboard.csv index 3f0632fed8cdb49aa4245b9f5033d1684c70e668..264a25adc798b1b0992393835fbfbca0bae97cde 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_medium/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_all/datasets_medium/website_leaderboard.csv @@ -1,51 +1,52 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1825.0,+86/-46,2.566,0.836,5.78,2.85,289.53,3.187,✔️,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1804.0,+82/-49,2.669,0.808,6.29,3.08,432.35,4.085,✔️,0.0,False,CPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1676.0,+164/-124,5.382,0.66,10.21,3.09,735.58,11.736,✔️,0.0,False,GPU -3,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1649.0,+109/-94,5.684,0.581,11.21,6.96,1719.82,1.675,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1635.0,+125/-116,5.75,0.598,11.74,6.46,2526.28,6.013,✔️,0.0,False,GPU -5,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1621.0,+86/-63,5.585,0.539,12.33,6.64,777.59,0.25,✔️,0.0,False,CPU -6,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1620.0,+145/-112,6.097,0.591,12.34,5.04,735.58,1.386,✔️,0.0,False,GPU -7,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1610.0,+94/-69,5.783,0.541,12.74,5.67,777.59,0.052,✔️,0.0,False,CPU -8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1602.0,+76/-56,7.351,0.493,13.08,8.87,131.56,2.639,✔️,0.0,False,CPU -9,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1567.0,+105/-96,5.97,0.524,14.58,7.14,3.24,0.031,✔️,0.0,False,CPU -10,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1560.0,+87/-61,7.478,0.447,14.9,8.9,282.13,0.563,✔️,0.0,False,CPU -11,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1549.0,+116/-94,7.016,0.405,15.39,8.76,1719.82,0.081,✔️,0.0,False,GPU -12,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1546.0,+155/-99,6.906,0.478,15.51,7.63,1.88,0.645,✔️,0.0,False,GPU -13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1542.0,+155/-95,7.41,0.426,15.7,7.33,1993.14,0.619,✔️,0.0,False,GPU -14,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1534.0,+141/-125,7.024,0.48,16.05,4.17,4786.55,444.544,✔️,0.0,False,GPU -15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1512.0,+80/-65,8.574,0.342,17.07,11.95,131.56,0.132,✔️,0.0,False,CPU -16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1510.0,+69/-51,8.062,0.356,17.18,13.18,282.13,0.066,✔️,0.0,False,CPU -17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1490.0,+152/-101,8.168,0.36,18.13,8.42,1993.14,0.061,✔️,0.0,False,GPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1481.0,+107/-91,8.151,0.32,18.51,12.5,2526.28,0.349,✔️,0.0,False,GPU -19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1438.0,+108/-100,9.676,0.271,20.59,12.75,566.74,0.951,✔️,0.0,False,GPU -20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1412.0,+122/-124,9.655,0.296,21.86,8.61,4786.55,42.641,✔️,0.0,False,GPU -21,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1387.0,+71/-71,9.424,0.179,23.12,16.95,1770.56,1.069,✔️,0.0,False,CPU -22,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1382.0,+97/-96,11.377,0.191,23.34,18.62,566.74,0.06,✔️,0.0,False,GPU -23,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1371.0,+104/-120,9.928,0.234,23.89,14.51,0.49,0.053,✔️,0.0,False,CPU -24,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1339.0,+130/-140,12.451,0.221,25.48,10.01,46.62,43.739,✔️,0.0,False,GPU -25,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1334.0,+119/-133,12.922,0.194,25.72,17.81,1517.65,0.134,✔️,0.0,False,CPU -26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1328.0,+132/-111,11.007,0.177,26.01,17.36,5.16,0.074,✔️,0.0,False,GPU -27,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1302.0,+77/-82,11.463,0.096,27.27,23.36,8.06,0.25,✔️,0.0,False,GPU -28,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1280.0,+84/-70,12.214,0.046,28.34,25.52,6.75,0.235,✔️,0.0,False,GPU -29,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1275.0,+78/-77,11.211,0.055,28.58,24.36,1770.56,0.051,✔️,0.0,False,CPU -30,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1271.0,+110/-134,13.812,0.125,28.79,22.32,1517.65,0.015,✔️,0.0,False,CPU -31,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1258.0,+62/-73,11.863,0.051,29.37,27.19,0.29,0.045,✔️,0.0,False,CPU -32,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1249.0,+86/-79,12.418,0.039,29.81,24.11,66.0,0.341,✔️,0.0,False,CPU -33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1220.0,+97/-140,15.399,0.085,31.14,26.84,3.12,0.016,✔️,0.0,False,CPU -34,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1216.0,+97/-87,13.131,0.04,31.35,25.48,66.0,0.024,✔️,0.0,False,CPU -35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1214.0,+108/-138,14.369,0.071,31.42,25.0,351.94,1.658,✔️,0.0,False,CPU -36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1206.0,+71/-86,12.542,0.021,31.77,29.62,36.48,0.329,✔️,0.0,False,CPU -37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1158.0,+63/-76,13.771,0.006,33.91,32.33,36.48,0.037,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1118.0,+67/-72,15.534,0.0,35.56,34.12,2.18,0.039,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1102.0,+85/-108,16.646,0.021,36.21,32.28,351.94,0.114,✔️,0.0,False,CPU -40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1046.0,+145/-191,23.683,0.018,38.34,33.71,2.49,0.145,✔️,0.0,False,GPU -41,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1022.0,+138/-176,20.34,0.024,39.18,35.21,1.51,0.111,✔️,0.0,False,CPU -42,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+60/-87,18.698,0.0,39.92,39.15,0.2,0.021,✔️,0.0,False,CPU -43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),998.0,+87/-126,22.252,0.006,39.99,34.2,150.3,10.071,✔️,0.0,False,CPU -44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),950.0,+152/-228,31.109,0.005,41.48,38.36,43.64,0.081,✔️,0.0,False,CPU -45,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),940.0,+106/-141,21.813,0.002,41.8,40.36,0.11,0.028,✔️,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),897.0,+156/-249,32.052,0.003,42.99,40.6,43.64,0.018,✔️,0.0,False,CPU -47,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),813.0,+58/-95,28.248,0.0,44.96,44.7,150.3,0.482,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),785.0,+139/-247,34.158,0.0,45.52,44.41,0.19,0.019,✔️,0.0,False,CPU -49,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),566.0,+107/-159,42.214,0.0,48.53,48.44,0.1,0.072,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1821.0,+90/-48,0.837,5.84,2.86,2.566,289.53,3.187,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1801.0,+83/-48,0.811,6.35,3.08,2.669,432.35,4.085,✔️,0.0,False,CPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1673.0,+164/-122,0.664,10.31,3.09,5.382,735.58,11.736,✔️,0.0,False,GPU +3,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1647.0,+107/-94,0.588,11.29,6.97,5.684,1719.82,1.675,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1633.0,+127/-117,0.605,11.84,6.46,5.75,2526.28,6.013,✔️,0.0,False,GPU +5,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1618.0,+85/-62,0.545,12.41,6.65,5.585,777.59,0.25,✔️,0.0,False,CPU +6,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1617.0,+146/-113,0.596,12.45,5.04,6.097,735.58,1.386,✔️,0.0,False,GPU +7,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1608.0,+93/-67,0.547,12.84,5.68,5.783,777.59,0.052,✔️,0.0,False,CPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1600.0,+77/-58,0.498,13.18,8.89,7.351,131.56,2.639,✔️,0.0,False,CPU +9,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1564.0,+109/-97,0.531,14.73,7.15,5.97,3.24,0.031,✔️,0.0,False,CPU +10,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1558.0,+87/-61,0.454,15.0,8.92,7.478,282.13,0.563,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1546.0,+116/-94,0.414,15.53,8.77,7.016,1719.82,0.081,✔️,0.0,False,GPU +12,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1545.0,+154/-100,0.484,15.58,7.64,6.906,1.88,0.645,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1540.0,+155/-94,0.429,15.84,7.36,7.41,1993.14,0.619,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1531.0,+142/-126,0.49,16.22,4.17,7.024,4786.55,444.544,✔️,0.0,False,GPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1509.0,+82/-66,0.35,17.26,11.99,8.574,131.56,0.132,✔️,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1509.0,+69/-50,0.364,17.28,13.22,8.062,282.13,0.066,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1487.0,+151/-102,0.365,18.31,8.45,8.168,1993.14,0.061,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1479.0,+104/-92,0.33,18.69,12.57,8.151,2526.28,0.349,✔️,0.0,False,GPU +19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1435.0,+111/-101,0.279,20.87,12.83,9.676,566.74,0.951,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1408.0,+123/-125,0.303,22.2,8.64,9.655,4786.55,42.641,✔️,0.0,False,GPU +21,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1386.0,+70/-70,0.186,23.3,17.02,9.424,1770.56,1.069,✔️,0.0,False,CPU +22,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1381.0,+96/-95,0.201,23.54,18.73,11.377,566.74,0.06,✔️,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1370.0,+100/-119,0.241,24.13,14.56,9.928,0.49,0.053,✔️,0.0,False,CPU +24,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1334.0,+129/-140,0.234,25.92,10.08,12.451,46.62,43.739,✔️,0.0,False,GPU +25,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1332.0,+119/-129,0.206,26.05,17.9,12.922,1517.65,0.134,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1326.0,+133/-110,0.181,26.33,17.46,11.007,5.16,0.074,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1301.0,+77/-82,0.105,27.6,23.67,11.463,8.06,0.25,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1279.0,+85/-71,0.049,28.71,25.76,12.214,6.75,0.235,✔️,0.0,False,GPU +29,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1276.0,+80/-77,0.061,28.87,24.51,11.211,1770.56,0.051,✔️,0.0,False,CPU +30,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1269.0,+109/-135,0.133,29.19,22.49,13.812,1517.65,0.015,✔️,0.0,False,CPU +31,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1257.0,+63/-75,0.056,29.79,27.46,11.863,0.29,0.045,✔️,0.0,False,CPU +32,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1250.0,+86/-78,0.04,30.14,24.3,12.418,66.0,0.341,✔️,0.0,False,CPU +33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1218.0,+99/-139,0.097,31.62,27.07,15.399,3.12,0.016,✔️,0.0,False,CPU +34,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1217.0,+97/-88,0.042,31.71,25.66,13.131,66.0,0.024,✔️,0.0,False,CPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1211.0,+110/-138,0.072,31.96,25.22,14.369,351.94,1.658,✔️,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1206.0,+71/-84,0.024,32.18,29.89,12.542,36.48,0.329,✔️,0.0,False,CPU +37,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1205.0,+85/-93,0.047,32.27,27.42,14.333,11.44,10.416,➖,0.0,False,GPU +38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1158.0,+63/-75,0.01,34.39,32.69,13.771,36.48,0.037,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1117.0,+65/-74,0.0,36.16,34.61,15.534,2.18,0.039,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1099.0,+85/-109,0.021,36.91,32.77,16.646,351.94,0.114,✔️,0.0,False,CPU +41,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1046.0,+144/-192,0.018,38.96,34.13,23.683,2.49,0.145,✔️,0.0,False,GPU +42,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1019.0,+138/-180,0.025,39.96,35.77,20.34,1.51,0.111,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+60/-88,0.0,40.61,39.78,18.698,0.2,0.021,✔️,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),993.0,+86/-123,0.006,40.84,34.85,22.252,150.3,10.071,✔️,0.0,False,CPU +45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),946.0,+151/-227,0.005,42.36,39.13,31.109,43.64,0.081,✔️,0.0,False,CPU +46,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),938.0,+107/-133,0.002,42.59,41.03,21.813,0.11,0.028,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),893.0,+156/-248,0.003,43.88,41.42,32.052,43.64,0.018,✔️,0.0,False,CPU +48,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),805.0,+58/-94,0.0,45.96,45.71,28.248,150.3,0.482,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),778.0,+139/-246,0.0,46.5,45.41,34.158,0.19,0.019,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),559.0,+106/-172,0.0,49.53,49.44,42.214,0.1,0.072,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_all/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_medium/winrate_matrix.png.zip index a4d5fbcdcc11f6f61bede594c4a663c6a8f3ff31..30408bb39845df0b5feb0d24e514a7d8fa7732e3 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:36be9d3ce0af98477238f9519e6a9a06ec46e0c5afe2bc615cc08bf456fb64a5 -size 1634342 +oid sha256:178e1d8cd7f4413a3d0a9ad09d3ffd96ebdb0cd6baa5b2a7df91724aeeac0845 +size 1828619 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index fa36ee16e6517350c9c52d474c9a39de7c45f00f..055cee8a668f25fe63e4f97b72352363c105a547 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a47eadc792672c3f41eff8c373824e252e6928b11cfe2a8e6324a4c6f55ef05b -size 397284 +oid sha256:1618d16bd5c48e3e89a5a98a374d540ce9bf77ad318f3176cbf888e82de39f74 +size 409271 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_small/pareto_n_configs_imp.png.zip index 2aeed6bab7d379e9842cace94374418231320724..bcbcfe13b78f850a4f7269c37f28f8935e17c365 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b9e72f35f0f94d4d7734e2cce25b19ad20f0e57b566c38b06c41176826f7e12c -size 955425 +oid sha256:241667ceb4d651ace86ff16c9c34ea19dac4b5281de4c269a31983f7f482eab5 +size 946630 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_small/tuning-impact-elo.png.zip index c8cae8036327bd14cfc1dbb79090d4a9335c721d..a0a338ceb04e39410ee049d8542480ddea3b5bcb 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:23200a9c9d5f32d204e821ebd0df849df50e338db30ee7af5a4ecd319eecdaf2 -size 107979 +oid sha256:aa66226bbb6de8b72abb536e6958317c2498758b0e35719c199258275df270cb +size 115405 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_all/datasets_small/website_leaderboard.csv index 2ea236aca7e999f24d82722bdfcab2a540ed1a99..9faa9e21b15cd942a48fa388e0572a193c1c8cbc 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_small/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_all/datasets_small/website_leaderboard.csv @@ -1,51 +1,52 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1640.0,+68/-54,6.733,0.769,7.42,3.38,659.63,7.444,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1625.0,+97/-63,4.491,0.772,7.9,2.83,2289.05,8.049,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1580.0,+79/-58,5.571,0.717,9.4,3.65,2289.05,1.143,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1565.0,+86/-62,6.23,0.728,9.98,4.34,6.76,0.636,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1506.0,+68/-49,10.012,0.546,12.38,6.91,3770.75,21.902,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1470.0,+83/-61,9.579,0.54,14.01,4.71,5119.36,218.711,✔️,0.0,False,GPU -6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1464.0,+54/-48,11.203,0.441,14.28,7.78,3716.07,2.547,✔️,0.0,False,CPU -7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1448.0,+59/-48,11.395,0.429,15.08,8.77,3553.12,1.741,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1433.0,+76/-59,10.468,0.471,15.79,5.38,5119.36,28.351,✔️,0.0,False,GPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1416.0,+56/-42,11.565,0.369,16.66,8.6,3770.75,1.778,✔️,0.0,False,GPU -10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1404.0,+35/-36,13.16,0.33,17.29,12.49,892.41,2.574,✔️,0.0,False,CPU -11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1381.0,+56/-51,12.378,0.333,18.49,10.3,3553.12,0.241,✔️,0.0,False,GPU -12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1374.0,+43/-44,13.197,0.286,18.91,13.0,2476.51,0.814,✔️,0.0,False,CPU -13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1367.0,+35/-28,13.63,0.254,19.29,14.08,892.41,0.346,✔️,0.0,False,CPU -14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1360.0,+46/-36,13.375,0.263,19.62,13.76,2476.51,0.101,✔️,0.0,False,CPU -15,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1347.0,+47/-42,13.796,0.265,20.39,11.59,1126.71,2.941,✔️,0.0,False,GPU -16,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1346.0,+72/-67,13.133,0.384,20.43,6.44,50.32,43.709,✔️,0.0,False,GPU -17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1340.0,+64/-61,13.813,0.296,20.78,8.22,6521.96,8.396,✔️,0.0,False,GPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1338.0,+46/-42,13.58,0.211,20.86,9.44,6521.96,0.506,✔️,0.0,False,GPU -19,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1335.0,+36/-38,14.196,0.233,21.01,15.28,884.18,2.368,✔️,0.0,False,CPU -20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1319.0,+37/-41,14.545,0.227,21.92,14.94,9.64,0.128,✔️,0.0,False,CPU -21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1317.0,+34/-37,14.406,0.193,22.02,15.86,884.18,0.393,✔️,0.0,False,CPU -22,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1302.0,+56/-46,14.758,0.261,22.86,13.15,13.18,0.17,✔️,0.0,False,GPU -23,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1294.0,+40/-44,14.927,0.201,23.35,13.19,1126.71,0.163,✔️,0.0,False,GPU -24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1273.0,+47/-50,15.112,0.158,24.52,16.34,3865.95,2.158,✔️,0.0,False,CPU -25,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1268.0,+47/-45,16.137,0.162,24.79,15.52,3351.28,0.47,✔️,0.0,False,CPU -26,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1246.0,+48/-54,17.276,0.135,26.05,13.88,16.27,0.321,✔️,0.0,False,GPU -27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1237.0,+48/-47,16.725,0.119,26.56,16.17,3351.28,0.056,✔️,0.0,False,CPU -28,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1230.0,+39/-42,16.1,0.103,26.95,17.53,15.69,4.691,✔️,0.0,False,GPU -29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1224.0,+58/-62,17.165,0.16,27.29,15.32,1678.92,8.069,✔️,0.0,False,CPU -30,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1217.0,+70/-64,18.687,0.135,27.68,16.72,499.14,1.466,✔️,0.0,False,CPU -31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1208.0,+51/-51,17.71,0.128,28.2,17.53,9.08,0.057,✔️,0.0,False,CPU -32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1201.0,+40/-47,17.016,0.087,28.58,18.73,3865.95,0.148,✔️,0.0,False,CPU -33,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1180.0,+64/-62,19.553,0.105,29.74,16.59,499.14,0.207,✔️,0.0,False,CPU -34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1172.0,+54/-51,18.34,0.072,30.18,18.64,1678.92,0.826,✔️,0.0,False,CPU -35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1169.0,+40/-39,17.793,0.046,30.3,25.59,3.61,0.17,✔️,0.0,False,CPU -36,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1168.0,+34/-37,17.712,0.031,30.4,24.97,3.29,0.253,✔️,0.0,False,CPU -37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1158.0,+58/-70,19.98,0.091,30.94,18.82,590.04,1.461,✔️,0.0,False,CPU -38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1117.0,+56/-70,20.776,0.057,33.05,18.3,590.04,0.15,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1058.0,+48/-47,21.427,0.017,35.95,26.97,13.69,0.155,✔️,0.0,False,CPU -40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1042.0,+69/-90,23.096,0.043,36.67,27.64,5.44,1.098,✔️,0.0,False,GPU -41,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+56/-75,25.48,0.008,38.52,32.45,0.51,0.089,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),999.0,+50/-60,23.71,0.008,38.56,26.27,5.48,0.685,✔️,0.0,False,CPU -43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),998.0,+73/-96,26.644,0.019,38.58,25.8,111.84,1.215,✔️,0.0,False,CPU -44,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),993.0,+72/-72,26.326,0.015,38.81,32.76,0.47,0.092,✔️,0.0,False,CPU -45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),976.0,+86/-100,31.336,0.043,39.48,27.89,520.95,0.647,✔️,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),953.0,+84/-107,31.808,0.031,40.38,28.96,520.95,0.113,✔️,0.0,False,CPU -47,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),910.0,+73/-92,31.579,0.004,41.9,29.64,111.84,0.111,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),872.0,+99/-124,34.809,0.02,43.07,32.15,2.59,0.138,✔️,0.0,False,CPU -49,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),654.0,+110/-151,45.125,0.0,47.7,42.25,0.37,0.033,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1635.0,+68/-53,0.755,7.64,3.45,7.001,659.63,7.444,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1620.0,+96/-63,0.762,8.11,2.89,4.825,2289.05,8.049,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1577.0,+77/-58,0.708,9.64,3.73,5.905,2289.05,1.143,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1562.0,+86/-61,0.709,10.21,4.59,6.586,6.76,0.636,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1502.0,+67/-48,0.522,12.71,7.2,10.299,3770.75,21.902,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1467.0,+83/-61,0.518,14.37,4.78,9.78,5119.36,218.711,✔️,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1460.0,+53/-47,0.431,14.68,8.1,11.383,3716.07,2.547,✔️,0.0,False,CPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1444.0,+58/-48,0.41,15.47,8.97,11.602,3553.12,1.741,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1430.0,+76/-58,0.466,16.19,5.48,10.64,5119.36,28.351,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1414.0,+54/-42,0.353,17.07,8.91,11.826,3770.75,1.778,✔️,0.0,False,GPU +10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1401.0,+34/-36,0.319,17.75,12.81,13.328,892.41,2.574,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1378.0,+55/-51,0.32,18.97,10.51,12.572,3553.12,0.241,✔️,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1372.0,+42/-42,0.27,19.34,13.36,13.45,2476.51,0.814,✔️,0.0,False,CPU +13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1364.0,+35/-29,0.244,19.77,14.42,13.799,892.41,0.346,✔️,0.0,False,CPU +14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1359.0,+46/-34,0.247,20.08,14.09,13.626,2476.51,0.101,✔️,0.0,False,CPU +15,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1344.0,+72/-67,0.378,20.89,6.58,13.253,50.32,43.709,✔️,0.0,False,GPU +16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1344.0,+47/-40,0.251,20.9,11.97,14.029,1126.71,2.941,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1337.0,+62/-60,0.288,21.27,8.36,14.021,6521.96,8.396,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1336.0,+45/-42,0.204,21.37,9.76,13.773,6521.96,0.506,✔️,0.0,False,GPU +19,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1333.0,+36/-38,0.22,21.53,15.7,14.385,884.18,2.368,✔️,0.0,False,CPU +20,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1332.0,+69/-59,0.366,21.57,7.0,14.443,16.22,1.278,➖,0.0,False,GPU +21,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1317.0,+37/-40,0.207,22.43,15.33,14.699,9.64,0.128,✔️,0.0,False,CPU +22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1315.0,+32/-36,0.178,22.55,16.24,14.601,884.18,0.393,✔️,0.0,False,CPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1300.0,+55/-47,0.245,23.41,13.79,14.918,13.18,0.17,✔️,0.0,False,GPU +24,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1292.0,+40/-43,0.185,23.9,13.52,15.143,1126.71,0.163,✔️,0.0,False,GPU +25,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1270.0,+46/-50,0.148,25.16,16.69,15.258,3865.95,2.158,✔️,0.0,False,CPU +26,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1266.0,+46/-44,0.144,25.37,16.13,16.306,3351.28,0.47,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1244.0,+48/-54,0.125,26.67,14.53,17.425,16.27,0.321,✔️,0.0,False,GPU +28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1236.0,+47/-46,0.101,27.18,16.52,16.897,3351.28,0.056,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1227.0,+38/-42,0.098,27.64,18.29,16.24,15.69,4.691,✔️,0.0,False,GPU +30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1222.0,+58/-63,0.153,27.98,15.77,17.292,1678.92,8.069,✔️,0.0,False,CPU +31,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1215.0,+69/-65,0.131,28.34,17.08,18.89,499.14,1.466,✔️,0.0,False,CPU +32,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1206.0,+50/-51,0.111,28.87,18.06,17.859,9.08,0.057,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1199.0,+40/-47,0.086,29.26,19.09,17.139,3865.95,0.148,✔️,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1179.0,+63/-62,0.102,30.42,17.04,19.742,499.14,0.207,✔️,0.0,False,CPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1170.0,+54/-52,0.07,30.91,19.28,18.462,1678.92,0.826,✔️,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1168.0,+40/-39,0.045,30.99,26.11,17.922,3.61,0.17,✔️,0.0,False,CPU +37,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1167.0,+35/-37,0.03,31.07,25.45,17.865,3.29,0.253,✔️,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1156.0,+58/-69,0.086,31.65,19.34,20.211,590.04,1.461,✔️,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1117.0,+54/-69,0.054,33.78,19.53,21.006,590.04,0.15,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1058.0,+45/-47,0.012,36.74,27.68,21.567,13.69,0.155,✔️,0.0,False,CPU +41,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1042.0,+67/-89,0.035,37.45,28.91,23.291,5.44,1.098,✔️,0.0,False,GPU +42,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+57/-73,0.007,39.34,33.6,25.61,0.51,0.089,✔️,0.0,False,CPU +43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),999.0,+72/-94,0.022,39.37,26.21,26.751,111.84,1.215,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),999.0,+49/-61,0.003,39.4,27.09,23.81,5.48,0.685,✔️,0.0,False,CPU +45,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),994.0,+71/-75,0.013,39.6,33.63,26.434,0.47,0.092,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),975.0,+84/-100,0.043,40.36,28.45,31.478,520.95,0.647,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),952.0,+83/-107,0.031,41.26,29.58,31.926,520.95,0.113,✔️,0.0,False,CPU +48,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),910.0,+73/-92,0.006,42.76,30.21,31.67,111.84,0.111,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),873.0,+99/-125,0.02,43.95,32.73,34.901,2.59,0.138,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),653.0,+111/-144,0.0,48.68,43.1,45.191,0.37,0.033,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_all/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_small/winrate_matrix.png.zip index 0b169a54f56bb98f1eeb49a7b94f750fd6057b8e..f55bed329a94730e7a7c15dda600ce86ba9ade4e 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:10c40e7138ab23aa1e2ad6038a50adbe7e3c7f7cfe5d376584fb9fd5f401f89c -size 1729917 +oid sha256:0cf778466ada9eac48b665e0deced41a66871292039f96dcf26d4a24b28bcc08 +size 1930980 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 04ce9977869e956f4fb4c12cd3b539305d44dc43..52948b5769a2afa2ef6bfc630d82922ad8eee9f6 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:814976a6f141963f3c1e07b42ce6e6474b20be3bdcaa25fde5755f544f3c8152 -size 428426 +oid sha256:6ac3a6e6c121da4b53f4113ad98316bb36f496148e10d689e58e842b52248677 +size 442564 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip index f42caa4d774570e00081799ea163f5f29ae56f34..60c554830e17a921b2af34abbc95739438385771 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a80d25054695934b924850f798a7d9b8ccfe0122e6b6c7279ac9107116285c62 -size 1031798 +oid sha256:e524c9ce9da0571d7222e7e68df51ca422c6fc30a9c5625d2120d702feb52555 +size 1001251 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip index 8d0c8de42a42b3a886affccfc5c8939e4c3e87d4..fbb1a7de1a1b5b6d6bfdd23e1a6459faecc6b1ec 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:10a5b863795e2f720c603c8ebd1977e82aa7685b1d9e4d6a96453a5aea59f906 -size 115758 +oid sha256:1c79dc92b06beb91d34ff5b2bc958d671472ae0d74fad73e09c977fb04bf1635 +size 121006 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/website_leaderboard.csv index d457704ce3fca7b09a4721e225ec12cf52e14791..201051bc62b92144a6012600e45bae6f81153861 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/website_leaderboard.csv @@ -1,56 +1,57 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1640.0,+95/-71,5.186,0.744,9.05,3.4,2166.18,7.945,✔️,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1637.0,+78/-54,7.606,0.72,9.17,4.23,611.55,7.01,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1602.0,+66/-63,6.822,0.723,10.54,5.18,6.14,0.586,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1597.0,+72/-62,6.284,0.693,10.73,4.24,2166.18,0.993,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1546.0,+74/-50,9.458,0.591,12.97,5.42,3899.42,55.833,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1516.0,+74/-47,10.623,0.559,14.46,5.84,136.59,3.391,✔️,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1514.0,+54/-45,11.014,0.504,14.56,8.05,3547.04,20.963,✔️,0.0,False,GPU -7,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1506.0,+74/-71,13.034,0.586,14.97,5.38,4.85,0.629,➖,0.0,False,GPU -8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1468.0,+75/-61,10.616,0.481,16.95,5.43,4980.83,207.372,✔️,0.0,False,GPU -9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1459.0,+46/-52,12.287,0.384,17.46,9.83,3376.59,2.54,✔️,0.0,False,CPU -10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1446.0,+55/-50,12.487,0.373,18.19,10.24,3372.56,1.658,✔️,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1442.0,+80/-69,11.415,0.445,18.39,5.94,4980.83,23.583,✔️,0.0,False,GPU -12,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1439.0,+82/-69,12.024,0.434,18.6,6.82,3899.42,0.983,✔️,0.0,False,GPU -13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1422.0,+56/-47,12.593,0.337,19.55,10.33,3547.04,1.57,✔️,0.0,False,GPU -14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1398.0,+36/-36,14.456,0.258,20.98,15.0,771.57,2.49,✔️,0.0,False,CPU -15,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1390.0,+71/-69,13.505,0.399,21.43,9.2,4.22,0.554,✔️,0.0,False,GPU -16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1384.0,+58/-51,13.499,0.284,21.83,11.77,3372.56,0.208,✔️,0.0,False,GPU -17,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1382.0,+42/-37,14.384,0.24,21.91,15.08,2034.85,0.805,✔️,0.0,False,CPU -18,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1368.0,+37/-33,14.581,0.219,22.76,16.26,2034.85,0.097,✔️,0.0,False,CPU -19,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1366.0,+90/-66,14.141,0.381,22.88,6.96,48.9,43.804,✔️,0.0,False,GPU -20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1363.0,+36/-30,14.932,0.189,23.09,16.97,771.57,0.322,✔️,0.0,False,CPU -21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1342.0,+50/-45,15.104,0.205,24.36,14.6,1105.59,2.552,✔️,0.0,False,GPU -22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1338.0,+36/-36,15.506,0.181,24.6,18.37,828.74,2.312,✔️,0.0,False,CPU -23,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1338.0,+73/-67,15.155,0.25,24.64,11.28,6147.69,8.149,✔️,0.0,False,GPU -24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1338.0,+46/-49,14.737,0.17,24.64,11.71,6147.69,0.485,✔️,0.0,False,GPU -25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1327.0,+48/-45,15.809,0.184,25.31,17.24,8.51,0.123,✔️,0.0,False,CPU -26,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1320.0,+32/-34,15.718,0.142,25.71,18.6,828.74,0.336,✔️,0.0,False,CPU -27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1313.0,+54/-53,15.881,0.226,26.15,14.69,12.24,0.154,✔️,0.0,False,GPU -28,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1296.0,+45/-40,16.243,0.151,27.25,16.29,1105.59,0.13,✔️,0.0,False,GPU -29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1282.0,+48/-45,17.283,0.12,28.07,18.33,2929.85,0.433,✔️,0.0,False,CPU -30,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1275.0,+48/-54,16.303,0.124,28.54,20.3,3704.3,2.074,✔️,0.0,False,CPU -31,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1267.0,+47/-42,17.906,0.118,28.99,15.86,16.16,0.307,✔️,0.0,False,GPU -32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1252.0,+52/-49,17.885,0.084,29.95,18.24,2929.85,0.054,✔️,0.0,False,CPU -33,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1245.0,+42/-38,17.002,0.082,30.35,19.8,15.48,4.535,✔️,0.0,False,GPU -34,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1236.0,+51/-62,18.355,0.142,30.9,16.77,1459.62,8.056,✔️,0.0,False,CPU -35,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1219.0,+49/-53,18.772,0.078,31.95,19.88,8.47,0.054,✔️,0.0,False,CPU -36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1213.0,+60/-60,20.26,0.101,32.33,21.51,416.39,1.393,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1203.0,+47/-48,18.22,0.065,32.91,21.43,3704.3,0.143,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1181.0,+56/-61,19.544,0.063,34.18,20.76,1459.62,0.89,✔️,0.0,False,CPU -39,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1177.0,+57/-61,21.094,0.069,34.43,22.54,416.39,0.179,✔️,0.0,False,CPU -40,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1172.0,+37/-38,19.131,0.017,34.71,29.77,3.05,0.241,✔️,0.0,False,CPU -41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1158.0,+33/-41,19.359,0.021,35.5,31.12,3.39,0.158,✔️,0.0,False,CPU -42,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1142.0,+56/-58,21.744,0.047,36.41,24.9,572.67,1.421,✔️,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1103.0,+47/-48,22.52,0.019,38.56,25.42,572.67,0.143,✔️,0.0,False,CPU -44,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1067.0,+50/-60,22.688,0.012,40.39,30.45,11.82,0.147,✔️,0.0,False,CPU -45,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1053.0,+71/-83,24.705,0.03,41.1,31.46,5.39,1.034,✔️,0.0,False,GPU -46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1010.0,+55/-66,24.827,0.0,43.07,28.88,5.18,0.652,✔️,0.0,False,CPU -47,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1004.0,+72/-90,28.179,0.012,43.34,31.16,100.58,1.106,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1003.0,+71/-108,31.292,0.044,43.36,30.05,504.17,0.629,✔️,0.0,False,CPU -49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1002.0,+83/-87,27.374,0.008,43.4,36.68,0.42,0.078,✔️,0.0,False,CPU -50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+55/-70,26.617,0.002,43.49,37.64,0.47,0.069,✔️,0.0,False,CPU -51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),980.0,+74/-108,31.725,0.03,44.33,31.71,504.17,0.103,✔️,0.0,False,CPU -52,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),908.0,+78/-103,33.337,0.0,47.0,33.68,100.58,0.101,✔️,0.0,False,CPU -53,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),903.0,+82/-129,34.211,0.019,47.19,34.88,2.51,0.137,✔️,0.0,False,CPU -54,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),680.0,+97/-169,45.686,0.0,52.44,45.72,0.29,0.028,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1636.0,+92/-70,0.738,9.28,3.47,5.487,2166.18,7.945,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1633.0,+76/-53,0.712,9.4,4.29,7.843,611.55,7.01,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1599.0,+64/-62,0.714,10.78,5.44,7.146,6.14,0.586,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1594.0,+73/-62,0.686,10.99,4.31,6.585,2166.18,0.993,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1543.0,+73/-49,0.583,13.29,5.59,9.672,3899.42,55.833,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),1513.0,+73/-47,0.555,14.79,6.01,10.779,136.59,3.391,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1510.0,+54/-44,0.493,14.92,8.32,11.276,3547.04,20.963,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1504.0,+73/-71,0.574,15.25,5.51,13.315,4.85,0.629,➖,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1466.0,+74/-61,0.471,17.34,5.5,10.795,4980.83,207.372,✔️,0.0,False,GPU +9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1456.0,+45/-50,0.381,17.88,10.09,12.435,3376.59,2.54,✔️,0.0,False,CPU +10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1444.0,+55/-50,0.364,18.6,10.4,12.664,3372.56,1.658,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1440.0,+79/-68,0.441,18.8,6.02,11.57,4980.83,23.583,✔️,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1437.0,+80/-68,0.426,19.0,6.98,12.223,3899.42,0.983,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1420.0,+56/-47,0.331,19.97,10.64,12.828,3547.04,1.57,✔️,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1395.0,+36/-34,0.255,21.47,15.3,14.595,771.57,2.49,✔️,0.0,False,CPU +15,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1388.0,+69/-67,0.388,21.92,9.49,13.784,4.22,0.554,✔️,0.0,False,GPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1381.0,+58/-50,0.278,22.34,11.97,13.664,3372.56,0.208,✔️,0.0,False,GPU +17,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1381.0,+42/-36,0.232,22.36,15.41,14.608,2034.85,0.805,✔️,0.0,False,CPU +18,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1367.0,+36/-33,0.213,23.25,16.61,14.804,2034.85,0.097,✔️,0.0,False,CPU +19,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1365.0,+89/-66,0.379,23.36,7.07,14.238,48.9,43.804,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1362.0,+65/-67,0.346,23.56,7.96,15.219,14.11,1.17,➖,0.0,False,GPU +21,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1361.0,+36/-30,0.185,23.61,17.3,15.073,771.57,0.322,✔️,0.0,False,CPU +22,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1340.0,+50/-45,0.201,24.91,15.09,15.311,1105.59,2.552,✔️,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1336.0,+36/-36,0.175,25.15,18.83,15.668,828.74,2.312,✔️,0.0,False,CPU +24,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1336.0,+73/-67,0.249,25.17,11.47,15.327,6147.69,8.149,✔️,0.0,False,GPU +25,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1336.0,+46/-49,0.171,25.17,11.99,14.894,6147.69,0.485,✔️,0.0,False,GPU +26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1325.0,+48/-44,0.178,25.85,17.59,15.936,8.51,0.123,✔️,0.0,False,CPU +27,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1318.0,+32/-33,0.135,26.28,18.98,15.886,828.74,0.336,✔️,0.0,False,CPU +28,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1312.0,+53/-53,0.219,26.71,15.44,16.015,12.24,0.154,✔️,0.0,False,GPU +29,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1294.0,+45/-40,0.149,27.83,16.61,16.438,1105.59,0.13,✔️,0.0,False,GPU +30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1281.0,+48/-44,0.113,28.67,18.84,17.425,2929.85,0.433,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1272.0,+48/-53,0.124,29.2,20.68,16.423,3704.3,2.074,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1266.0,+48/-43,0.116,29.61,16.23,18.028,16.16,0.307,✔️,0.0,False,GPU +33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1251.0,+52/-49,0.079,30.59,18.58,18.029,2929.85,0.054,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1243.0,+42/-38,0.082,31.04,20.45,17.117,15.48,4.535,✔️,0.0,False,GPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1234.0,+51/-62,0.14,31.61,17.15,18.454,1459.62,8.056,✔️,0.0,False,CPU +36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1218.0,+49/-53,0.073,32.64,20.44,18.893,8.47,0.054,✔️,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1211.0,+60/-62,0.097,33.03,21.91,20.439,416.39,1.393,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1202.0,+47/-48,0.067,33.62,21.81,18.319,3704.3,0.143,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1180.0,+56/-61,0.061,34.94,21.38,19.639,1459.62,0.89,✔️,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1176.0,+57/-61,0.065,35.14,23.11,21.259,416.39,0.179,✔️,0.0,False,CPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1172.0,+38/-38,0.02,35.42,30.3,19.257,3.05,0.241,✔️,0.0,False,CPU +42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1158.0,+33/-42,0.022,36.23,31.72,19.461,3.39,0.158,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1141.0,+56/-58,0.044,37.17,25.69,21.955,572.67,1.421,✔️,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1102.0,+48/-48,0.017,39.34,27.82,22.731,572.67,0.143,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1066.0,+50/-61,0.013,41.23,31.06,22.804,11.82,0.147,✔️,0.0,False,CPU +46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1052.0,+72/-82,0.027,41.91,32.45,24.878,5.39,1.034,✔️,0.0,False,GPU +47,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1010.0,+54/-67,0.001,43.91,29.79,24.901,5.18,0.652,✔️,0.0,False,CPU +48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1005.0,+72/-88,0.012,44.14,31.64,28.265,100.58,1.106,✔️,0.0,False,CPU +49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1003.0,+81/-87,0.008,44.21,37.49,27.463,0.42,0.078,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1003.0,+72/-107,0.044,44.22,30.57,31.413,504.17,0.629,✔️,0.0,False,CPU +51,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+56/-71,0.002,44.34,38.83,26.728,0.47,0.069,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),980.0,+74/-108,0.03,45.19,32.31,31.822,504.17,0.103,✔️,0.0,False,CPU +53,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),909.0,+77/-103,0.0,47.87,34.26,33.409,100.58,0.101,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),904.0,+80/-126,0.019,48.07,35.41,34.282,2.51,0.137,✔️,0.0,False,CPU +55,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),679.0,+97/-168,0.0,53.42,46.54,45.736,0.29,0.028,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/winrate_matrix.png.zip index 03d6d0cf1d70379371b4e24e09cc179487c64bfc..e570052e2b0c9a43dfa1c81ba15436bd3364c3c8 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6f6d351346ebb8ff8cb6cdd49a2cb2fa092dddbf6561bbaaaecbbbe2af43173a -size 2212024 +oid sha256:e58763ac13cff7c9cce0aaca4d2187ee8e1710234aa10560313196c7b14f377c +size 2394027 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index f6786727aec371c9d5ce331ccb676b29dc688b5a..bdb21eb849e0361d9ae2d44e2ac4e88b07340c17 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f1e3a277f7edcdee89d89e95d1b5480b0d72b7271e7a90bbfa3944f443df0c9e -size 429276 +oid sha256:fac4e26f5a7e7ca288c2c57115a104635f3738f1321c1711bdfec10610a3900d +size 442773 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip index ce98a89faab97a1068586de90bee92e2d1fad4c5..d78fdd6dd4c26437e278f462f1482347097a862f 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e34bbe80e56bc3cda9aa62b92aeab5758bd1ddb6d7c51a51d9c3f1f753eb6f0c -size 958750 +oid sha256:86a37085d7f67faa4342c4d9f111bf5b24f57d88ef6fe5f0631d387ed7445e8b +size 936941 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_all/tuning-impact-elo.png.zip index cb6ee0b1ea88565f9712615a49014dc30a6ee1bb..4a1cf90fc6d4b4584de18b5d879ea8c55dd8958e 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:71fd051104b26359c822debf594b3a093680f7a2d753ec01e6f2dd257bdcc831 -size 113807 +oid sha256:520671463bfa8ac1313b8b0d0100144b6f7ffac121c3cb0ac38c752fb74174c4 +size 118287 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_binary/datasets_all/website_leaderboard.csv index 00ce45e6dcdcad6f26f56051778f91962a81e830..d96c41cf43ac60dff836d1a207167aca6db8dddb 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_all/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_binary/datasets_all/website_leaderboard.csv @@ -1,53 +1,54 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1657.0,+98/-71,5.749,0.803,7.38,2.99,442.09,6.381,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1568.0,+110/-63,6.02,0.681,10.55,3.6,1923.88,9.103,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1545.0,+101/-57,7.098,0.687,11.52,4.71,4.91,0.694,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1535.0,+101/-59,7.13,0.632,11.94,4.35,1923.88,1.329,✔️,0.0,False,GPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1515.0,+73/-65,9.038,0.555,12.85,5.23,1152.95,1.784,✔️,0.0,False,CPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1500.0,+53/-56,10.144,0.508,13.58,8.76,2775.44,8.849,✔️,0.0,False,GPU -6,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1475.0,+76/-57,10.795,0.463,14.8,7.23,2229.7,1.387,✔️,0.0,False,GPU -7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1438.0,+55/-38,12.3,0.382,16.76,10.85,328.96,1.102,✔️,0.0,False,CPU -8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1422.0,+72/-62,9.134,0.436,17.64,5.97,4982.66,358.753,✔️,0.0,False,GPU -9,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1416.0,+81/-59,11.711,0.371,17.95,8.6,2229.7,0.133,✔️,0.0,False,GPU -10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1408.0,+61/-52,12.08,0.353,18.37,9.9,1055.6,0.482,✔️,0.0,False,CPU -11,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1403.0,+59/-53,12.174,0.353,18.64,9.24,1055.6,0.063,✔️,0.0,False,CPU -12,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1403.0,+56/-50,12.048,0.374,18.67,9.35,4.27,0.064,✔️,0.0,False,CPU -13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1400.0,+52/-56,11.944,0.298,18.83,9.7,2775.44,0.415,✔️,0.0,False,GPU -14,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1385.0,+62/-53,13.208,0.314,19.7,11.17,472.99,0.621,✔️,0.0,False,CPU -15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1384.0,+49/-31,13.159,0.268,19.73,13.42,328.96,0.126,✔️,0.0,False,CPU -16,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1380.0,+68/-68,14.06,0.371,19.99,9.43,250.36,1.708,➖,0.0,False,GPU -17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1367.0,+94/-94,12.88,0.374,20.73,8.42,3383.91,8.396,✔️,0.0,False,GPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1366.0,+58/-59,12.838,0.248,20.75,9.56,3383.91,0.51,✔️,0.0,False,GPU -19,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1355.0,+65/-58,10.825,0.301,21.39,7.03,4982.66,42.199,✔️,0.0,False,GPU -20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1350.0,+49/-43,13.724,0.236,21.73,14.36,472.99,0.13,✔️,0.0,False,CPU -21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1331.0,+58/-56,13.837,0.21,22.85,12.96,642.33,2.767,✔️,0.0,False,GPU -22,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1318.0,+49/-41,13.457,0.197,23.61,15.72,2241.65,2.415,✔️,0.0,False,CPU -23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1312.0,+69/-65,14.413,0.259,23.95,12.93,8.72,0.126,✔️,0.0,False,GPU -24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1311.0,+52/-48,15.332,0.18,24.03,15.4,1560.02,0.404,✔️,0.0,False,CPU -25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1302.0,+47/-58,15.116,0.178,24.57,13.95,642.33,0.146,✔️,0.0,False,GPU -26,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1283.0,+64/-72,14.197,0.243,25.68,9.36,49.67,43.824,✔️,0.0,False,GPU -27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1274.0,+51/-46,16.045,0.121,26.2,16.49,1560.02,0.052,✔️,0.0,False,CPU -28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1267.0,+50/-42,15.891,0.15,26.62,13.82,13.5,0.311,✔️,0.0,False,GPU -29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1267.0,+60/-64,15.482,0.152,26.62,16.59,559.38,4.08,✔️,0.0,False,CPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1265.0,+44/-44,14.803,0.11,26.75,17.68,9.33,1.116,✔️,0.0,False,GPU -31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1248.0,+49/-52,17.1,0.123,27.77,17.29,4.98,0.049,✔️,0.0,False,CPU -32,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1231.0,+66/-56,16.365,0.117,28.74,18.33,1.24,0.11,✔️,0.0,False,CPU -33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1226.0,+46/-39,15.668,0.077,29.08,19.82,2241.65,0.136,✔️,0.0,False,CPU -34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1191.0,+57/-63,17.521,0.061,31.05,19.79,559.38,0.276,✔️,0.0,False,CPU -35,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1188.0,+51/-53,18.0,0.061,31.26,22.1,125.38,0.674,✔️,0.0,False,CPU -36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1179.0,+49/-34,17.148,0.047,31.73,27.14,0.81,0.093,✔️,0.0,False,CPU -37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1154.0,+51/-57,19.362,0.035,33.12,22.93,169.75,0.603,✔️,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1146.0,+50/-53,19.162,0.038,33.56,23.89,125.38,0.069,✔️,0.0,False,CPU -39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1119.0,+43/-54,20.091,0.018,35.01,23.64,169.75,0.059,✔️,0.0,False,CPU -40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1093.0,+65/-48,20.071,0.019,36.39,27.59,4.95,0.091,✔️,0.0,False,CPU -41,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1057.0,+64/-82,21.46,0.021,38.16,24.5,2.56,0.267,✔️,0.0,False,CPU -42,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1051.0,+84/-107,26.808,0.054,38.43,26.2,162.68,0.405,✔️,0.0,False,CPU -43,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1045.0,+109/-116,25.52,0.099,38.71,22.36,0.85,0.129,➖,0.0,False,GPU -44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1022.0,+61/-103,24.212,0.017,39.74,28.61,139.84,3.917,✔️,0.0,False,CPU -45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1022.0,+91/-118,27.502,0.039,39.75,27.56,162.68,0.054,✔️,0.0,False,CPU -46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+62/-79,24.287,0.013,40.7,33.84,0.25,0.029,✔️,0.0,False,CPU -47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),983.0,+73/-101,26.581,0.026,41.38,33.73,2.8,0.564,✔️,0.0,False,GPU -48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),959.0,+86/-119,29.448,0.024,42.33,30.48,0.84,0.094,✔️,0.0,False,CPU -49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),955.0,+75/-92,25.759,0.01,42.47,36.5,0.2,0.037,✔️,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),915.0,+82/-136,29.325,0.0,43.91,31.65,139.84,0.193,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),602.0,+105/-199,43.863,0.0,50.36,46.66,0.15,0.023,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1655.0,+97/-71,0.791,7.53,3.0,5.801,442.09,6.381,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1567.0,+111/-62,0.674,10.74,3.65,6.072,1923.88,9.103,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1545.0,+101/-57,0.672,11.67,4.88,7.151,4.91,0.694,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1535.0,+100/-58,0.626,12.14,4.4,7.181,1923.88,1.329,✔️,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1513.0,+74/-64,0.552,13.16,5.3,9.088,1152.95,1.784,✔️,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1498.0,+52/-55,0.489,13.85,8.9,10.194,2775.44,8.849,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1474.0,+76/-58,0.448,15.11,7.31,10.846,2229.7,1.387,✔️,0.0,False,GPU +7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1436.0,+56/-38,0.379,17.09,10.99,12.349,328.96,1.102,✔️,0.0,False,CPU +8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1421.0,+72/-63,0.42,17.97,6.01,9.185,4982.66,358.753,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1414.0,+80/-59,0.366,18.34,8.71,11.761,2229.7,0.133,✔️,0.0,False,GPU +10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1408.0,+62/-51,0.35,18.68,10.01,12.129,1055.6,0.482,✔️,0.0,False,CPU +11,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1403.0,+60/-53,0.351,18.99,9.32,12.223,1055.6,0.063,✔️,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1402.0,+56/-50,0.362,19.01,9.45,12.097,4.27,0.064,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1399.0,+51/-56,0.291,19.2,9.83,11.992,2775.44,0.415,✔️,0.0,False,GPU +14,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1384.0,+61/-52,0.31,20.09,11.3,13.257,472.99,0.621,✔️,0.0,False,CPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1383.0,+49/-31,0.267,20.14,13.61,13.208,328.96,0.126,✔️,0.0,False,CPU +16,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1380.0,+68/-66,0.355,20.34,9.6,14.109,250.36,1.708,➖,0.0,False,GPU +17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1366.0,+94/-94,0.372,21.13,8.49,12.927,3383.91,8.396,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1365.0,+56/-59,0.244,21.17,9.89,12.885,3383.91,0.51,✔️,0.0,False,GPU +19,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1354.0,+64/-59,0.3,21.84,7.11,10.877,4982.66,42.199,✔️,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1349.0,+48/-42,0.231,22.16,14.53,13.773,472.99,0.13,✔️,0.0,False,CPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1330.0,+58/-56,0.206,23.31,13.25,13.888,642.33,2.767,✔️,0.0,False,GPU +22,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1317.0,+50/-42,0.191,24.1,15.97,13.504,2241.65,2.415,✔️,0.0,False,CPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1311.0,+68/-66,0.246,24.43,13.5,14.462,8.72,0.126,✔️,0.0,False,GPU +24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1311.0,+51/-46,0.171,24.46,15.88,15.381,1560.02,0.404,✔️,0.0,False,CPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1302.0,+47/-58,0.169,25.01,14.17,15.164,642.33,0.146,✔️,0.0,False,GPU +26,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1295.0,+65/-62,0.243,25.42,10.31,15.901,12.86,3.167,➖,0.0,False,GPU +27,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1282.0,+65/-72,0.244,26.22,9.53,14.248,49.67,43.824,✔️,0.0,False,GPU +28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1275.0,+50/-46,0.111,26.67,16.79,16.094,1560.02,0.052,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1267.0,+49/-43,0.142,27.15,14.44,15.94,13.5,0.311,✔️,0.0,False,GPU +30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1266.0,+60/-63,0.148,27.18,16.95,15.531,559.38,4.08,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1264.0,+43/-43,0.106,27.33,18.23,14.851,9.33,1.116,✔️,0.0,False,GPU +32,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1248.0,+48/-51,0.114,28.32,17.77,17.149,4.98,0.049,✔️,0.0,False,CPU +33,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1232.0,+66/-57,0.117,29.26,18.52,16.412,1.24,0.11,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1225.0,+46/-40,0.073,29.68,20.15,15.714,2241.65,0.136,✔️,0.0,False,CPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1190.0,+56/-62,0.058,31.72,20.46,17.57,559.38,0.276,✔️,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1187.0,+51/-52,0.06,31.9,22.47,18.049,125.38,0.674,✔️,0.0,False,CPU +37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1180.0,+50/-35,0.048,32.35,27.56,17.195,0.81,0.093,✔️,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1155.0,+51/-57,0.034,33.76,23.41,19.411,169.75,0.603,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1146.0,+50/-51,0.037,34.27,24.55,19.211,125.38,0.069,✔️,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1120.0,+42/-55,0.018,35.69,25.47,20.141,169.75,0.059,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1092.0,+64/-49,0.014,37.14,28.23,20.117,4.95,0.091,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1057.0,+64/-81,0.017,38.93,25.22,21.507,2.56,0.267,✔️,0.0,False,CPU +43,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1050.0,+83/-106,0.054,39.25,26.66,26.853,162.68,0.405,✔️,0.0,False,CPU +44,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1043.0,+108/-117,0.094,39.57,23.05,25.571,0.85,0.129,➖,0.0,False,GPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1022.0,+60/-104,0.017,40.56,29.08,24.258,139.84,3.917,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1021.0,+91/-118,0.039,40.58,28.09,27.547,162.68,0.054,✔️,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+62/-79,0.012,41.52,35.08,24.336,0.25,0.029,✔️,0.0,False,CPU +48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),983.0,+72/-103,0.021,42.22,34.49,26.628,2.8,0.564,✔️,0.0,False,GPU +49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),958.0,+86/-118,0.024,43.22,31.03,29.49,0.84,0.094,✔️,0.0,False,CPU +50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),956.0,+73/-90,0.01,43.3,37.32,25.807,0.2,0.037,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),915.0,+81/-134,0.001,44.8,32.2,29.37,139.84,0.193,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),601.0,+106/-200,0.0,51.34,47.51,43.9,0.15,0.023,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_all/winrate_matrix.png.zip index 097c95c97446ac6867eb98e1e58a4b568bf80fbd..0c41db8372fbca78a3a8045f4ab422cad10a607f 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ccd8f2c2d9058269966c9ba26269dc384f77ba122a369a0a275ecfa62c24acdf -size 2071274 +oid sha256:3c5a7a8c549d85a3455629e911869573d7c46ceff24aa614a69d6a6a2aee34d9 +size 2255151 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index e2181cf98dd316af8a806593aa39bd16fc8ef05f..68d6a07e91339fe97a1de4576dc7965ca97ff51a 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:37573f734e5396b487c7796eb775df4fbc8e00a9d50f06378ce92b8ae7ed5634 -size 419894 +oid sha256:e49df088101fd65ac80b606f45df307c27ee7dd0ac00fc4130cc42156ed3b008 +size 434196 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip index 5a1f7d5d5d36e45e6f181b04442b21d9166a95e1..c0f21efd8b2f92dcb137a344d7cdcf4b6d338752 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4a9b23b48d00a250039cfd2765632d6e5bb7cfe45d3bb6fd9de91cf1fd374183 -size 932514 +oid sha256:4250d8514d25432ab0371f39c76a9f05c6674e6bc4bb9d0101da89fae4c8d2ac +size 904943 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_medium/tuning-impact-elo.png.zip index dd3ec9f360edc5513b4af9045cd919a550e75597..0bfec23b0336b85d32686edf2a5c69c95fbdb0e1 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2249bd657be968fbaec47b182e32bd1eed209524c3c08f04c03eff4517edf112 -size 118425 +oid sha256:808a8dd5bbfd5d44f283ef8815b0a1a5239c73148fd7d5df8d3e1b01234fa1a9 +size 124955 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_binary/datasets_medium/website_leaderboard.csv index 8faf9eec1b35da574394391d25a2752f0e3a4754..74bedd369a8354e86552db101dfc506eb078ddc3 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_medium/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_binary/datasets_medium/website_leaderboard.csv @@ -1,54 +1,55 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1966.0,+155/-60,2.212,0.904,4.95,2.42,275.07,2.448,✔️,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1904.0,+176/-64,2.49,0.851,6.35,2.7,325.91,2.156,✔️,0.0,False,CPU -2,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1759.0,+141/-88,4.725,0.631,10.98,5.87,619.15,0.198,✔️,0.0,False,CPU -3,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1754.0,+144/-81,5.018,0.636,11.17,4.87,619.15,0.016,✔️,0.0,False,CPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1751.0,+113/-38,5.44,0.575,11.28,8.0,1633.8,2.053,✔️,0.0,False,GPU -5,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1736.0,+126/-79,4.763,0.63,11.87,5.47,2.21,0.025,✔️,0.0,False,CPU -6,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1718.0,+152/-92,7.508,0.545,12.6,6.95,110.13,0.386,✔️,0.0,False,CPU -7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1701.0,+149/-83,7.923,0.524,13.3,8.51,86.85,0.668,✔️,0.0,False,CPU -8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1681.0,+192/-136,7.27,0.513,14.19,5.59,804.65,0.71,✔️,0.0,False,GPU -9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1679.0,+154/-138,6.907,0.544,14.26,7.1,2139.17,6.41,✔️,0.0,False,GPU -10,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1660.0,+210/-122,8.381,0.489,15.1,5.39,842.24,15.119,✔️,0.0,False,GPU -11,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1652.0,+124/-66,7.028,0.376,15.51,8.78,1633.8,0.088,✔️,0.0,False,GPU -12,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1640.0,+124/-80,8.376,0.421,16.04,11.5,110.13,0.061,✔️,0.0,False,CPU -13,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1625.0,+197/-156,7.978,0.458,16.73,6.29,804.65,0.063,✔️,0.0,False,GPU -14,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1610.0,+194/-117,9.033,0.419,17.44,6.83,842.24,1.465,✔️,0.0,False,GPU -15,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1582.0,+158/-128,8.63,0.416,18.81,7.95,2.3,0.896,✔️,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1580.0,+163/-89,9.74,0.323,18.95,12.1,86.85,0.078,✔️,0.0,False,CPU -17,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1564.0,+170/-166,9.211,0.42,19.72,8.41,8.53,3.578,✔️,0.0,False,GPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1552.0,+126/-84,9.782,0.253,20.35,12.42,2139.17,0.349,✔️,0.0,False,GPU -19,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1536.0,+178/-158,9.019,0.347,21.12,7.53,4860.88,475.791,✔️,0.0,False,GPU -20,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1527.0,+95/-112,9.12,0.248,21.62,14.48,1156.69,1.558,✔️,0.0,False,CPU -21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1490.0,+141/-106,10.678,0.196,23.53,15.19,572.82,1.935,✔️,0.0,False,GPU -22,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1478.0,+124/-111,12.87,0.174,24.2,17.96,270.2,0.192,✔️,0.0,False,CPU -23,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1461.0,+126/-146,10.861,0.228,25.07,13.69,0.49,0.053,✔️,0.0,False,CPU -24,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1454.0,+132/-118,12.698,0.163,25.44,19.32,572.82,0.109,✔️,0.0,False,GPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1453.0,+156/-178,10.49,0.241,25.48,14.6,5.5,0.074,✔️,0.0,False,GPU -26,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1431.0,+105/-129,11.432,0.123,26.67,20.62,371.59,2.386,✔️,0.0,False,CPU -27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1430.0,+81/-65,10.923,0.077,26.75,23.68,6.41,0.247,✔️,0.0,False,GPU -28,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1417.0,+121/-138,13.96,0.136,27.44,20.16,96.61,0.754,➖,0.0,False,GPU -29,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1404.0,+129/-107,13.997,0.091,28.11,24.03,270.2,0.015,✔️,0.0,False,CPU -30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1380.0,+97/-126,11.531,0.069,29.38,23.47,1156.69,0.054,✔️,0.0,False,CPU -31,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1374.0,+134/-137,12.762,0.087,29.67,18.69,4860.88,42.641,✔️,0.0,False,GPU -32,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1361.0,+129/-110,15.831,0.05,30.35,26.41,1.07,0.018,✔️,0.0,False,CPU -33,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1343.0,+98/-137,12.461,0.069,31.22,28.09,0.29,0.033,✔️,0.0,False,CPU -34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1319.0,+52/-59,14.148,0.002,32.42,30.58,42.95,0.226,✔️,0.0,False,CPU -35,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1314.0,+88/-85,13.953,0.008,32.69,29.68,8.34,0.25,✔️,0.0,False,GPU -36,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1284.0,+174/-186,17.019,0.035,34.12,20.85,47.03,45.876,✔️,0.0,False,GPU -37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1280.0,+61/-90,13.457,0.0,34.32,32.02,40.36,0.318,✔️,0.0,False,CPU -38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1279.0,+74/-74,15.128,0.005,34.35,30.6,42.95,0.021,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1278.0,+84/-119,15.031,0.036,34.41,29.27,371.59,0.102,✔️,0.0,False,CPU -40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1224.0,+66/-92,15.173,0.0,36.86,35.23,1.86,0.046,✔️,0.0,False,CPU -41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1218.0,+64/-73,15.174,0.0,37.12,35.45,40.36,0.028,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1206.0,+134/-218,18.046,0.042,37.63,32.31,1.52,0.115,✔️,0.0,False,CPU -43,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1162.0,+147/-233,24.034,0.009,39.45,35.34,43.64,0.085,✔️,0.0,False,CPU -44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1140.0,+101/-190,20.729,0.011,40.28,31.54,513.06,78.02,✔️,0.0,False,CPU -45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1112.0,+166/-225,25.027,0.005,41.35,38.03,43.64,0.022,✔️,0.0,False,CPU -46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+88/-208,22.275,0.0,44.96,44.23,0.12,0.014,✔️,0.0,False,CPU -47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),982.0,+135/-219,26.73,0.0,45.44,43.9,0.22,0.024,✔️,0.0,False,CPU -48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),972.0,+202/-455,28.513,0.025,45.72,39.86,2.62,0.186,✔️,0.0,False,GPU -49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),918.0,+91/-188,30.352,0.0,47.05,46.26,0.24,0.036,➖,0.0,False,GPU -50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),916.0,+100/-247,24.142,0.0,47.09,46.63,0.1,0.021,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),883.0,+76/-126,27.794,0.0,47.81,47.49,513.06,2.568,✔️,0.0,False,CPU -52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),514.0,+107/-398,45.591,0.0,52.27,52.24,0.11,0.087,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1968.0,+156/-62,0.906,4.96,2.42,2.212,275.07,2.448,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1905.0,+178/-66,0.854,6.38,2.7,2.49,325.91,2.156,✔️,0.0,False,CPU +2,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1761.0,+143/-87,0.637,11.0,5.87,4.725,619.15,0.198,✔️,0.0,False,CPU +3,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1755.0,+146/-82,0.642,11.22,4.87,5.018,619.15,0.016,✔️,0.0,False,CPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1753.0,+113/-39,0.582,11.32,8.0,5.44,1633.8,2.053,✔️,0.0,False,GPU +5,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1738.0,+125/-77,0.636,11.91,5.47,4.763,2.21,0.025,✔️,0.0,False,CPU +6,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1720.0,+153/-91,0.549,12.65,6.95,7.508,110.13,0.386,✔️,0.0,False,CPU +7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1703.0,+149/-82,0.529,13.35,8.51,7.923,86.85,0.668,✔️,0.0,False,CPU +8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1681.0,+192/-136,0.518,14.32,5.6,7.27,804.65,0.71,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1679.0,+157/-141,0.551,14.42,7.11,6.907,2139.17,6.41,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1660.0,+212/-122,0.491,15.26,5.39,8.381,842.24,15.119,✔️,0.0,False,GPU +11,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1652.0,+126/-66,0.386,15.63,8.8,7.028,1633.8,0.088,✔️,0.0,False,GPU +12,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1642.0,+124/-79,0.427,16.1,11.52,8.376,110.13,0.061,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1626.0,+199/-159,0.465,16.9,6.3,7.978,804.65,0.063,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1611.0,+198/-117,0.424,17.63,6.84,9.033,842.24,1.465,✔️,0.0,False,GPU +15,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1584.0,+160/-128,0.421,18.93,7.96,8.63,2.3,0.896,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1580.0,+164/-90,0.331,19.15,12.14,9.74,86.85,0.078,✔️,0.0,False,CPU +17,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1564.0,+171/-168,0.425,19.96,8.43,9.211,8.53,3.578,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1552.0,+127/-86,0.26,20.58,12.46,9.782,2139.17,0.349,✔️,0.0,False,GPU +19,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1536.0,+179/-160,0.353,21.41,7.55,9.019,4860.88,475.791,✔️,0.0,False,GPU +20,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1528.0,+96/-114,0.254,21.81,14.53,9.12,1156.69,1.558,✔️,0.0,False,CPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1491.0,+143/-105,0.199,23.77,15.23,10.678,572.82,1.935,✔️,0.0,False,GPU +22,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1481.0,+125/-108,0.186,24.33,18.02,12.87,270.2,0.192,✔️,0.0,False,CPU +23,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1464.0,+128/-144,0.233,25.26,13.72,10.861,0.49,0.053,✔️,0.0,False,CPU +24,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1458.0,+131/-115,0.165,25.56,19.35,12.698,572.82,0.109,✔️,0.0,False,GPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1454.0,+159/-177,0.246,25.8,14.65,10.49,5.5,0.074,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1432.0,+106/-131,0.127,26.99,20.72,11.432,371.59,2.386,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1431.0,+85/-63,0.083,27.06,23.85,10.923,6.41,0.247,✔️,0.0,False,GPU +28,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1418.0,+125/-137,0.143,27.73,20.28,13.96,96.61,0.754,➖,0.0,False,GPU +29,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1407.0,+128/-104,0.093,28.33,24.18,13.997,270.2,0.015,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1380.0,+98/-128,0.072,29.77,23.63,11.531,1156.69,0.054,✔️,0.0,False,CPU +31,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1373.0,+134/-137,0.093,30.18,18.82,12.762,4860.88,42.641,✔️,0.0,False,GPU +32,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1363.0,+130/-112,0.056,30.7,26.58,15.831,1.07,0.018,✔️,0.0,False,CPU +33,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1344.0,+100/-140,0.076,31.68,28.34,12.461,0.29,0.033,✔️,0.0,False,CPU +34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1321.0,+50/-60,0.002,32.85,30.95,14.148,42.95,0.226,✔️,0.0,False,CPU +35,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1314.0,+91/-87,0.009,33.16,29.98,13.953,8.34,0.25,✔️,0.0,False,GPU +36,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1306.0,+69/-110,0.012,33.6,30.5,15.989,16.41,16.24,➖,0.0,False,GPU +37,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1283.0,+174/-189,0.046,34.7,21.0,17.019,47.03,45.876,✔️,0.0,False,GPU +38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1281.0,+74/-73,0.005,34.82,30.93,15.128,42.95,0.021,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1280.0,+61/-90,0.0,34.88,32.44,13.457,40.36,0.318,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1278.0,+82/-120,0.037,34.98,29.62,15.031,371.59,0.102,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1224.0,+69/-95,0.0,37.47,35.73,15.173,1.86,0.046,✔️,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1217.0,+64/-77,0.0,37.8,36.02,15.174,40.36,0.028,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1205.0,+135/-218,0.043,38.31,32.73,18.046,1.52,0.115,✔️,0.0,False,CPU +44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1159.0,+148/-233,0.008,40.24,36.0,24.034,43.64,0.085,✔️,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1136.0,+102/-192,0.011,41.16,32.1,20.729,513.06,78.02,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1110.0,+167/-222,0.005,42.18,38.76,25.027,43.64,0.022,✔️,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+90/-206,0.0,45.8,45.01,22.275,0.12,0.014,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),978.0,+135/-218,0.0,46.42,44.9,26.73,0.22,0.024,✔️,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),974.0,+201/-459,0.025,46.53,40.56,28.513,2.62,0.186,✔️,0.0,False,GPU +50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),917.0,+105/-246,0.0,47.96,47.47,24.142,0.1,0.021,✔️,0.0,False,CPU +51,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),915.0,+91/-188,0.0,48.01,47.21,30.352,0.24,0.036,➖,0.0,False,GPU +52,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),878.0,+76/-122,0.0,48.81,48.49,27.794,513.06,2.568,✔️,0.0,False,CPU +53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),511.0,+107/-401,0.0,53.27,53.24,45.591,0.11,0.087,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_medium/winrate_matrix.png.zip index c4773ddbf4e42ad14311d24ea8b280aecc4165da..740778e62a4effcca7aad9dadb91ee208e259fec 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:58c56a938a79e3e723918c17ab82e3c3d3b8ff9cc40b8406dc8b50fc9e44b474 -size 2102015 +oid sha256:38b9ee06f92035e8fc665f627f37d91e94c5834aed5261937efb6cdcd3402c95 +size 2292501 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index e5be691b3307f0609e62f6e030307f1e72466738..df3fd89519965dd25de7ebd91d83858f11e24d27 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d40895726f44af8a1e29ac7e059ab41df14c611842441e5a9b78be9f3322053f -size 430099 +oid sha256:60bf147936485bc9dbcd719634c64f9be1b0a8ff291f4d5460c8fc264fc3d789 +size 444040 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip index d67bf57dd2f3de70218eecde530406006d933c15..c3d4ac88ab34f0cc07af5f3b9951526cf5849297 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ab0ea4c367c7c476f7e280f5bf57a910fdc799f4cba65092cf45b083492ecbff -size 948765 +oid sha256:eb4a0d2bb4bef977ed289b9d286e4630091a23bdd9f2dbf55674aeb360896b5a +size 939014 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_small/tuning-impact-elo.png.zip index 62a3f9b1cf07d4136d238b73fd4ebf12c1f89433..34cdbe342ab828656a385484a8d4297d85ce6297 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f64abe4468182633e34180de1e59952ebdfea38b4510064e0d19100c01498bdc -size 124173 +oid sha256:d40c8835529c9fbf2450789660d93de66a7652095b93103d90725a0fa8a46c90 +size 128439 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_binary/datasets_small/website_leaderboard.csv index f58e76d008b313ab70e5b241e779e59f560fd177..13c4d921fa18c3092eede5e7865b70654e3b1c49 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_small/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_binary/datasets_small/website_leaderboard.csv @@ -1,53 +1,54 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1582.0,+110/-77,7.287,0.748,8.48,3.33,593.57,8.084,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1579.0,+98/-56,6.466,0.789,8.57,4.04,5.96,0.554,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1574.0,+142/-69,5.03,0.753,8.74,3.16,2052.28,8.252,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1547.0,+128/-57,6.337,0.713,9.74,3.78,2052.28,1.266,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1435.0,+81/-50,12.183,0.471,14.72,9.21,2950.72,15.595,✔️,0.0,False,GPU -5,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1425.0,+85/-56,12.33,0.434,15.23,8.35,2457.54,1.658,✔️,0.0,False,GPU -6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1415.0,+90/-69,11.866,0.417,15.75,8.92,1921.63,1.718,✔️,0.0,False,CPU -7,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1402.0,+82/-72,9.204,0.465,16.4,5.52,5351.68,268.579,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1390.0,+81/-66,14.126,0.467,17.07,7.72,317.91,1.983,➖,0.0,False,GPU -9,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1370.0,+83/-73,10.014,0.385,18.2,5.58,5351.68,37.621,✔️,0.0,False,GPU -10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1366.0,+49/-38,14.198,0.31,18.4,12.57,512.57,1.18,✔️,0.0,False,CPU -11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1361.0,+85/-59,13.333,0.327,18.67,10.34,2457.54,0.178,✔️,0.0,False,GPU -12,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1333.0,+44/-34,14.644,0.237,20.3,14.26,512.57,0.243,✔️,0.0,False,CPU -13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1330.0,+80/-60,14.072,0.259,20.46,10.21,2950.72,1.225,✔️,0.0,False,GPU -14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1319.0,+88/-68,14.168,0.243,21.12,8.79,4806.04,0.512,✔️,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1310.0,+52/-43,15.257,0.224,21.68,14.73,1096.5,0.584,✔️,0.0,False,CPU -16,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1309.0,+64/-47,15.195,0.253,21.73,13.86,4.81,0.08,✔️,0.0,False,CPU -17,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1305.0,+53/-43,15.264,0.222,21.97,15.14,1096.5,0.074,✔️,0.0,False,CPU -18,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1297.0,+88/-76,13.006,0.324,22.45,7.6,51.83,43.615,✔️,0.0,False,GPU -19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1290.0,+51/-68,15.211,0.214,22.82,12.25,711.84,2.983,✔️,0.0,False,GPU -20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1289.0,+47/-43,15.673,0.203,22.93,15.54,693.49,1.398,✔️,0.0,False,CPU -21,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1278.0,+68/-61,16.117,0.262,23.55,12.39,10.08,0.152,✔️,0.0,False,GPU -22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1276.0,+114/-107,15.466,0.292,23.67,9.47,4806.04,8.461,✔️,0.0,False,GPU -23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1267.0,+48/-48,16.409,0.176,24.26,14.62,1634.54,0.422,✔️,0.0,False,CPU -24,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1265.0,+62/-65,16.741,0.21,24.38,11.3,14.37,0.334,✔️,0.0,False,GPU -25,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1264.0,+42/-33,16.038,0.149,24.39,16.25,693.49,0.194,✔️,0.0,False,CPU -26,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1263.0,+53/-61,16.17,0.182,24.48,12.54,711.84,0.162,✔️,0.0,False,GPU -27,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1259.0,+81/-56,15.336,0.167,24.72,16.57,2665.36,2.619,✔️,0.0,False,CPU -28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1243.0,+49/-50,16.945,0.134,25.68,14.64,1634.54,0.054,✔️,0.0,False,CPU -29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1224.0,+62/-72,17.235,0.163,26.88,15.4,644.55,5.974,✔️,0.0,False,CPU -30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1222.0,+52/-53,17.663,0.153,26.97,15.18,7.32,0.054,✔️,0.0,False,CPU -31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1221.0,+56/-50,16.483,0.123,27.02,16.05,10.89,2.249,✔️,0.0,False,GPU -32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1185.0,+72/-49,17.457,0.08,29.23,18.73,2665.36,0.143,✔️,0.0,False,CPU -33,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1174.0,+64/-77,18.602,0.071,29.9,17.46,644.55,0.398,✔️,0.0,False,CPU -34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1167.0,+63/-83,19.966,0.087,30.28,19.64,203.15,0.968,✔️,0.0,False,CPU -35,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1162.0,+61/-59,18.743,0.058,30.6,22.99,1.94,0.121,✔️,0.0,False,CPU -36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1132.0,+45/-51,19.177,0.033,32.32,27.15,2.44,0.121,✔️,0.0,False,CPU -37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1131.0,+59/-76,20.887,0.054,32.38,21.08,203.15,0.091,✔️,0.0,False,CPU -38,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1107.0,+54/-85,21.614,0.049,33.76,20.87,484.22,1.025,✔️,0.0,False,CPU -39,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1074.0,+108/-127,23.463,0.142,35.56,18.38,1.3,0.15,➖,0.0,False,GPU -40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1073.0,+51/-82,22.235,0.024,35.64,21.78,484.22,0.086,✔️,0.0,False,CPU -41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1056.0,+61/-76,22.185,0.028,36.52,25.42,9.39,0.13,✔️,0.0,False,CPU -42,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1021.0,+89/-132,28.012,0.073,38.3,23.69,278.97,0.873,✔️,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1012.0,+76/-85,22.938,0.012,38.71,22.33,2.97,0.388,✔️,0.0,False,CPU -44,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+58/-111,25.166,0.018,39.3,30.97,0.42,0.041,✔️,0.0,False,CPU -45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),998.0,+96/-137,28.578,0.053,39.38,24.75,278.97,0.103,✔️,0.0,False,CPU -46,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),988.0,+75/-112,25.72,0.019,39.87,27.69,90.71,1.623,✔️,0.0,False,CPU -47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),986.0,+81/-105,25.768,0.027,39.92,31.85,3.71,0.919,✔️,0.0,False,GPU -48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),964.0,+79/-130,26.469,0.015,40.92,33.63,0.33,0.043,✔️,0.0,False,CPU -49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),952.0,+88/-162,30.627,0.034,41.4,27.1,1.48,0.139,✔️,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),921.0,+80/-144,29.993,0.001,42.66,27.85,90.71,0.125,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),616.0,+109/-176,43.126,0.0,49.97,44.93,0.22,0.02,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1579.0,+110/-76,0.73,8.69,3.35,7.361,593.57,8.084,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1578.0,+97/-57,0.767,8.74,4.21,6.542,5.96,0.554,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1572.0,+145/-68,0.741,8.94,3.22,5.104,2052.28,8.252,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1545.0,+131/-56,0.703,9.95,3.82,6.409,2052.28,1.266,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1433.0,+79/-50,0.441,15.09,9.43,12.254,2950.72,15.595,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1423.0,+84/-57,0.412,15.62,8.49,12.402,2457.54,1.658,✔️,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1412.0,+87/-69,0.411,16.17,9.22,11.938,1921.63,1.718,✔️,0.0,False,CPU +7,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1402.0,+83/-72,0.441,16.75,5.56,9.277,5351.68,268.579,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1389.0,+81/-66,0.442,17.46,7.88,14.196,317.91,1.983,➖,0.0,False,GPU +9,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1368.0,+83/-73,0.383,18.62,5.64,10.087,5351.68,37.621,✔️,0.0,False,GPU +10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1364.0,+49/-38,0.304,18.85,12.83,14.268,512.57,1.18,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1359.0,+84/-59,0.317,19.15,10.56,13.405,2457.54,0.178,✔️,0.0,False,GPU +12,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1332.0,+44/-34,0.233,20.79,14.54,14.714,512.57,0.243,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1329.0,+78/-60,0.246,20.93,10.42,14.141,2950.72,1.225,✔️,0.0,False,GPU +14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1318.0,+88/-70,0.235,21.62,9.19,14.237,4806.04,0.512,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1310.0,+52/-43,0.217,22.11,15.1,15.327,1096.5,0.584,✔️,0.0,False,CPU +16,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1308.0,+62/-46,0.233,22.19,14.15,15.265,4.81,0.08,✔️,0.0,False,CPU +17,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1308.0,+70/-72,0.341,22.23,8.05,15.885,9.05,1.763,➖,0.0,False,GPU +18,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1304.0,+53/-43,0.216,22.45,15.42,15.334,1096.5,0.074,✔️,0.0,False,CPU +19,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1295.0,+88/-75,0.322,22.98,7.75,13.079,51.83,43.615,✔️,0.0,False,GPU +20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1289.0,+50/-68,0.207,23.38,12.61,15.283,711.84,2.983,✔️,0.0,False,GPU +21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1287.0,+46/-42,0.195,23.47,15.89,15.743,693.49,1.398,✔️,0.0,False,CPU +22,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1277.0,+68/-60,0.243,24.1,13.14,16.187,10.08,0.152,✔️,0.0,False,GPU +23,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1276.0,+112/-108,0.286,24.17,9.59,15.532,4806.04,8.461,✔️,0.0,False,GPU +24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1266.0,+47/-48,0.158,24.82,15.22,16.479,1634.54,0.422,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1264.0,+60/-64,0.198,24.94,11.88,16.81,14.37,0.334,✔️,0.0,False,GPU +26,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1263.0,+41/-34,0.139,24.97,16.55,16.108,693.49,0.194,✔️,0.0,False,CPU +27,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1262.0,+52/-62,0.169,25.06,12.79,16.239,711.84,0.162,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1257.0,+81/-56,0.157,25.34,16.93,15.403,2665.36,2.619,✔️,0.0,False,CPU +29,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1242.0,+49/-49,0.118,26.26,14.95,17.014,1634.54,0.054,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1222.0,+60/-72,0.156,27.55,15.83,17.306,644.55,5.974,✔️,0.0,False,CPU +31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1221.0,+51/-54,0.137,27.6,15.68,17.733,7.32,0.054,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1219.0,+55/-50,0.114,27.71,16.67,16.552,10.89,2.249,✔️,0.0,False,GPU +33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1184.0,+72/-49,0.073,29.91,19.12,17.523,2665.36,0.143,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1172.0,+63/-77,0.066,30.61,18.16,18.673,644.55,0.398,✔️,0.0,False,CPU +35,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1166.0,+64/-81,0.086,30.96,19.99,20.037,203.15,0.968,✔️,0.0,False,CPU +36,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1162.0,+59/-57,0.056,31.26,23.37,18.81,1.94,0.121,✔️,0.0,False,CPU +37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1132.0,+45/-52,0.032,33.0,27.65,19.243,2.44,0.121,✔️,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1131.0,+60/-76,0.054,33.09,21.73,20.959,203.15,0.091,✔️,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1107.0,+53/-85,0.048,34.49,21.37,21.684,484.22,1.025,✔️,0.0,False,CPU +40,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1073.0,+108/-128,0.134,36.37,18.98,23.536,1.3,0.15,➖,0.0,False,GPU +41,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1073.0,+51/-83,0.024,36.4,23.97,22.305,484.22,0.086,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1055.0,+60/-79,0.02,37.33,26.07,22.252,9.39,0.13,✔️,0.0,False,CPU +43,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1020.0,+89/-134,0.073,39.13,24.1,28.077,278.97,0.873,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1012.0,+77/-84,0.005,39.52,23.1,23.004,2.97,0.388,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+56/-111,0.018,40.12,32.28,25.235,0.42,0.041,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),998.0,+96/-137,0.053,40.21,25.22,28.642,278.97,0.103,✔️,0.0,False,CPU +47,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),988.0,+75/-112,0.019,40.66,28.13,25.787,90.71,1.623,✔️,0.0,False,CPU +48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),986.0,+81/-107,0.02,40.77,32.62,25.835,3.71,0.919,✔️,0.0,False,GPU +49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),965.0,+80/-126,0.015,41.74,34.42,26.537,0.33,0.043,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),953.0,+88/-165,0.034,42.25,27.54,30.687,1.48,0.139,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),922.0,+80/-143,0.001,43.51,28.3,30.058,90.71,0.125,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),616.0,+110/-170,0.0,50.94,45.74,43.18,0.22,0.02,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_small/winrate_matrix.png.zip index 925697745b0a416fe30abcfef3e5ed7d44affdbe..90b3145cd4bbe0fcc0bde1365bcdc9afa14e2e0e 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4e57ff1a51e7be9c3d7151b16069c74a1f7f9e3f496c85bae2adec08d1e77971 -size 2084457 +oid sha256:451fa68829f359020f611702c326ebe2f21cf475567b036577ffc70af6482870 +size 2288454 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index decb6cd3cbfff460cf1890b30f9a8dcead7475ca..cb895c193064b01ed66377d90db7ffb980380fd1 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fc63af627d7d502e12dd05fc5f18682e1b650cb18c1e0bb177b285928b9554b3 -size 483576 +oid sha256:20a32305ccc350545c5ed1bf9dcde1c8d723f8fc80d906fc92a8400e246d28cb +size 498546 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip index 441ddcb6d8b81460ae65c8af6325fea082ee4d86..4625ed78ab60a45262b87159656292713848a012 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b8727986cd9b18c958c81b0a916b643bec00333f9951d32fa6a0f2dd174260bc -size 1037072 +oid sha256:b3e0bdeb7a5748cf52834d6acbaa62422d0b71fcf1b0fe36b98e283cbc1c3921 +size 1017812 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip index 535c74cbf4aba51654e4e5ee8fa87ffcd3383773..b5cba50bbab9d914d967cd4221ef5f84a71cf07a 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:81d0e1e01312078f508feaa69219e0a44ff430aa9dc161d2f04a568393e1a8a2 -size 131179 +oid sha256:e9e00a3632dacd90463b03673a6a80a2d206a91aa959e9bee41c007fa269a533 +size 136669 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/website_leaderboard.csv index 27e234f4851f3594c157236e6b1e39eca32dae81..44a8f600299a84276935de595a073267406f6fbb 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1600.0,+91/-54,7.184,0.729,10.36,5.33,5.52,0.529,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1588.0,+113/-69,5.826,0.685,10.87,4.46,2046.25,8.114,✔️,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1584.0,+88/-56,8.183,0.662,11.04,4.67,592.15,8.082,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1569.0,+104/-64,7.089,0.657,11.74,4.79,2046.25,1.254,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1510.0,+92/-60,10.733,0.585,14.67,5.88,144.27,3.772,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1497.0,+79/-54,10.68,0.512,15.33,6.23,3445.6,51.274,✔️,0.0,False,GPU -6,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1486.0,+98/-79,11.297,0.565,15.96,5.68,3.76,0.527,➖,0.0,False,GPU -7,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1479.0,+79/-67,10.213,0.493,16.35,5.93,8.06,1.714,✔️,0.0,False,GPU -8,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1442.0,+65/-50,13.145,0.385,18.49,11.6,2879.46,14.626,✔️,0.0,False,GPU -9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1438.0,+75/-59,13.186,0.362,18.78,9.99,2379.18,1.595,✔️,0.0,False,GPU -10,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1430.0,+69/-59,14.423,0.406,19.25,9.32,316.0,1.931,➖,0.0,False,GPU -11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1419.0,+76/-58,12.9,0.339,19.93,11.82,1828.68,1.648,✔️,0.0,False,CPU -12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1419.0,+84/-68,10.011,0.391,19.94,6.64,5319.2,249.314,✔️,0.0,False,GPU -13,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1415.0,+108/-75,13.416,0.407,20.2,6.42,3445.6,0.995,✔️,0.0,False,GPU -14,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1399.0,+104/-71,10.637,0.361,21.18,6.68,5319.2,29.488,✔️,0.0,False,GPU -15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1381.0,+86/-58,14.163,0.273,22.36,12.32,2379.18,0.176,✔️,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1376.0,+45/-44,15.222,0.233,22.67,15.32,449.53,1.149,✔️,0.0,False,CPU -17,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1372.0,+96/-85,15.503,0.405,22.91,8.63,3.73,0.488,✔️,0.0,False,GPU -18,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1352.0,+65/-52,14.896,0.22,24.25,12.67,2879.46,0.944,✔️,0.0,False,GPU -19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1347.0,+63/-58,14.824,0.225,24.59,10.35,4575.49,0.51,✔️,0.0,False,GPU -20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1347.0,+38/-37,15.649,0.17,24.6,17.27,449.53,0.215,✔️,0.0,False,CPU -21,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1326.0,+105/-79,13.65,0.303,25.96,8.98,51.44,43.709,✔️,0.0,False,GPU -22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1326.0,+61/-49,16.157,0.189,25.97,16.69,4.7,0.08,✔️,0.0,False,CPU -23,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1323.0,+56/-53,16.287,0.151,26.23,17.99,1081.93,0.556,✔️,0.0,False,CPU -24,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1316.0,+56/-45,16.312,0.159,26.64,19.09,1081.93,0.074,✔️,0.0,False,CPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1308.0,+67/-67,16.789,0.219,27.19,14.07,9.81,0.146,✔️,0.0,False,GPU -26,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1302.0,+53/-58,16.244,0.162,27.58,15.63,592.34,2.767,✔️,0.0,False,GPU -27,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1298.0,+52/-52,16.778,0.138,27.86,19.56,645.57,1.032,✔️,0.0,False,CPU -28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1295.0,+66/-71,17.352,0.174,28.07,13.53,14.36,0.311,✔️,0.0,False,GPU -29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1292.0,+50/-50,17.207,0.129,28.32,17.46,1618.46,0.415,✔️,0.0,False,CPU -30,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1290.0,+97/-103,16.484,0.254,28.39,11.16,4575.49,8.396,✔️,0.0,False,GPU -31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1286.0,+69/-58,15.992,0.14,28.68,19.64,2576.18,2.415,✔️,0.0,False,CPU -32,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1280.0,+54/-58,17.188,0.127,29.14,15.16,592.34,0.146,✔️,0.0,False,GPU -33,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1275.0,+42/-42,17.137,0.084,29.47,19.6,645.57,0.172,✔️,0.0,False,CPU -34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1271.0,+52/-55,17.71,0.1,29.74,16.79,1618.46,0.054,✔️,0.0,False,CPU -35,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1254.0,+54/-58,18.247,0.115,30.88,16.81,6.86,0.052,✔️,0.0,False,CPU -36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1253.0,+67/-78,17.887,0.143,30.93,17.0,618.9,5.468,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1250.0,+55/-56,17.097,0.088,31.15,18.15,10.47,1.987,✔️,0.0,False,GPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1213.0,+63/-50,17.997,0.062,33.59,21.56,2576.18,0.136,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1206.0,+67/-74,19.212,0.058,34.09,19.4,618.9,0.352,✔️,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1183.0,+84/-72,21.088,0.067,35.56,23.19,199.83,0.865,✔️,0.0,False,CPU -41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1172.0,+64/-60,19.888,0.025,36.23,29.08,1.77,0.119,✔️,0.0,False,CPU -42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1150.0,+78/-78,21.933,0.04,37.6,25.09,199.83,0.086,✔️,0.0,False,CPU -43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1144.0,+40/-43,20.296,0.018,38.0,32.13,2.2,0.12,✔️,0.0,False,CPU -44,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1110.0,+95/-131,23.531,0.115,40.03,21.41,1.24,0.143,➖,0.0,False,GPU -45,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1108.0,+65/-85,22.968,0.02,40.14,26.17,440.56,0.933,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1086.0,+75/-74,22.66,0.018,41.44,29.26,8.39,0.127,✔️,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1078.0,+52/-79,23.518,0.004,41.88,24.91,440.56,0.085,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1057.0,+98/-123,28.113,0.071,43.04,26.02,273.27,0.715,✔️,0.0,False,CPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1043.0,+78/-87,23.546,0.0,43.76,24.97,2.84,0.381,✔️,0.0,False,CPU -50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1034.0,+94/-126,28.584,0.047,44.19,27.79,273.27,0.098,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1020.0,+83/-108,26.329,0.02,44.93,30.67,89.68,1.55,✔️,0.0,False,CPU -52,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1002.0,+98/-112,26.937,0.011,45.79,36.55,2.96,0.796,✔️,0.0,False,GPU -53,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+63/-105,26.578,0.003,45.87,37.86,0.42,0.04,✔️,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),989.0,+102/-139,30.404,0.03,46.39,30.04,1.44,0.138,✔️,0.0,False,CPU -55,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),976.0,+93/-114,27.729,0.012,46.96,38.72,0.31,0.042,✔️,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),953.0,+86/-138,30.595,0.001,47.95,30.7,89.68,0.12,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),639.0,+130/-215,43.912,0.0,55.9,49.79,0.21,0.019,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1599.0,+90/-55,0.723,10.53,5.5,7.231,5.52,0.529,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1587.0,+114/-70,0.683,11.08,4.49,5.873,2046.25,8.114,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1583.0,+88/-56,0.658,11.25,4.69,8.229,592.15,8.082,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1567.0,+106/-65,0.656,11.95,4.83,7.134,2046.25,1.254,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),1509.0,+91/-59,0.583,14.91,5.98,10.778,144.27,3.772,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1495.0,+80/-55,0.508,15.65,6.32,10.726,3445.6,51.274,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1486.0,+98/-78,0.555,16.17,5.79,11.342,3.76,0.527,➖,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1478.0,+80/-66,0.489,16.63,6.0,10.261,8.06,1.714,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1441.0,+65/-51,0.377,18.88,11.78,13.189,2879.46,14.626,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1436.0,+74/-57,0.355,19.16,10.11,13.231,2379.18,1.595,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1429.0,+70/-59,0.403,19.6,9.45,14.468,316.0,1.931,➖,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1418.0,+85/-67,0.384,20.3,6.67,10.056,5319.2,249.314,✔️,0.0,False,GPU +12,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1417.0,+77/-58,0.341,20.36,12.1,12.945,1828.68,1.648,✔️,0.0,False,CPU +13,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1414.0,+107/-75,0.401,20.53,6.55,13.459,3445.6,0.995,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1398.0,+105/-70,0.362,21.58,6.73,10.683,5319.2,29.488,✔️,0.0,False,GPU +15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1379.0,+86/-58,0.269,22.84,12.52,14.209,2379.18,0.176,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1374.0,+44/-43,0.233,23.14,15.56,15.267,449.53,1.149,✔️,0.0,False,CPU +17,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1371.0,+97/-83,0.393,23.35,8.81,15.549,3.73,0.488,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1351.0,+65/-52,0.22,24.7,12.84,14.939,2879.46,0.944,✔️,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1346.0,+63/-58,0.227,25.06,10.58,14.868,4575.49,0.51,✔️,0.0,False,GPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1345.0,+38/-37,0.172,25.11,17.55,15.693,449.53,0.215,✔️,0.0,False,CPU +21,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1326.0,+61/-49,0.187,26.43,16.97,16.201,4.7,0.08,✔️,0.0,False,CPU +22,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1325.0,+106/-80,0.304,26.47,9.08,13.696,51.44,43.709,✔️,0.0,False,GPU +23,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1323.0,+83/-71,0.282,26.63,10.3,16.884,8.81,1.55,➖,0.0,False,GPU +24,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1322.0,+56/-53,0.151,26.66,18.33,16.332,1081.93,0.556,✔️,0.0,False,CPU +25,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1316.0,+55/-43,0.16,27.12,19.39,16.356,1081.93,0.074,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1307.0,+67/-67,0.213,27.72,15.01,16.834,9.81,0.146,✔️,0.0,False,GPU +27,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1301.0,+53/-58,0.161,28.16,16.07,16.289,592.34,2.767,✔️,0.0,False,GPU +28,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1297.0,+51/-52,0.139,28.42,19.99,16.823,645.57,1.032,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1294.0,+66/-71,0.17,28.61,13.79,17.396,14.36,0.311,✔️,0.0,False,GPU +30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1291.0,+49/-50,0.126,28.86,17.91,17.251,1618.46,0.415,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1290.0,+95/-103,0.256,28.89,11.23,16.525,4575.49,8.396,✔️,0.0,False,GPU +32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1285.0,+69/-59,0.142,29.28,19.96,16.034,2576.18,2.415,✔️,0.0,False,CPU +33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1278.0,+55/-57,0.128,29.72,15.38,17.23,592.34,0.146,✔️,0.0,False,GPU +34,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1273.0,+40/-43,0.083,30.07,19.91,17.181,645.57,0.172,✔️,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1270.0,+51/-55,0.097,30.3,17.07,17.754,1618.46,0.054,✔️,0.0,False,CPU +36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1253.0,+54/-58,0.107,31.5,17.3,18.291,6.86,0.052,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1251.0,+67/-78,0.143,31.58,17.4,17.932,618.9,5.468,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1248.0,+55/-56,0.088,31.83,18.75,17.14,10.47,1.987,✔️,0.0,False,GPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1212.0,+63/-51,0.064,34.26,21.93,18.039,2576.18,0.136,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1204.0,+67/-73,0.058,34.79,20.06,19.257,618.9,0.352,✔️,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1182.0,+84/-72,0.065,36.24,23.51,21.134,199.83,0.865,✔️,0.0,False,CPU +42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1172.0,+64/-62,0.028,36.89,29.5,19.93,1.77,0.119,✔️,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1150.0,+78/-74,0.038,38.3,25.55,21.978,199.83,0.086,✔️,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1144.0,+40/-44,0.019,38.69,32.62,20.337,2.2,0.12,✔️,0.0,False,CPU +45,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1109.0,+94/-132,0.115,40.84,21.9,23.577,1.24,0.143,➖,0.0,False,GPU +46,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1108.0,+65/-83,0.021,40.89,26.68,23.012,440.56,0.933,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1085.0,+76/-74,0.02,42.24,29.74,22.701,8.39,0.127,✔️,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1078.0,+52/-80,0.004,42.65,27.65,23.562,440.56,0.085,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1056.0,+97/-122,0.071,43.86,26.41,28.155,273.27,0.715,✔️,0.0,False,CPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1043.0,+79/-88,0.0,44.57,25.84,23.586,2.84,0.381,✔️,0.0,False,CPU +51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1034.0,+95/-124,0.048,45.02,28.26,28.625,273.27,0.098,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1020.0,+84/-108,0.02,45.71,31.07,26.372,89.68,1.55,✔️,0.0,False,CPU +53,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1002.0,+98/-114,0.008,46.63,37.26,26.979,2.96,0.796,✔️,0.0,False,GPU +54,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+63/-102,0.004,46.71,39.28,26.621,0.42,0.04,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),989.0,+101/-141,0.03,47.23,30.44,30.443,1.44,0.138,✔️,0.0,False,CPU +56,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),977.0,+92/-114,0.012,47.78,39.45,27.771,0.31,0.042,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),954.0,+87/-138,0.001,48.79,31.12,30.638,89.68,0.12,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),640.0,+129/-216,0.0,56.88,50.58,43.946,0.21,0.019,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip index b5366efe7d501801d000b176d670477926f0b280..5303fe9f02e10d7bfecb17071d3f686c5862fad2 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:72ab1298a6f3165d254d76a2c5aa9e6ea2499200355cb437bfdcf786928aa9d8 -size 2690375 +oid sha256:8640c9c709b634cf85846d074101e4a5112dd521ad730aaeebafa8aa51ed138c +size 2706479 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index c47bf98234fa0788bb44e5fea02621e4587e1b94..587db7f60ca8a510b321ad40e48793ffadd11509 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6bd7e126d5b12ad8d2b6f45ebf02c1452f74196f161c38c111ba598254da30cb -size 439786 +oid sha256:bef075960a412db7d54869057aa3071e0e1b2acd9e87da68f6776bd1e52d863c +size 452817 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip index 0e8129363e8b8a456e07db6b3aa3ad2563fcdca9..2103cf27918fb50631f758e3141895512d91b6a1 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e55ea134c309bccc8dcd6084b333f316ab1c52773176fb266c6c7cb5e741b316 -size 954824 +oid sha256:c2b6794c6e4b8a37d717193e09e45332c79ba5e2042aceb406b06c1e51597ce7 +size 934561 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_all/tuning-impact-elo.png.zip index 1543dc1c1f199869ee441354203147e76116fb15..df37efbedb3072021ea264f615797fc55f118c90 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7cdd342b55a7bf109dbe983c9c4f7bcc6c822faea1908ad566d313d241d44775 -size 113343 +oid sha256:8635c9734b4820b42f3cf14274f9d82e2f62466995adc7fb2e409d29327ed29f +size 129422 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_classification/datasets_all/website_leaderboard.csv index 0607ce1a12a2a33c77b04fd6a4152dc88e12db83..495203ce042ce8b77ca29d1b52963a6df4251d15 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_all/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_classification/datasets_all/website_leaderboard.csv @@ -1,53 +1,54 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1673.0,+76/-63,6.447,0.809,7.05,3.08,545.23,6.469,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1593.0,+81/-76,5.921,0.709,9.74,3.16,2046.25,8.976,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1566.0,+79/-70,6.967,0.671,10.79,3.98,2046.25,1.329,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1550.0,+72/-69,7.392,0.683,11.47,4.59,5.76,0.794,✔️,0.0,False,GPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1516.0,+78/-58,10.128,0.533,13.01,5.2,1684.49,1.97,✔️,0.0,False,CPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1500.0,+62/-54,10.684,0.508,13.74,8.2,2879.46,12.49,✔️,0.0,False,GPU -6,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1472.0,+76/-48,12.019,0.449,15.12,7.64,2466.21,1.502,✔️,0.0,False,GPU -7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1440.0,+48/-36,13.132,0.374,16.81,11.12,382.05,1.488,✔️,0.0,False,CPU -8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1421.0,+77/-56,11.076,0.428,17.83,6.22,4940.61,307.751,✔️,0.0,False,GPU -9,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1417.0,+73/-58,12.922,0.367,18.06,8.85,2466.21,0.176,✔️,0.0,False,GPU -10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1416.0,+58/-47,12.344,0.328,18.11,9.33,2879.46,0.598,✔️,0.0,False,GPU -11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1415.0,+54/-46,12.625,0.349,18.15,10.54,1372.94,0.556,✔️,0.0,False,CPU -12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1406.0,+58/-44,12.832,0.338,18.65,10.04,1372.94,0.074,✔️,0.0,False,CPU -13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1396.0,+41/-34,13.856,0.279,19.26,13.36,382.05,0.254,✔️,0.0,False,CPU -14,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1386.0,+53/-47,13.978,0.305,19.81,11.85,685.87,1.455,✔️,0.0,False,CPU -15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1383.0,+50/-46,13.472,0.325,19.97,10.59,5.72,0.076,✔️,0.0,False,CPU -16,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1376.0,+57/-54,15.277,0.354,20.4,10.32,282.72,1.886,➖,0.0,False,GPU -17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1371.0,+89/-82,13.51,0.345,20.64,9.1,4879.89,8.744,✔️,0.0,False,GPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1370.0,+60/-41,13.555,0.238,20.74,9.81,4879.89,0.525,✔️,0.0,False,GPU -19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1359.0,+47/-44,14.358,0.245,21.35,14.58,685.87,0.205,✔️,0.0,False,CPU -20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1358.0,+67/-58,12.802,0.307,21.4,7.33,4940.61,41.606,✔️,0.0,False,GPU -21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1337.0,+44/-45,14.984,0.226,22.61,13.17,934.1,3.063,✔️,0.0,False,GPU -22,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1318.0,+43/-36,16.035,0.183,23.73,15.62,2686.11,0.47,✔️,0.0,False,CPU -23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1315.0,+61/-54,15.709,0.254,23.93,13.32,10.21,0.138,✔️,0.0,False,GPU -24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1314.0,+45/-43,14.908,0.184,23.95,16.27,2389.22,2.158,✔️,0.0,False,CPU -25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1294.0,+51/-54,16.495,0.175,25.17,14.33,934.1,0.169,✔️,0.0,False,GPU -26,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1280.0,+45/-38,16.759,0.126,25.99,16.88,2686.11,0.054,✔️,0.0,False,CPU -27,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1277.0,+67/-64,16.109,0.238,26.17,9.08,49.21,43.824,✔️,0.0,False,GPU -28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1266.0,+61/-60,17.277,0.169,26.82,15.31,618.9,4.766,✔️,0.0,False,CPU -29,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1252.0,+44/-46,17.998,0.128,27.62,18.04,6.86,0.057,✔️,0.0,False,CPU -30,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1252.0,+46/-45,17.837,0.125,27.62,15.26,14.78,0.346,✔️,0.0,False,GPU -31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1243.0,+40/-37,17.108,0.087,28.19,19.4,10.47,1.707,✔️,0.0,False,GPU -32,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1232.0,+48/-52,17.195,0.106,28.81,19.48,1.77,0.117,✔️,0.0,False,CPU -33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1228.0,+47/-37,17.147,0.081,29.06,20.0,2389.22,0.152,✔️,0.0,False,CPU -34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1214.0,+52/-52,18.724,0.1,29.84,19.19,189.76,0.743,✔️,0.0,False,CPU -35,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1195.0,+53/-59,19.123,0.074,30.91,19.12,618.9,0.298,✔️,0.0,False,CPU -36,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1188.0,+64/-57,19.635,0.09,31.32,18.57,323.74,0.743,✔️,0.0,False,CPU -37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1188.0,+48/-38,18.263,0.065,31.32,26.28,1.79,0.12,✔️,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1172.0,+54/-54,19.957,0.073,32.24,19.76,189.76,0.079,✔️,0.0,False,CPU -39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1150.0,+60/-56,20.504,0.062,33.44,18.14,323.74,0.076,✔️,0.0,False,CPU -40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1085.0,+45/-56,21.856,0.015,36.8,28.14,6.83,0.147,✔️,0.0,False,CPU -41,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1041.0,+86/-88,29.56,0.043,38.9,28.09,252.58,0.528,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1038.0,+63/-67,24.207,0.017,39.03,26.67,2.91,0.368,✔️,0.0,False,CPU -43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1019.0,+67/-93,27.374,0.02,39.87,26.93,139.84,3.615,✔️,0.0,False,CPU -44,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1013.0,+86/-103,28.483,0.079,40.12,25.02,1.11,0.189,➖,0.0,False,GPU -45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1010.0,+87/-92,30.319,0.031,40.23,29.46,252.58,0.089,✔️,0.0,False,CPU -46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1005.0,+73/-85,27.121,0.032,40.44,33.02,3.61,0.939,✔️,0.0,False,GPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+58/-63,26.637,0.01,40.67,34.46,0.38,0.037,✔️,0.0,False,CPU -48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),947.0,+68/-72,28.882,0.008,42.72,37.34,0.25,0.041,✔️,0.0,False,CPU -49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),927.0,+94/-111,33.16,0.019,43.43,32.94,1.29,0.131,✔️,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),900.0,+84/-117,33.585,0.004,44.32,32.53,139.84,0.193,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),563.0,+111/-112,49.469,0.0,50.66,47.64,0.18,0.029,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1669.0,+76/-62,0.799,7.22,3.13,6.694,545.23,6.469,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1590.0,+84/-75,0.702,9.93,3.23,6.231,2046.25,8.976,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1564.0,+79/-69,0.665,10.99,4.06,7.277,2046.25,1.329,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1549.0,+70/-69,0.668,11.64,4.83,7.723,5.76,0.794,✔️,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1512.0,+75/-57,0.529,13.34,5.33,10.293,1684.49,1.97,✔️,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1497.0,+62/-53,0.491,14.04,8.47,10.95,2879.46,12.49,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1470.0,+74/-48,0.438,15.44,7.76,12.209,2466.21,1.502,✔️,0.0,False,GPU +7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1438.0,+49/-36,0.371,17.15,11.3,13.284,382.05,1.488,✔️,0.0,False,CPU +8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1419.0,+76/-55,0.412,18.2,6.28,11.26,4940.61,307.751,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1414.0,+71/-57,0.362,18.46,8.99,13.099,2466.21,0.176,✔️,0.0,False,GPU +10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1414.0,+54/-46,0.342,18.49,10.72,12.859,1372.94,0.556,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1414.0,+58/-47,0.32,18.5,9.62,12.585,2879.46,0.598,✔️,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1404.0,+58/-44,0.331,19.02,10.17,13.063,1372.94,0.074,✔️,0.0,False,CPU +13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1393.0,+41/-34,0.277,19.66,13.6,14.01,382.05,0.254,✔️,0.0,False,CPU +14,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1384.0,+52/-46,0.299,20.21,12.02,14.152,685.87,1.455,✔️,0.0,False,CPU +15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1382.0,+50/-46,0.314,20.34,10.74,13.612,5.72,0.076,✔️,0.0,False,CPU +16,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1374.0,+56/-53,0.341,20.8,10.53,15.45,282.72,1.886,➖,0.0,False,GPU +17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1370.0,+87/-81,0.341,21.05,9.2,13.701,4879.89,8.744,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1368.0,+61/-42,0.234,21.16,10.12,13.731,4879.89,0.525,✔️,0.0,False,GPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1357.0,+47/-43,0.238,21.77,14.81,14.536,685.87,0.205,✔️,0.0,False,CPU +20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1356.0,+67/-57,0.307,21.86,7.42,12.959,4940.61,41.606,✔️,0.0,False,GPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1335.0,+44/-45,0.218,23.1,13.49,15.198,934.1,3.063,✔️,0.0,False,GPU +22,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1317.0,+44/-35,0.175,24.18,16.09,16.189,2686.11,0.47,✔️,0.0,False,CPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1313.0,+59/-53,0.246,24.44,13.9,15.854,10.21,0.138,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1312.0,+46/-43,0.176,24.47,16.54,15.04,2389.22,2.158,✔️,0.0,False,CPU +25,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1302.0,+58/-72,0.276,25.08,8.55,16.27,13.7,2.033,➖,0.0,False,GPU +26,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1293.0,+51/-54,0.167,25.65,14.59,16.694,934.1,0.169,✔️,0.0,False,GPU +27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1279.0,+45/-38,0.117,26.47,17.19,16.916,2686.11,0.054,✔️,0.0,False,CPU +28,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1275.0,+67/-65,0.239,26.72,9.22,16.216,49.21,43.824,✔️,0.0,False,GPU +29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1264.0,+60/-59,0.164,27.4,15.69,17.391,618.9,4.766,✔️,0.0,False,CPU +30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1251.0,+44/-45,0.121,28.17,18.5,18.133,6.86,0.057,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1251.0,+46/-45,0.119,28.17,15.89,17.971,14.78,0.346,✔️,0.0,False,GPU +32,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1241.0,+40/-36,0.084,28.81,20.01,17.234,10.47,1.707,✔️,0.0,False,GPU +33,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1232.0,+48/-52,0.106,29.32,19.69,17.334,1.77,0.117,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1226.0,+47/-39,0.08,29.66,20.33,17.256,2389.22,0.152,✔️,0.0,False,CPU +35,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1213.0,+52/-51,0.098,30.45,19.47,18.91,189.76,0.743,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1194.0,+53/-57,0.071,31.59,19.73,19.231,618.9,0.298,✔️,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1188.0,+64/-56,0.087,31.93,18.98,19.848,323.74,0.743,✔️,0.0,False,CPU +38,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1187.0,+48/-39,0.066,31.94,26.71,18.379,1.79,0.12,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1171.0,+54/-53,0.072,32.9,20.18,20.13,189.76,0.079,✔️,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1150.0,+60/-57,0.061,34.08,19.24,20.715,323.74,0.076,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1084.0,+45/-56,0.011,37.55,28.82,21.982,6.83,0.147,✔️,0.0,False,CPU +42,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1039.0,+85/-86,0.042,39.73,28.62,29.688,252.58,0.528,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1037.0,+64/-66,0.013,39.83,27.44,24.295,2.91,0.368,✔️,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1019.0,+66/-94,0.022,40.65,27.31,27.469,139.84,3.615,✔️,0.0,False,CPU +45,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1012.0,+85/-101,0.074,40.95,25.75,28.567,1.11,0.189,➖,0.0,False,GPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1010.0,+86/-88,0.03,41.07,30.05,30.426,252.58,0.089,✔️,0.0,False,CPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1006.0,+72/-84,0.027,41.24,33.86,27.299,3.61,0.939,✔️,0.0,False,GPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+59/-60,0.01,41.48,35.59,26.755,0.38,0.037,✔️,0.0,False,CPU +49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),948.0,+69/-70,0.008,43.55,38.13,28.978,0.25,0.041,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),926.0,+93/-110,0.019,44.32,33.53,33.241,1.29,0.131,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),900.0,+84/-113,0.006,45.19,33.06,33.665,139.84,0.193,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),563.0,+112/-114,0.0,51.64,48.53,49.526,0.18,0.029,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_all/winrate_matrix.png.zip index 9689db2383f16efea97dd4a44998dda025206909..bdeebda5f2b7b448ba5d9423743758f282294d58 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:597cf8a2debe07fef7194817772d424e9b90a8e5f88e729d87c6341999c84549 -size 2041622 +oid sha256:db72e662f286b00c2c3cb7219cee77ea31d7942f2cdda891ac3a5d1b44d251f7 +size 2232084 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index 0d69d1eb6040c27d5df279b04d4bb43269463529..9d8ddc1d283812f139c0ad3cb7b03ae7766a5e20 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3de46251d70777fedf2009f664158f3a1cd46dacd611adc6a93e81b90ea28f65 -size 420152 +oid sha256:302b07241d8634309bb8ef8846b09f68fda276904b6eb1a7d767235508e81849 +size 436728 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip index 82d2c46b8e0567393e14501a1a72e241e9a880ae..eea4ca3ea600403e5c20adc27b44fbaf3f66d19d 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1c48c1cc97b1dfd4312245ceb7067e74666b06c9186291e43ecc8d89924f64f4 -size 948975 +oid sha256:23dd51cebaaa9b90df0c413fa46de62cf211c8149ca4982b3704042bbe79c0b4 +size 918132 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_medium/tuning-impact-elo.png.zip index 42d7e906f6eb60e18a313a886587491e44e981ca..9ba28b8aa79e20629a40ea445ae3f92ab2165ea9 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5be0ed1ffd3ea931c4655b13af19386480aeb6e9d5856f5840543c4b489bb95a -size 118664 +oid sha256:04662e06d0bdc8f825e73a086c5524e5e651d5a0c1ce3c72c6eab3549ac3dbbc +size 122715 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_classification/datasets_medium/website_leaderboard.csv index 2286344fd351cbaf8cd052f34c36713535526adb..2ba052aa3188ae1469b53010a586f2eee45cc93d 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_medium/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_classification/datasets_medium/website_leaderboard.csv @@ -1,54 +1,55 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1929.0,+143/-60,2.178,0.907,4.76,2.46,275.97,2.818,✔️,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1873.0,+172/-66,2.411,0.859,5.97,2.64,314.17,2.249,✔️,0.0,False,CPU -2,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1707.0,+129/-66,5.406,0.618,11.18,6.2,697.8,0.199,✔️,0.0,False,CPU -3,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1705.0,+141/-72,5.651,0.623,11.24,5.16,697.8,0.023,✔️,0.0,False,CPU -4,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1676.0,+124/-64,5.593,0.609,12.42,5.87,2.44,0.027,✔️,0.0,False,CPU -5,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1673.0,+138/-96,7.874,0.542,12.54,7.25,207.65,0.398,✔️,0.0,False,CPU -6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1669.0,+144/-74,8.132,0.529,12.7,8.34,93.21,0.758,✔️,0.0,False,CPU -7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1651.0,+120/-91,7.253,0.517,13.46,8.65,1624.54,1.763,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1646.0,+218/-129,7.545,0.54,13.7,3.82,783.29,13.79,✔️,0.0,False,GPU -9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1626.0,+148/-132,7.556,0.53,14.59,7.55,2338.19,14.248,✔️,0.0,False,GPU -10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1604.0,+180/-118,8.633,0.462,15.6,6.08,1398.9,0.645,✔️,0.0,False,GPU -11,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1599.0,+111/-77,8.689,0.429,15.82,11.69,207.65,0.056,✔️,0.0,False,CPU -12,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1594.0,+205/-121,8.372,0.468,16.03,6.06,783.29,1.924,✔️,0.0,False,GPU -13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1567.0,+129/-90,8.727,0.338,17.34,9.49,1624.54,0.083,✔️,0.0,False,GPU -14,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1554.0,+167/-78,9.811,0.346,17.97,11.64,93.21,0.087,✔️,0.0,False,CPU -15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1554.0,+190/-148,9.345,0.412,17.98,6.83,1398.9,0.062,✔️,0.0,False,GPU -16,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1541.0,+154/-119,9.121,0.414,18.63,8.37,2.3,0.82,✔️,0.0,False,GPU -17,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1518.0,+166/-148,9.743,0.413,19.76,8.93,7.51,3.146,✔️,0.0,False,GPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1508.0,+114/-82,10.245,0.263,20.28,12.89,2338.19,0.54,✔️,0.0,False,GPU -19,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1486.0,+156/-141,9.963,0.326,21.43,8.09,4805.66,461.797,✔️,0.0,False,GPU -20,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1476.0,+89/-75,10.081,0.234,21.91,15.1,1169.3,1.442,✔️,0.0,False,CPU -21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1466.0,+146/-106,12.538,0.216,22.47,15.25,893.93,0.205,✔️,0.0,False,CPU -22,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1438.0,+156/-129,10.925,0.255,23.92,13.66,0.47,0.052,✔️,0.0,False,CPU -23,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1422.0,+131/-134,12.437,0.177,24.79,16.12,709.85,2.929,✔️,0.0,False,GPU -24,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1398.0,+160/-107,13.588,0.139,26.12,19.84,893.93,0.016,✔️,0.0,False,CPU -25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1388.0,+130/-132,14.508,0.146,26.66,20.31,709.85,0.19,✔️,0.0,False,GPU -26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1385.0,+149/-155,12.673,0.217,26.8,15.57,5.67,0.081,✔️,0.0,False,GPU -27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1359.0,+84/-82,13.533,0.069,28.18,24.73,6.98,0.233,✔️,0.0,False,GPU -28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1357.0,+96/-140,14.453,0.11,28.29,21.75,389.62,2.114,✔️,0.0,False,CPU -29,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1350.0,+138/-132,15.914,0.122,28.69,21.21,84.36,0.919,➖,0.0,False,GPU -30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1345.0,+81/-84,12.262,0.073,28.91,23.58,1169.3,0.053,✔️,0.0,False,CPU -31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1342.0,+125/-102,15.622,0.084,29.11,25.2,2.09,0.019,✔️,0.0,False,CPU -32,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1329.0,+117/-121,13.696,0.078,29.75,19.44,4805.66,43.211,✔️,0.0,False,GPU -33,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1327.0,+118/-103,13.691,0.062,29.86,22.08,52.45,0.219,✔️,0.0,False,CPU -34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1294.0,+94/-111,13.707,0.062,31.52,28.6,0.28,0.033,✔️,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1289.0,+127/-116,14.607,0.063,31.78,23.78,52.45,0.021,✔️,0.0,False,CPU -36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1281.0,+79/-74,14.625,0.007,32.17,29.44,10.5,0.606,✔️,0.0,False,GPU -37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1262.0,+71/-88,13.747,0.025,33.1,30.65,38.42,0.32,✔️,0.0,False,CPU -38,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1249.0,+123/-152,17.564,0.032,33.75,21.52,46.9,45.623,✔️,0.0,False,GPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1218.0,+79/-144,17.632,0.032,35.18,30.19,389.62,0.112,✔️,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1203.0,+77/-86,15.534,0.011,35.87,33.94,38.42,0.028,✔️,0.0,False,CPU -41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1179.0,+62/-68,16.733,0.0,36.96,35.47,1.87,0.046,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1145.0,+110/-215,21.37,0.038,38.39,33.26,1.61,0.112,✔️,0.0,False,CPU -43,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1103.0,+142/-227,26.799,0.008,40.08,36.17,43.46,0.087,✔️,0.0,False,CPU -44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1070.0,+93/-195,24.389,0.01,41.31,32.77,331.68,44.947,✔️,0.0,False,CPU -45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1051.0,+151/-241,28.022,0.005,41.96,38.8,43.46,0.021,✔️,0.0,False,CPU -46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+102/-183,22.382,0.0,43.65,42.56,0.12,0.013,✔️,0.0,False,CPU -47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),927.0,+140/-232,29.594,0.0,45.73,44.31,0.2,0.023,✔️,0.0,False,CPU -48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),916.0,+205/-468,31.186,0.022,46.02,40.6,2.8,0.224,✔️,0.0,False,GPU -49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),884.0,+91/-212,26.748,0.0,46.79,46.36,0.09,0.019,✔️,0.0,False,CPU -50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),861.0,+107/-173,32.966,0.0,47.32,46.59,0.2,0.033,➖,0.0,False,GPU -51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),816.0,+64/-150,32.095,0.0,48.23,47.9,331.68,1.626,✔️,0.0,False,CPU -52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),451.0,+130/-393,49.299,0.0,52.34,52.31,0.12,0.101,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1931.0,+144/-62,0.908,4.77,2.46,2.178,275.97,2.818,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1875.0,+174/-67,0.862,6.0,2.64,2.411,314.17,2.249,✔️,0.0,False,CPU +2,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1710.0,+130/-66,0.624,11.2,6.2,5.406,697.8,0.199,✔️,0.0,False,CPU +3,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1708.0,+141/-72,0.629,11.28,5.16,5.651,697.8,0.023,✔️,0.0,False,CPU +4,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1679.0,+124/-63,0.615,12.45,5.87,5.593,2.44,0.027,✔️,0.0,False,CPU +5,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1675.0,+138/-94,0.546,12.59,7.25,7.874,207.65,0.398,✔️,0.0,False,CPU +6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1672.0,+146/-74,0.534,12.74,8.35,8.132,93.21,0.758,✔️,0.0,False,CPU +7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1653.0,+122/-92,0.523,13.57,8.66,7.253,1624.54,1.763,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1646.0,+220/-128,0.542,13.84,3.83,7.545,783.29,13.79,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1626.0,+150/-134,0.537,14.73,7.56,7.556,2338.19,14.248,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1605.0,+182/-118,0.466,15.72,6.09,8.633,1398.9,0.645,✔️,0.0,False,GPU +11,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1602.0,+112/-76,0.435,15.88,11.71,8.689,207.65,0.056,✔️,0.0,False,CPU +12,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1595.0,+205/-121,0.472,16.2,6.06,8.372,783.29,1.924,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1567.0,+131/-91,0.347,17.56,9.51,8.727,1624.54,0.083,✔️,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1555.0,+169/-78,0.353,18.14,11.67,9.811,93.21,0.087,✔️,0.0,False,CPU +15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1555.0,+193/-149,0.418,18.16,6.83,9.345,1398.9,0.062,✔️,0.0,False,GPU +16,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1544.0,+156/-120,0.419,18.73,8.38,9.121,2.3,0.82,✔️,0.0,False,GPU +17,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1519.0,+166/-148,0.418,19.98,8.94,9.743,7.51,3.146,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1509.0,+117/-85,0.27,20.49,12.93,10.245,2338.19,0.54,✔️,0.0,False,GPU +19,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1486.0,+157/-142,0.331,21.69,8.1,9.963,4805.66,461.797,✔️,0.0,False,GPU +20,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1479.0,+89/-75,0.241,22.09,15.15,10.081,1169.3,1.442,✔️,0.0,False,CPU +21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1469.0,+146/-106,0.227,22.59,15.28,12.538,893.93,0.205,✔️,0.0,False,CPU +22,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1442.0,+155/-130,0.26,24.09,13.69,10.925,0.47,0.052,✔️,0.0,False,CPU +23,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1423.0,+130/-132,0.179,25.1,16.19,12.437,709.85,2.929,✔️,0.0,False,GPU +24,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1401.0,+160/-106,0.141,26.32,19.93,13.588,893.93,0.016,✔️,0.0,False,CPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1391.0,+131/-130,0.149,26.86,20.37,14.508,709.85,0.19,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1385.0,+153/-156,0.221,27.19,15.63,12.673,5.67,0.081,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1360.0,+88/-84,0.075,28.56,24.93,13.533,6.98,0.233,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1358.0,+96/-144,0.114,28.68,21.87,14.453,389.62,2.114,✔️,0.0,False,CPU +29,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1351.0,+137/-132,0.129,29.04,21.35,15.914,84.36,0.919,➖,0.0,False,GPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1347.0,+82/-85,0.076,29.26,23.72,12.262,1169.3,0.053,✔️,0.0,False,CPU +31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1344.0,+128/-104,0.09,29.43,25.34,15.622,2.09,0.019,✔️,0.0,False,CPU +32,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1329.0,+118/-122,0.084,30.23,19.57,13.696,4805.66,43.211,✔️,0.0,False,GPU +33,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1329.0,+119/-102,0.062,30.24,22.25,13.691,52.45,0.219,✔️,0.0,False,CPU +34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1294.0,+96/-114,0.068,32.03,28.9,13.707,0.28,0.033,✔️,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1291.0,+125/-116,0.063,32.21,23.96,14.607,52.45,0.021,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1284.0,+81/-75,0.009,32.59,29.7,14.625,10.5,0.606,✔️,0.0,False,GPU +37,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1267.0,+66/-87,0.011,33.4,30.6,16.633,14.85,13.554,➖,0.0,False,GPU +38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1263.0,+73/-89,0.026,33.6,30.99,13.747,38.42,0.32,✔️,0.0,False,CPU +39,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1249.0,+123/-152,0.042,34.33,21.7,17.564,46.9,45.623,✔️,0.0,False,GPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1218.0,+81/-145,0.033,35.79,30.58,17.632,389.62,0.112,✔️,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1204.0,+79/-87,0.012,36.48,34.41,15.534,38.42,0.028,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1179.0,+64/-71,0.0,37.6,36.01,16.733,1.87,0.046,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1145.0,+111/-217,0.038,39.1,33.72,21.37,1.61,0.112,✔️,0.0,False,CPU +44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1101.0,+144/-226,0.007,40.89,36.85,26.799,43.46,0.087,✔️,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1066.0,+95/-196,0.009,42.2,33.36,24.389,331.68,44.947,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1050.0,+152/-240,0.004,42.81,39.55,28.022,43.46,0.021,✔️,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+102/-176,0.0,44.47,43.32,22.382,0.12,0.013,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),923.0,+140/-230,0.0,46.71,45.3,29.594,0.2,0.023,✔️,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),918.0,+204/-476,0.022,46.86,41.32,31.186,2.8,0.224,✔️,0.0,False,GPU +50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),884.0,+94/-216,0.0,47.68,47.22,26.748,0.09,0.019,✔️,0.0,False,CPU +51,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),858.0,+107/-172,0.0,48.29,47.55,32.966,0.2,0.033,➖,0.0,False,GPU +52,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),811.0,+63/-148,0.0,49.23,48.91,32.095,331.68,1.626,✔️,0.0,False,CPU +53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),448.0,+130/-396,0.0,53.34,53.31,49.299,0.12,0.101,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_medium/winrate_matrix.png.zip index d4409a41585bf3b0274c232c4c84c4b432c88c4e..44b408e4332155a5c14bad501b4c9f8244e890e2 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a2e75580dd3f3c6afecb6c21e7f64ae847ec30929e333433a88451173b51bd16 -size 2104005 +oid sha256:97a54e9fb0a50d52cc2306a830ec60d5b0a77ec5c41dcd03a9a466c3fdbc2139 +size 2309021 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 5b385da2ef35154fa03f28ef132a93289089ae94..9334434632c3add8390f835cab4dc61e6b3b2aa8 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b68ab421a3bda314adeded5f2b2fd1d2d3844a597aed2f6ba7647f3724f0562e -size 420646 +oid sha256:9e6a94a54c2ff74cf1545db169f60c947a50f421b66c4ccf373711d4b5f27a28 +size 435340 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip index 31a2558814ee137da8a22d407611ce57a5cfe093..036f8206caced5b32ac8298d0b39eddebd540510 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e5191ebaf888402c65abc94e978357b0102867a75dbf7539a0ef3ba4d1973cb8 -size 952213 +oid sha256:73b049af82c00fd424c0d8116af43675868ee975c80f151aa34b86473a9448f3 +size 925949 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_small/tuning-impact-elo.png.zip index 2c2741cc99cee98c8e40330b918a25b307feffa1..b2844df365cf10afa274538300cfe59bad5839eb 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:851c168ac8aba376fa906cd6d598d0bd80af847498978961fb367476367d974f -size 124086 +oid sha256:821e6f8dd661b78ea068dd791c685843be3c3aa90531e45e67faae5128a24b80 +size 128623 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_classification/datasets_small/website_leaderboard.csv index 204a78d70f070d6eabfbb2f0575514b4a7a17aa4..b27b3deedc43c8fc7acd8a18600ab25c55afeb50 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_small/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_classification/datasets_small/website_leaderboard.csv @@ -1,53 +1,54 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1621.0,+89/-68,7.988,0.765,7.91,3.4,610.76,8.082,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1605.0,+95/-83,5.358,0.761,8.43,2.98,2289.05,8.114,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1588.0,+82/-68,6.481,0.736,9.04,3.55,2289.05,1.254,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1587.0,+87/-84,6.793,0.769,9.05,3.97,6.07,0.762,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1470.0,+71/-56,11.927,0.499,13.99,8.09,3422.82,18.936,✔️,0.0,False,GPU -5,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1448.0,+70/-57,13.246,0.44,15.11,8.48,3170.35,1.694,✔️,0.0,False,GPU -6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1438.0,+81/-56,12.901,0.408,15.62,8.07,3143.8,1.884,✔️,0.0,False,CPU -7,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1416.0,+66/-62,11.489,0.457,16.78,5.78,5271.34,231.986,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1399.0,+74/-53,15.067,0.434,17.68,8.76,330.95,2.359,➖,0.0,False,GPU -9,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1388.0,+62/-63,14.216,0.345,18.27,10.0,3170.35,0.241,✔️,0.0,False,GPU -10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1386.0,+46/-33,14.934,0.31,18.39,12.83,735.1,1.901,✔️,0.0,False,CPU -11,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1383.0,+68/-54,13.652,0.32,18.57,9.32,3422.82,1.534,✔️,0.0,False,GPU -12,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1380.0,+70/-62,12.497,0.384,18.72,6.02,5271.34,33.986,✔️,0.0,False,GPU -13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1360.0,+48/-29,15.316,0.249,19.89,14.24,735.1,0.311,✔️,0.0,False,CPU -14,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1345.0,+47/-39,15.222,0.246,20.75,14.59,1714.62,0.768,✔️,0.0,False,CPU -15,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1340.0,+55/-50,14.753,0.227,21.06,9.12,6047.72,0.525,✔️,0.0,False,GPU -16,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1334.0,+42/-38,15.413,0.228,21.39,15.26,1714.62,0.098,✔️,0.0,False,CPU -17,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1322.0,+58/-58,15.908,0.242,22.07,12.42,1053.84,3.063,✔️,0.0,False,GPU -18,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1314.0,+49/-44,16.175,0.21,22.55,15.66,797.9,2.228,✔️,0.0,False,CPU -19,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1310.0,+53/-40,16.305,0.216,22.77,15.24,7.19,0.118,✔️,0.0,False,CPU -20,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1308.0,+81/-81,15.656,0.273,22.93,10.09,6047.72,8.744,✔️,0.0,False,GPU -21,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1304.0,+66/-53,16.81,0.265,23.13,12.73,10.78,0.17,✔️,0.0,False,GPU -22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1298.0,+46/-42,16.399,0.173,23.49,16.12,797.9,0.353,✔️,0.0,False,CPU -23,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1293.0,+77/-78,15.602,0.306,23.79,7.55,51.44,43.709,✔️,0.0,False,GPU -24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1283.0,+48/-40,17.3,0.166,24.41,15.85,3351.28,0.544,✔️,0.0,False,CPU -25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1275.0,+57/-55,17.218,0.183,24.89,13.03,1053.84,0.169,✔️,0.0,False,GPU -26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1275.0,+57/-53,16.647,0.16,24.91,16.94,2977.49,2.415,✔️,0.0,False,CPU -27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1254.0,+47/-42,17.908,0.121,26.17,16.12,3351.28,0.064,✔️,0.0,False,CPU -28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1251.0,+61/-61,18.997,0.166,26.3,13.09,15.54,0.346,✔️,0.0,False,GPU -29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1248.0,+71/-62,18.299,0.189,26.54,13.9,1423.89,9.208,✔️,0.0,False,CPU -30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1234.0,+57/-48,18.862,0.142,27.33,16.49,8.03,0.059,✔️,0.0,False,CPU -31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1216.0,+46/-41,18.399,0.092,28.43,18.13,14.89,4.433,✔️,0.0,False,GPU -32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1207.0,+79/-69,20.517,0.126,28.96,17.02,399.55,1.466,✔️,0.0,False,CPU -33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1200.0,+46/-51,18.904,0.084,29.36,19.11,2977.49,0.156,✔️,0.0,False,CPU -34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1196.0,+61/-50,19.665,0.088,29.63,16.97,1423.89,0.749,✔️,0.0,False,CPU -35,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1176.0,+43/-48,19.449,0.044,30.77,24.27,2.79,0.238,✔️,0.0,False,CPU -36,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1168.0,+80/-76,21.549,0.094,31.24,17.28,399.55,0.174,✔️,0.0,False,CPU -37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1162.0,+48/-50,19.906,0.063,31.56,25.83,3.15,0.141,✔️,0.0,False,CPU -38,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1153.0,+71/-87,21.772,0.099,32.1,17.65,571.65,1.39,✔️,0.0,False,CPU -39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1114.0,+72/-74,22.622,0.062,34.3,16.83,571.65,0.129,✔️,0.0,False,CPU -40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1062.0,+62/-63,23.697,0.021,37.04,26.37,11.29,0.204,✔️,0.0,False,CPU -41,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1044.0,+97/-100,26.892,0.107,37.89,21.55,2.59,0.533,➖,0.0,False,GPU -42,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1027.0,+75/-96,30.557,0.055,38.74,26.12,483.02,0.874,✔️,0.0,False,CPU -43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1026.0,+72/-102,25.68,0.035,38.78,31.12,4.95,1.012,✔️,0.0,False,GPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1009.0,+65/-65,25.23,0.009,39.55,25.04,5.04,0.646,✔️,0.0,False,CPU -45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1007.0,+84/-107,28.452,0.023,39.66,25.44,121.17,1.63,✔️,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1002.0,+76/-98,31.151,0.04,39.88,27.22,483.02,0.113,✔️,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+64/-71,28.169,0.014,39.96,32.48,0.46,0.068,✔️,0.0,False,CPU -48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),960.0,+77/-98,29.657,0.011,41.63,35.13,0.42,0.074,✔️,0.0,False,CPU -49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),926.0,+96/-133,34.444,0.026,42.94,30.32,2.39,0.143,✔️,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),917.0,+99/-112,34.126,0.005,43.28,29.33,121.17,0.136,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),582.0,+111/-168,49.533,0.0,50.41,46.46,0.28,0.023,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1616.0,+86/-65,0.75,8.15,3.47,8.323,610.76,8.082,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1601.0,+96/-84,0.751,8.64,3.07,5.779,2289.05,8.114,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1584.0,+85/-84,0.747,9.25,4.22,7.242,6.07,0.762,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1584.0,+81/-69,0.727,9.25,3.63,6.901,2289.05,1.254,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1466.0,+71/-54,0.473,14.36,8.46,12.288,3422.82,18.936,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1444.0,+68/-56,0.423,15.5,8.67,13.504,3170.35,1.694,✔️,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1434.0,+79/-55,0.401,16.05,8.48,13.125,3143.8,1.884,✔️,0.0,False,CPU +7,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1413.0,+66/-63,0.434,17.18,5.84,11.74,5271.34,231.986,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1396.0,+74/-53,0.414,18.1,8.96,15.301,330.95,2.359,➖,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1385.0,+61/-63,0.337,18.75,10.23,14.457,3170.35,0.241,✔️,0.0,False,GPU +10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1383.0,+46/-33,0.304,18.84,13.15,15.141,735.1,1.901,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1380.0,+68/-53,0.307,19.03,9.7,13.979,3422.82,1.534,✔️,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1378.0,+70/-62,0.382,19.17,6.1,12.71,5271.34,33.986,✔️,0.0,False,GPU +13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1357.0,+48/-30,0.244,20.37,14.6,15.526,735.1,0.311,✔️,0.0,False,CPU +14,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1343.0,+46/-38,0.235,21.2,15.06,15.539,1714.62,0.768,✔️,0.0,False,CPU +15,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1337.0,+53/-50,0.219,21.56,9.47,14.991,6047.72,0.525,✔️,0.0,False,GPU +16,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1332.0,+41/-37,0.217,21.88,15.67,15.727,1714.62,0.098,✔️,0.0,False,CPU +17,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1323.0,+78/-75,0.371,22.38,6.81,16.156,13.27,1.624,➖,0.0,False,GPU +18,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1319.0,+57/-58,0.23,22.62,12.79,16.198,1053.84,3.063,✔️,0.0,False,GPU +19,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1312.0,+47/-43,0.202,23.08,16.06,16.41,797.9,2.228,✔️,0.0,False,CPU +20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1309.0,+53/-39,0.199,23.27,15.65,16.495,7.19,0.118,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1306.0,+81/-81,0.265,23.43,10.26,15.914,6047.72,8.744,✔️,0.0,False,GPU +22,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1302.0,+64/-54,0.252,23.68,13.44,17.008,10.78,0.17,✔️,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1296.0,+45/-40,0.162,24.04,16.49,16.64,797.9,0.353,✔️,0.0,False,CPU +24,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1291.0,+77/-77,0.304,24.33,7.67,15.748,51.44,43.709,✔️,0.0,False,GPU +25,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1281.0,+47/-40,0.152,24.97,16.5,17.509,3351.28,0.544,✔️,0.0,False,CPU +26,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1273.0,+57/-54,0.172,25.47,13.32,17.488,1053.84,0.169,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1272.0,+55/-56,0.147,25.54,17.32,16.826,2977.49,2.415,✔️,0.0,False,CPU +28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1252.0,+47/-41,0.108,26.75,16.48,18.121,3351.28,0.064,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1250.0,+60/-62,0.157,26.9,13.7,19.18,15.54,0.346,✔️,0.0,False,GPU +30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1245.0,+71/-63,0.181,27.19,14.32,18.454,1423.89,9.208,✔️,0.0,False,CPU +31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1232.0,+57/-48,0.13,27.96,16.99,19.044,8.03,0.059,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1213.0,+46/-40,0.086,29.15,18.8,18.57,14.89,4.433,✔️,0.0,False,GPU +33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1205.0,+78/-71,0.124,29.62,17.28,20.768,399.55,1.466,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1198.0,+46/-53,0.081,30.04,19.49,19.052,2977.49,0.156,✔️,0.0,False,CPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1193.0,+61/-51,0.084,30.34,17.58,19.813,1423.89,0.749,✔️,0.0,False,CPU +36,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1176.0,+43/-48,0.042,31.4,24.7,19.637,2.79,0.238,✔️,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1167.0,+79/-74,0.093,31.92,17.66,21.783,399.55,0.174,✔️,0.0,False,CPU +38,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1162.0,+48/-51,0.062,32.22,26.3,20.062,3.15,0.141,✔️,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1152.0,+71/-87,0.096,32.79,18.12,22.06,571.65,1.39,✔️,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1113.0,+71/-75,0.061,35.01,18.1,22.908,571.65,0.129,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1061.0,+60/-64,0.015,37.82,27.06,23.869,11.29,0.204,✔️,0.0,False,CPU +42,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1044.0,+94/-97,0.101,38.68,22.21,27.006,2.59,0.533,➖,0.0,False,GPU +43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1026.0,+70/-101,0.029,39.57,31.98,25.922,4.95,1.012,✔️,0.0,False,GPU +44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1025.0,+74/-95,0.055,39.58,26.61,30.732,483.02,0.874,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1009.0,+65/-65,0.004,40.37,25.87,25.35,5.04,0.646,✔️,0.0,False,CPU +46,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1008.0,+83/-108,0.026,40.4,25.77,28.581,121.17,1.63,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1001.0,+76/-98,0.04,40.72,27.77,31.295,483.02,0.113,✔️,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+63/-74,0.013,40.76,33.69,28.328,0.46,0.068,✔️,0.0,False,CPU +49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),961.0,+76/-104,0.011,42.43,35.89,29.787,0.42,0.074,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),927.0,+94/-132,0.026,43.79,30.83,34.553,2.39,0.143,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),918.0,+97/-110,0.009,44.1,29.77,34.235,121.17,0.136,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),582.0,+112/-169,0.0,51.39,47.3,49.61,0.28,0.023,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_small/winrate_matrix.png.zip index a59f6bcb090e5855bbc57cf6b35d1650b79b3287..bb105186bface4d0763fd668a649dedfec1c8273 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:47b5ac64d47ef026c71a1cbec9e7ca65f2bab86b9c5b85ad7859e5202f98e093 -size 2071503 +oid sha256:0ef7532ab74712ad24011ddcd54f0af7352109048acd1623196f1032f92fff8e +size 2243389 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index d0789551547daa806cd048d487b830c24347101d..954095a8d9cac4f53fec0d85ea6cbb86abf4bf34 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c5da55dc35c03ecd57aa841f4bcf3f2ba689959fa1cc7422af2258e654132c31 -size 471650 +oid sha256:631d54b41d73a61bc44ea306bf80342c6a27154ba3099a7a41860ece50b8074f +size 485745 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip index d5488542d481e8975273be1732609a1bad753463..0223a03d746474bfe62bbddf92bf7e1d5f4acbb9 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:da4d76b4ddee7f40f2f4099a92c7a1c66f1430dafc58cc1df67047a2eae597ef -size 1051061 +oid sha256:fce35d562f908b5a65d150fd3ef23d1e67e673cc4145df09d2334bbdf1ce34f8 +size 1031647 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip index 3c944c29bc3eefa8f0efa4d9baae08dd77cb0c64..1ceae16bbf3eb6c6121567039bb48c1519b805a4 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5a57f5ff76c703004367a4e66acca6c3b122bf30dff44eca330154f76fd45acd -size 120230 +oid sha256:6ab2b819a83da4610f983f30e40dfc15c853d009e1b442cb75e1904908c6e7cc +size 121995 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/website_leaderboard.csv index 9eb8e140ebb7cc10688c97ebfcf83baa97abecd1..1f9cf6227b8613a285e17df780d5726301fd6b67 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1634.0,+83/-79,6.234,0.728,9.78,3.8,2168.17,7.961,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1631.0,+70/-57,7.553,0.747,9.87,4.93,5.99,0.633,✔️,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1625.0,+85/-64,9.074,0.701,10.13,4.67,604.44,8.082,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1614.0,+81/-69,7.377,0.702,10.59,4.34,2168.17,1.23,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1544.0,+69/-56,11.21,0.576,13.78,5.67,3445.6,48.236,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1536.0,+80/-59,11.857,0.593,14.17,6.16,146.65,4.101,✔️,0.0,False,GPU -6,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1518.0,+87/-73,11.42,0.58,15.09,5.58,4.37,0.585,➖,0.0,False,GPU -7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1486.0,+61/-44,13.101,0.447,16.86,9.64,3169.9,16.793,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1459.0,+71/-61,12.877,0.447,18.45,6.6,8.89,1.743,✔️,0.0,False,GPU -9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1454.0,+71/-46,14.494,0.373,18.73,10.01,2828.45,1.596,✔️,0.0,False,GPU -10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1452.0,+82/-72,13.931,0.442,18.89,6.45,3445.6,0.995,✔️,0.0,False,GPU -11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1444.0,+66/-55,14.093,0.356,19.32,10.43,2898.23,1.884,✔️,0.0,False,CPU -12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1425.0,+77/-70,12.671,0.388,20.47,6.97,5119.36,231.986,✔️,0.0,False,GPU -13,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1424.0,+60/-53,15.951,0.369,20.56,10.58,330.95,2.023,➖,0.0,False,GPU -14,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1410.0,+84/-72,15.11,0.421,21.42,8.85,4.06,0.436,✔️,0.0,False,GPU -15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1400.0,+56/-54,14.802,0.297,22.04,11.42,3169.9,1.446,✔️,0.0,False,GPU -16,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1400.0,+82/-72,13.572,0.355,22.07,7.06,5119.36,26.967,✔️,0.0,False,GPU -17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1399.0,+72/-51,15.476,0.29,22.12,11.71,2828.45,0.216,✔️,0.0,False,GPU -18,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1388.0,+42/-35,16.363,0.23,22.83,15.77,647.56,1.72,✔️,0.0,False,CPU -19,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1362.0,+37/-30,16.741,0.175,24.46,17.6,647.56,0.284,✔️,0.0,False,CPU -20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1356.0,+50/-36,16.602,0.186,24.9,17.76,1465.86,0.692,✔️,0.0,False,CPU -21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1355.0,+56/-50,15.889,0.195,24.97,11.03,5944.88,0.516,✔️,0.0,False,GPU -22,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1342.0,+48/-35,16.823,0.173,25.77,18.98,1465.86,0.091,✔️,0.0,False,CPU -23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1326.0,+64/-56,18.007,0.23,26.84,14.3,10.42,0.155,✔️,0.0,False,GPU -24,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1325.0,+46/-41,17.617,0.157,26.9,19.33,766.06,1.917,✔️,0.0,False,CPU -25,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1323.0,+95/-83,17.015,0.242,27.06,11.78,5944.88,8.396,✔️,0.0,False,GPU -26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1320.0,+45/-44,17.726,0.159,27.23,18.32,5.72,0.11,✔️,0.0,False,CPU -27,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1320.0,+87/-95,16.755,0.3,27.25,8.62,50.32,43.824,✔️,0.0,False,GPU -28,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1320.0,+50/-53,17.396,0.172,27.25,16.63,934.1,2.767,✔️,0.0,False,GPU -29,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1309.0,+39/-36,17.841,0.119,27.97,19.24,766.06,0.278,✔️,0.0,False,CPU -30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1297.0,+46/-41,18.633,0.117,28.77,18.8,2686.11,0.47,✔️,0.0,False,CPU -31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1294.0,+56/-54,17.813,0.135,28.96,20.03,2862.05,2.158,✔️,0.0,False,CPU -32,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1283.0,+45/-58,19.49,0.14,29.68,15.39,14.8,0.337,✔️,0.0,False,GPU -33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1277.0,+50/-50,18.738,0.12,30.07,16.88,934.1,0.165,✔️,0.0,False,GPU -34,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1275.0,+77/-87,19.42,0.174,30.22,15.28,1358.63,8.067,✔️,0.0,False,CPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1269.0,+45/-48,19.24,0.083,30.63,18.46,2686.11,0.056,✔️,0.0,False,CPU -36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1244.0,+50/-47,20.124,0.089,32.25,19.07,7.4,0.057,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1243.0,+44/-39,19.291,0.068,32.34,20.31,13.83,3.953,✔️,0.0,False,GPU -38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1219.0,+64/-78,20.796,0.076,33.88,19.08,1358.63,0.897,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1213.0,+53/-43,20.109,0.061,34.25,22.19,2862.05,0.152,✔️,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1199.0,+60/-67,22.25,0.076,35.15,24.0,370.85,1.466,✔️,0.0,False,CPU -41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1184.0,+45/-43,21.0,0.019,36.07,30.03,2.4,0.218,✔️,0.0,False,CPU -42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1160.0,+62/-70,23.266,0.046,37.55,25.98,370.85,0.161,✔️,0.0,False,CPU -43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1153.0,+36/-44,21.599,0.027,37.95,32.6,2.9,0.133,✔️,0.0,False,CPU -44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1133.0,+70/-69,23.712,0.043,39.17,25.14,527.42,1.39,✔️,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1093.0,+61/-65,24.551,0.011,41.45,24.97,527.42,0.123,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1081.0,+56/-63,24.892,0.014,42.1,30.05,10.38,0.188,✔️,0.0,False,CPU -47,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1071.0,+83/-130,27.865,0.088,42.63,24.72,2.43,0.495,➖,0.0,False,GPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1058.0,+100/-110,30.113,0.054,43.29,28.54,436.81,0.874,✔️,0.0,False,CPU -49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1034.0,+89/-103,27.438,0.024,44.5,35.91,4.4,0.974,✔️,0.0,False,GPU -50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1033.0,+101/-105,30.656,0.036,44.53,30.27,436.81,0.106,✔️,0.0,False,CPU -51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1032.0,+72/-81,26.107,0.0,44.59,27.79,4.73,0.623,✔️,0.0,False,CPU -52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1008.0,+77/-109,30.107,0.015,45.72,32.67,113.26,1.55,✔️,0.0,False,CPU -53,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+61/-83,29.212,0.003,46.1,38.93,0.45,0.066,✔️,0.0,False,CPU -54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),969.0,+75/-112,30.601,0.009,47.45,40.18,0.4,0.07,✔️,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),958.0,+91/-127,33.436,0.023,47.9,33.34,2.16,0.139,✔️,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),909.0,+87/-143,36.035,0.001,49.71,34.2,113.26,0.12,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),600.0,+121/-261,49.89,0.0,56.33,51.43,0.26,0.023,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1630.0,+84/-80,0.722,10.0,3.9,6.61,2168.17,7.961,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1628.0,+70/-56,0.738,10.07,5.21,7.958,5.99,0.633,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1620.0,+83/-64,0.696,10.38,4.74,9.368,604.44,8.082,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1610.0,+80/-71,0.697,10.81,4.41,7.752,2168.17,1.23,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1540.0,+70/-55,0.57,14.09,5.86,11.475,3445.6,48.236,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),1533.0,+79/-59,0.59,14.47,6.28,12.049,146.65,4.101,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1516.0,+88/-72,0.568,15.35,5.73,11.77,4.37,0.585,➖,0.0,False,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1482.0,+62/-44,0.437,17.25,10.01,13.427,3169.9,16.793,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1456.0,+72/-61,0.442,18.82,6.68,13.063,8.89,1.743,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1450.0,+70/-46,0.366,19.14,10.17,14.712,2828.45,1.596,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1449.0,+81/-72,0.436,19.25,6.6,14.177,3445.6,0.995,✔️,0.0,False,GPU +11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1440.0,+63/-54,0.357,19.77,10.75,14.274,2898.23,1.884,✔️,0.0,False,CPU +12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1422.0,+77/-69,0.38,20.9,7.02,12.892,5119.36,231.986,✔️,0.0,False,GPU +13,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1421.0,+60/-51,0.365,20.98,10.75,16.153,330.95,2.023,➖,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1407.0,+83/-71,0.408,21.87,9.13,15.457,4.06,0.436,✔️,0.0,False,GPU +15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1397.0,+55/-54,0.293,22.5,11.81,15.095,3169.9,1.446,✔️,0.0,False,GPU +16,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1397.0,+83/-72,0.355,22.53,7.11,13.762,5119.36,26.967,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1396.0,+71/-50,0.286,22.61,11.93,15.679,2828.45,0.216,✔️,0.0,False,GPU +18,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1385.0,+41/-34,0.23,23.31,16.06,16.533,647.56,1.72,✔️,0.0,False,CPU +19,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1360.0,+36/-29,0.175,24.97,17.95,16.914,647.56,0.284,✔️,0.0,False,CPU +20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1354.0,+49/-35,0.183,25.37,18.19,16.88,1465.86,0.692,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1352.0,+55/-50,0.195,25.47,11.28,16.083,5944.88,0.516,✔️,0.0,False,GPU +22,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1349.0,+77/-82,0.339,25.69,8.08,17.238,12.28,1.411,➖,0.0,False,GPU +23,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1340.0,+46/-34,0.17,26.28,19.43,17.1,1465.86,0.091,✔️,0.0,False,CPU +24,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1323.0,+64/-56,0.225,27.4,15.15,18.17,10.42,0.155,✔️,0.0,False,GPU +25,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1322.0,+44/-41,0.156,27.46,19.79,17.817,766.06,1.917,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1321.0,+95/-83,0.243,27.58,11.91,17.226,5944.88,8.396,✔️,0.0,False,GPU +27,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1318.0,+47/-42,0.157,27.75,18.7,17.881,5.72,0.11,✔️,0.0,False,CPU +28,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1318.0,+87/-95,0.301,27.79,8.7,16.871,50.32,43.824,✔️,0.0,False,GPU +29,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1317.0,+50/-52,0.169,27.84,17.12,17.652,934.1,2.767,✔️,0.0,False,GPU +30,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1306.0,+39/-35,0.117,28.55,19.61,18.047,766.06,0.278,✔️,0.0,False,CPU +31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1295.0,+46/-40,0.114,29.34,19.33,18.806,2686.11,0.47,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1291.0,+57/-54,0.137,29.6,20.38,17.96,2862.05,2.158,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1281.0,+46/-57,0.138,30.25,15.68,19.639,14.8,0.337,✔️,0.0,False,GPU +34,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1275.0,+50/-49,0.119,30.68,17.18,18.979,934.1,0.165,✔️,0.0,False,GPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1272.0,+77/-86,0.173,30.88,15.62,19.54,1358.63,8.067,✔️,0.0,False,CPU +36,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1267.0,+44/-46,0.08,31.22,18.79,19.416,2686.11,0.056,✔️,0.0,False,CPU +37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1242.0,+49/-45,0.083,32.89,19.61,20.271,7.4,0.057,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1240.0,+44/-39,0.068,33.04,20.96,19.431,13.83,3.953,✔️,0.0,False,GPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1216.0,+64/-77,0.075,34.6,19.66,20.91,1358.63,0.897,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1211.0,+52/-43,0.063,34.95,22.57,20.228,2862.05,0.152,✔️,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1197.0,+60/-66,0.073,35.83,24.35,22.47,370.85,1.466,✔️,0.0,False,CPU +42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1183.0,+45/-43,0.022,36.73,30.5,21.153,2.4,0.218,✔️,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1159.0,+62/-69,0.043,38.24,26.46,23.47,370.85,0.161,✔️,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1152.0,+36/-44,0.027,38.64,33.15,21.722,2.9,0.133,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1132.0,+70/-68,0.041,39.91,25.86,23.974,527.42,1.39,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1092.0,+60/-65,0.01,42.2,27.78,24.812,527.42,0.123,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1079.0,+55/-62,0.016,42.9,30.59,25.033,10.38,0.188,✔️,0.0,False,CPU +48,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1070.0,+82/-130,0.089,43.44,25.27,27.954,2.43,0.495,➖,0.0,False,GPU +49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1057.0,+99/-105,0.054,44.12,28.99,30.261,436.81,0.874,✔️,0.0,False,CPU +50,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1033.0,+90/-101,0.02,45.32,36.7,27.651,4.4,0.974,✔️,0.0,False,GPU +51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1032.0,+100/-102,0.037,45.36,30.81,30.773,436.81,0.106,✔️,0.0,False,CPU +52,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1032.0,+72/-82,0.0,45.4,28.7,26.194,4.73,0.623,✔️,0.0,False,CPU +53,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1009.0,+76/-105,0.016,46.49,33.08,30.21,113.26,1.55,✔️,0.0,False,CPU +54,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+61/-84,0.003,46.9,40.2,29.346,0.45,0.066,✔️,0.0,False,CPU +55,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),970.0,+75/-110,0.009,48.23,40.88,30.707,0.4,0.07,✔️,0.0,False,CPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),958.0,+90/-124,0.023,48.74,33.8,33.52,2.16,0.139,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),910.0,+86/-140,0.001,50.55,34.68,36.12,113.26,0.12,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),599.0,+121/-261,0.0,57.31,52.26,49.949,0.26,0.023,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip index 9c86d79e7227d9f64f40a6c1cbf6f5b69bdb8da4..f6d584f86051b065df5e92dddc4bbf9fa70c797a 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d76acd884961fc3c50d365e6564270d36b5266513ae03863ce4503e941ea2c89 -size 2801926 +oid sha256:52b39ceac2a644e798e2c55434f3c950aa0782e23a056afdeb88642060e3afa2 +size 2820614 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index 81df7d8182fad1a644649ba7073479bbe7897d97..991e388a41d92d01d1f61ba2ba8f9549ab874c5c 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:528e3be3e90c0609f350f4d34275e13e6edacb5d78a5da5d8b3fa60747443560 -size 419969 +oid sha256:4e5a090cd62c3b18cfa4ca76d572239f90d87855e0503b4dba37dc000e92fcca +size 433613 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip index 50d621f5bf87a8055167469f64b4635aba567364..e38d08581efb971e260f00e35c558a17a73955ff 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:136db68f3f27fdaf8d2ef5249f6063cfe5dd18fa3f8ee9aff193a091b5c2dd9b -size 924416 +oid sha256:f6471fa355fcc69744795389e442e4e83a616116d891f5f177b39398a4161b49 +size 913796 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip index 2558ec6f2f3306a0e16a4868818c311bc63353f2..3a0f04c22fb6f7468aac729519429f59a64528e7 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:207a1db3d79883293969ace68570ac4fddc4d2da1db09628feb0d83e8926c865 -size 117390 +oid sha256:fdee073b43b22a80b3ac4c7db9e5ff3eedc2aa91111451de787e700c086f2d03 +size 120127 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/website_leaderboard.csv index 06a285f296ecc9a24b2ca1a8abf29994fc92c071..4736fa8070ebbebfa4d23f08c78d0e861ebcd569 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/website_leaderboard.csv @@ -1,53 +1,54 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1760.0,+178/-83,9.062,0.832,5.81,3.5,1045.67,7.086,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1734.0,+245/-102,6.353,0.819,6.48,3.01,2710.27,1.347,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1725.0,+327/-154,5.551,0.812,6.69,2.17,2710.27,8.099,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1588.0,+259/-161,8.494,0.668,11.28,4.18,10.23,0.844,✔️,0.0,False,GPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1534.0,+219/-124,14.217,0.45,13.62,5.1,6219.24,3.78,✔️,0.0,False,CPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1519.0,+190/-118,12.711,0.51,14.32,6.59,4150.25,19.131,✔️,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1496.0,+137/-88,13.842,0.441,15.41,8.18,4150.25,3.423,✔️,0.0,False,GPU -7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1478.0,+125/-87,16.609,0.398,16.31,9.73,5885.87,1.632,✔️,0.0,False,GPU -8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1465.0,+122/-75,16.251,0.344,17.0,12.25,1377.29,5.202,✔️,0.0,False,CPU -9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1458.0,+101/-59,14.67,0.335,17.34,13.96,3104.39,1.095,✔️,0.0,False,CPU -10,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1456.0,+100/-70,16.47,0.322,17.47,13.14,1377.29,1.259,✔️,0.0,False,CPU -11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1438.0,+123/-87,17.463,0.349,18.46,9.94,5885.87,0.299,✔️,0.0,False,GPU -12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1435.0,+170/-123,18.358,0.397,18.57,7.39,4635.14,212.285,✔️,0.0,False,GPU -13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1433.0,+102/-67,15.297,0.278,18.67,14.85,3104.39,0.212,✔️,0.0,False,CPU -14,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1410.0,+103/-98,16.733,0.276,19.91,15.48,2210.02,0.974,✔️,0.0,False,CPU -15,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1404.0,+101/-101,16.868,0.269,20.24,15.39,2210.02,3.985,✔️,0.0,False,CPU -16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1403.0,+113/-97,15.873,0.239,20.3,13.01,7127.77,9.214,✔️,0.0,False,GPU -17,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1396.0,+76/-45,16.245,0.201,20.68,10.91,7127.77,0.776,✔️,0.0,False,GPU -18,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1382.0,+179/-128,20.214,0.333,21.41,8.7,4635.14,34.77,✔️,0.0,False,GPU -19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1376.0,+153/-120,19.283,0.284,21.74,14.05,1980.6,3.605,✔️,0.0,False,GPU -20,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1373.0,+139/-108,19.839,0.293,21.93,15.97,507.73,4.428,➖,0.0,False,GPU -21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1361.0,+115/-90,18.672,0.197,22.61,16.5,36484.23,0.827,✔️,0.0,False,CPU -22,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1339.0,+128/-120,20.568,0.238,23.84,15.04,17.83,0.197,✔️,0.0,False,GPU -23,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1327.0,+185/-126,21.441,0.245,24.53,12.84,717.36,1.612,✔️,0.0,False,CPU -24,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1326.0,+185/-166,20.662,0.293,24.56,10.84,737.91,1.864,✔️,0.0,False,CPU -25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1322.0,+77/-52,18.814,0.144,24.83,21.0,50.54,0.227,✔️,0.0,False,CPU -26,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1315.0,+106/-103,19.435,0.146,25.21,18.54,36484.23,0.127,✔️,0.0,False,CPU -27,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1314.0,+158/-113,20.348,0.136,25.25,18.72,3901.28,2.028,✔️,0.0,False,CPU -28,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1282.0,+125/-114,21.368,0.145,27.06,21.54,19.31,0.108,✔️,0.0,False,CPU -29,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1278.0,+182/-138,22.94,0.203,27.3,11.99,717.36,0.326,✔️,0.0,False,CPU -30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1276.0,+123/-114,21.665,0.163,27.43,15.98,1980.6,0.61,✔️,0.0,False,GPU -31,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1273.0,+194/-174,24.011,0.234,27.57,11.87,2644.99,12.294,✔️,0.0,False,CPU -32,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1273.0,+198/-187,22.052,0.227,27.57,9.69,737.91,0.316,✔️,0.0,False,CPU -33,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1265.0,+210/-138,23.278,0.221,28.02,8.16,43.39,42.575,✔️,0.0,False,GPU -34,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1247.0,+118/-131,22.692,0.098,28.99,20.71,3901.28,0.223,✔️,0.0,False,CPU -35,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1246.0,+115/-89,20.308,0.063,29.06,25.43,4.72,0.542,✔️,0.0,False,CPU -36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1232.0,+137/-118,22.446,0.133,29.78,23.49,5.04,0.429,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1221.0,+151/-140,25.129,0.122,30.37,16.99,2644.99,1.148,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1202.0,+94/-115,25.131,0.032,31.38,25.09,17.94,0.691,✔️,0.0,False,GPU -39,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1159.0,+68/-94,25.754,0.0,33.59,30.64,19.2,7.624,✔️,0.0,False,GPU -40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1089.0,+135/-197,29.146,0.054,36.92,30.62,9.42,1.675,✔️,0.0,False,GPU -41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1055.0,+75/-117,28.55,0.0,38.37,30.41,13.69,0.314,✔️,0.0,False,CPU -42,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1006.0,+229/-348,39.232,0.031,40.34,22.09,151.69,2.346,✔️,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+118/-185,35.45,0.0,40.55,36.98,0.74,0.137,✔️,0.0,False,CPU -44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),997.0,+100/-244,39.878,0.0,40.66,38.52,892.95,0.818,✔️,0.0,False,CPU -45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),959.0,+92/-261,40.883,0.0,42.01,39.7,892.95,0.146,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),949.0,+131/-194,34.505,0.003,42.33,39.94,8.51,0.734,✔️,0.0,False,CPU -47,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),906.0,+139/-185,40.594,0.0,43.69,40.86,0.69,0.137,✔️,0.0,False,CPU -48,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),844.0,+66/-133,39.593,0.0,45.37,45.16,5.05,1.013,➖,0.0,False,GPU -49,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),824.0,+265/-559,49.561,0.016,45.88,36.33,151.69,0.258,✔️,0.0,False,CPU -50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),746.0,+93/-250,47.08,0.0,47.53,47.17,4.53,0.164,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),198.0,+134/-662,70.493,0.0,51.77,51.76,0.52,0.076,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1745.0,+171/-74,0.826,6.07,3.71,10.043,1045.67,7.086,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1722.0,+226/-98,0.813,6.67,3.14,7.633,2710.27,1.347,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1714.0,+304/-148,0.808,6.89,2.26,6.828,2710.27,8.099,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1580.0,+241/-156,0.654,11.51,4.67,9.867,10.23,0.844,✔️,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1524.0,+205/-119,0.443,14.04,5.41,14.813,6219.24,3.78,✔️,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1509.0,+172/-113,0.499,14.74,7.18,13.788,4150.25,19.131,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1486.0,+133/-88,0.43,15.89,8.9,14.808,4150.25,3.423,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1471.0,+120/-83,0.398,16.69,10.11,17.321,5885.87,1.632,✔️,0.0,False,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1458.0,+124/-74,0.337,17.38,12.63,16.791,1377.29,5.202,✔️,0.0,False,CPU +9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1450.0,+99/-54,0.315,17.78,14.6,15.594,3104.39,1.095,✔️,0.0,False,CPU +10,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1448.0,+101/-70,0.313,17.87,13.58,17.021,1377.29,1.259,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1430.0,+120/-85,0.348,18.89,10.22,18.117,5885.87,0.299,✔️,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1427.0,+168/-122,0.382,19.06,7.54,19.043,4635.14,212.285,✔️,0.0,False,GPU +13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1425.0,+93/-67,0.256,19.14,15.5,16.211,3104.39,0.212,✔️,0.0,False,CPU +14,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1404.0,+101/-93,0.266,20.32,15.96,17.394,2210.02,0.974,✔️,0.0,False,CPU +15,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1398.0,+98/-96,0.26,20.65,15.81,17.505,2210.02,3.985,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1396.0,+105/-94,0.227,20.74,13.39,16.604,7127.77,9.214,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1390.0,+79/-46,0.194,21.12,11.08,16.901,7127.77,0.776,✔️,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1375.0,+182/-126,0.333,21.93,8.84,20.768,4635.14,34.77,✔️,0.0,False,GPU +19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1368.0,+149/-122,0.263,22.32,14.48,20.11,1980.6,3.605,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1365.0,+135/-108,0.287,22.52,16.55,20.477,507.73,4.428,➖,0.0,False,GPU +21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1354.0,+107/-87,0.19,23.11,16.94,19.217,36484.23,0.827,✔️,0.0,False,CPU +22,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1342.0,+203/-281,0.403,23.81,5.22,17.651,22.52,1.682,➖,0.0,False,GPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1331.0,+125/-119,0.246,24.46,15.66,21.075,17.83,0.197,✔️,0.0,False,GPU +24,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1322.0,+180/-126,0.243,25.02,12.97,22.136,717.36,1.612,✔️,0.0,False,CPU +25,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1321.0,+188/-165,0.288,25.06,11.1,21.486,737.91,1.864,✔️,0.0,False,CPU +26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1316.0,+76/-54,0.136,25.35,21.99,19.294,50.54,0.227,✔️,0.0,False,CPU +27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1310.0,+102/-101,0.139,25.73,18.9,20.0,36484.23,0.127,✔️,0.0,False,CPU +28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1308.0,+148/-110,0.12,25.83,19.12,20.799,3901.28,2.028,✔️,0.0,False,CPU +29,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1277.0,+124/-113,0.147,27.62,21.87,21.825,19.31,0.108,✔️,0.0,False,CPU +30,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1274.0,+177/-141,0.2,27.8,12.1,23.572,717.36,0.326,✔️,0.0,False,CPU +31,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1269.0,+195/-186,0.223,28.08,10.03,22.868,737.91,0.316,✔️,0.0,False,CPU +32,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1269.0,+126/-111,0.16,28.08,16.42,22.43,1980.6,0.61,✔️,0.0,False,GPU +33,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1267.0,+189/-176,0.222,28.2,12.27,24.365,2644.99,12.294,✔️,0.0,False,CPU +34,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1260.0,+209/-138,0.221,28.58,8.19,23.595,43.39,42.575,✔️,0.0,False,GPU +35,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1243.0,+111/-84,0.063,29.56,25.81,20.791,4.72,0.542,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1242.0,+117/-132,0.108,29.59,21.0,23.04,3901.28,0.223,✔️,0.0,False,CPU +37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1227.0,+137/-116,0.135,30.42,23.93,22.819,5.04,0.429,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1215.0,+141/-142,0.121,31.09,17.39,25.46,2644.99,1.148,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1198.0,+94/-116,0.033,32.0,25.47,25.586,17.94,0.691,✔️,0.0,False,GPU +40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1154.0,+68/-96,0.0,34.39,31.5,26.173,19.2,7.624,✔️,0.0,False,GPU +41,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1088.0,+129/-183,0.047,37.58,31.7,29.814,9.42,1.675,✔️,0.0,False,GPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1054.0,+73/-110,0.0,39.11,31.29,28.977,13.69,0.314,✔️,0.0,False,CPU +43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1008.0,+226/-342,0.041,40.98,22.24,39.509,151.69,2.346,✔️,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+116/-180,0.0,41.3,37.66,35.825,0.74,0.137,✔️,0.0,False,CPU +45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),994.0,+98/-236,0.0,41.54,39.5,40.319,892.95,0.818,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),956.0,+90/-238,0.0,42.89,40.7,41.22,892.95,0.146,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),948.0,+127/-169,0.0,43.19,40.9,34.749,8.51,0.734,✔️,0.0,False,CPU +48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),908.0,+138/-182,0.0,44.47,41.49,40.871,0.69,0.137,✔️,0.0,False,CPU +49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),849.0,+67/-136,0.0,46.14,45.9,39.802,5.05,1.013,➖,0.0,False,GPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),829.0,+259/-495,0.026,46.65,36.75,49.77,151.69,0.258,✔️,0.0,False,CPU +51,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),749.0,+83/-218,0.0,48.42,48.07,47.305,4.53,0.164,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),198.0,+129/-639,0.0,52.77,52.76,70.62,0.52,0.076,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/winrate_matrix.png.zip index 0e92baa4364090afe172dadf9dbb69737eeb9b83..24ee65206bfece8bbcb66a214cbf911090af7595 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6c16c319c7936bbd481690c97e48d8d4a09c18dc3b09f8e304fde2b85e9689f7 -size 1988199 +oid sha256:d4857c2842400be41941bfe2429e659d199d6a8a46ea23baffe119ce73be12a6 +size 2143437 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index e0e57b9fb3dd95172c81d50e1a62f56ad3702fcc..8d659f97536eed4c57447db4a8949c2fb90e410d 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:23051d990f26b358d1b0774145ea8528299d7870d2bb6b5b6aeb021b3c51fa2e -size 444887 +oid sha256:0839d18de82b315495651d898712bbedc219007c148c2082e8df2523dc478faf +size 459298 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip index 1c26b24a6cffb5b1ba6e3130b498b0c3504e1386..83c845a1ad46d9e33656c816a2fa4226b6f8034b 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fc6db28c154ad18bed6811d344d3bf9c6ac2b5b2628235e0b92cecdfb4e40aed -size 951729 +oid sha256:58a53c7caabf7a758b5994534d0f8fd72570d000bbc1c41bc10a6b0c44a63469 +size 981338 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip index ea04436cc646b011eb2e76e69a3c66af1df8a60f..1fce8cc435aee486a66f90db119d44ee6bb0965e 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cee6ff6f9ea7e012f32526d1c3e1e2a27fd4068b6f9675f2a442b7a4c234cd33 -size 106892 +oid sha256:d74d9bbfee3b5c81053199cc027d6534bbaf6405951be99694c25f1d3c9506c2 +size 111706 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/website_leaderboard.csv index a419ca2395ccc34bf4d1abe9827fdbf5cbfafd8c..b7eaf658849154ebe3eb9e60d6f5b2f7a7c7ffa4 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/website_leaderboard.csv @@ -1,54 +1,55 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),9270.0,+0/-0,0.019,1.0,1.11,1.06,133.98,8.253,✔️,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",8803.0,+0/-0,1.7,0.934,2.56,2.2,276.87,4.662,✔️,0.0,False,CPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",8700.0,+0/-0,1.878,0.927,3.0,2.84,276.86,3.44,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),8621.0,+0/-0,2.424,0.905,3.33,3.0,133.98,2.816,✔️,0.0,False,GPU -4,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),6195.0,+0/-0,9.581,0.594,6.78,6.31,61.95,0.17,✔️,0.0,False,CPU -5,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),6182.0,+0/-0,9.545,0.595,6.89,6.45,7606.4,0.826,✔️,0.0,False,CPU -6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),6134.0,+0/-0,10.011,0.573,7.33,7.13,157.72,3.649,✔️,0.0,False,CPU -7,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),6038.0,+0/-0,9.898,0.578,8.22,7.72,7606.4,0.048,✔️,0.0,False,CPU -8,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),5997.0,+0/-0,9.921,0.578,8.67,7.91,61.95,0.016,✔️,0.0,False,CPU -9,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),5959.0,+0/-0,10.441,0.552,9.11,8.66,157.72,0.374,✔️,0.0,False,CPU -10,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),5682.0,+0/-0,11.347,0.508,11.89,10.98,2688.02,0.24,✔️,0.0,False,CPU -11,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),5671.0,+0/-0,11.168,0.518,12.0,11.9,466.47,1.036,✔️,0.0,False,CPU -12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),5573.0,+0/-0,11.536,0.499,13.0,12.51,2688.02,0.726,✔️,0.0,False,CPU -13,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),5519.0,+0/-0,11.505,0.501,13.56,13.39,0.39,0.051,✔️,0.0,False,CPU -14,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),5486.0,+0/-0,11.505,0.501,13.89,13.77,466.47,0.051,✔️,0.0,False,CPU -15,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),5145.0,+0/-0,13.544,0.399,17.0,16.0,1.48,0.53,✔️,0.0,False,GPU -16,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),5108.0,+0/-0,13.063,0.423,17.33,17.24,7.54,0.047,✔️,0.0,False,CPU -17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),5082.0,+0/-0,13.396,0.407,17.56,17.52,4423.24,29.2,✔️,0.0,False,GPU -18,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),5032.0,+0/-0,13.739,0.389,18.0,17.84,17.7,0.086,✔️,0.0,False,CPU -19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),4812.0,+0/-0,14.412,0.354,19.67,19.62,4423.24,2.237,✔️,0.0,False,GPU -20,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),4739.0,+0/-0,14.523,0.348,20.11,20.02,2.82,1.073,✔️,0.0,False,GPU -21,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),3202.0,+0/-0,16.353,0.25,22.11,22.11,20.98,0.329,✔️,0.0,False,CPU -22,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),2532.0,+0/-0,18.457,0.132,24.22,24.17,4638.91,343.215,✔️,0.0,False,GPU -23,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),2475.0,+0/-0,18.732,0.115,24.56,24.47,2389.31,0.267,✔️,0.0,False,CPU -24,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),2475.0,+0/-0,18.777,0.113,24.56,24.51,20.98,0.027,✔️,0.0,False,CPU -25,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),2456.0,+0/-0,18.841,0.109,24.67,24.61,2389.31,0.023,✔️,0.0,False,CPU -26,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1738.0,+0/-0,20.674,0.0,27.44,27.43,14.87,1.855,✔️,0.0,False,GPU -27,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1554.0,+0/-0,20.901,0.0,28.33,28.3,1993.14,0.452,✔️,0.0,False,GPU -28,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1404.0,+0/-0,21.65,0.0,29.22,29.16,1993.14,0.059,✔️,0.0,False,GPU -29,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1218.0,+0/-0,22.097,0.0,30.44,30.4,4638.91,43.781,✔️,0.0,False,GPU -30,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1218.0,+0/-0,22.475,0.0,30.44,30.35,40.54,38.741,✔️,0.0,False,GPU -31,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+0/-0,23.341,0.0,31.89,31.77,0.18,0.008,✔️,0.0,False,CPU -32,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),821.0,+0/-0,23.568,0.0,33.0,32.98,1294.93,0.417,✔️,0.0,False,GPU -33,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),665.0,+0/-0,24.01,0.0,33.89,33.86,1294.93,0.06,✔️,0.0,False,GPU -34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),599.0,+0/-0,24.919,0.0,34.22,34.18,0.18,0.04,✔️,0.0,False,CPU -35,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),-215.0,+0/-0,28.274,0.0,36.11,36.11,1863.58,6.302,✔️,0.0,False,GPU -36,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),-734.0,+0/-0,30.79,0.0,37.56,37.54,1863.58,1.131,✔️,0.0,False,GPU -37,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),-788.0,+0/-0,30.768,0.0,37.78,37.75,3.96,0.025,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),-1010.0,+0/-0,32.321,0.0,38.67,38.66,6.85,0.088,✔️,0.0,False,GPU -39,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),-1462.0,+0/-0,33.496,0.0,39.89,39.89,66.06,1.792,➖,0.0,False,GPU -40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),-2063.0,+0/-0,37.017,0.0,41.0,41.0,7.55,0.149,✔️,0.0,False,GPU -41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),-2631.0,+0/-0,41.044,0.0,42.11,42.11,569.0,0.17,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),-2955.0,+0/-0,41.644,0.0,42.89,42.89,569.0,1.658,✔️,0.0,False,CPU -43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),-3626.0,+0/-0,50.207,0.0,44.11,44.11,0.07,0.015,✔️,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),-4076.0,+0/-0,51.289,0.0,45.22,45.21,2.86,0.085,✔️,0.0,False,CPU -45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),-4255.0,+0/-0,51.687,0.0,45.78,45.77,30.41,0.089,✔️,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),-4823.0,+0/-0,54.976,0.0,47.44,47.44,30.41,0.015,✔️,0.0,False,CPU -47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),-5018.0,+0/-0,55.365,0.0,48.33,48.32,0.13,0.014,✔️,0.0,False,CPU -48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),-5106.0,+0/-0,55.242,0.0,48.78,48.72,3.09,0.997,✔️,0.0,False,GPU -49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),-5308.0,+0/-0,56.49,0.0,49.78,49.77,0.11,0.023,➖,0.0,False,GPU -50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),-5500.0,+0/-0,57.332,0.0,50.56,50.54,76.27,3.054,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),-6498.0,+0/-0,70.796,0.0,52.0,52.0,76.27,0.37,✔️,0.0,False,CPU -52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-7418.0,+0/-0,82.671,0.0,53.0,53.0,0.14,0.136,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),9356.0,+0/-0,1.0,1.11,1.06,0.019,133.98,8.253,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",8889.0,+0/-0,0.935,2.56,2.2,1.7,276.87,4.662,✔️,0.0,False,CPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",8788.0,+0/-0,0.928,3.0,2.84,1.878,276.86,3.44,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),8709.0,+0/-0,0.906,3.33,3.0,2.424,133.98,2.816,✔️,0.0,False,GPU +4,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),6299.0,+0/-0,0.597,6.78,6.31,9.581,61.95,0.17,✔️,0.0,False,CPU +5,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),6288.0,+0/-0,0.598,6.89,6.45,9.545,7606.4,0.826,✔️,0.0,False,CPU +6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),6240.0,+0/-0,0.576,7.33,7.13,10.011,157.72,3.649,✔️,0.0,False,CPU +7,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),6149.0,+0/-0,0.581,8.22,7.72,9.898,7606.4,0.048,✔️,0.0,False,CPU +8,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),6106.0,+0/-0,0.581,8.67,7.91,9.921,61.95,0.016,✔️,0.0,False,CPU +9,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),6064.0,+0/-0,0.555,9.11,8.66,10.441,157.72,0.374,✔️,0.0,False,CPU +10,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),5788.0,+0/-0,0.512,11.89,10.98,11.347,2688.02,0.24,✔️,0.0,False,CPU +11,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),5778.0,+0/-0,0.521,12.0,11.9,11.168,466.47,1.036,✔️,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),5680.0,+0/-0,0.503,13.0,12.51,11.536,2688.02,0.726,✔️,0.0,False,CPU +13,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),5624.0,+0/-0,0.505,13.56,13.39,11.505,0.39,0.051,✔️,0.0,False,CPU +14,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),5589.0,+0/-0,0.505,13.89,13.77,11.505,466.47,0.051,✔️,0.0,False,CPU +15,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),5256.0,+0/-0,0.403,17.0,16.0,13.544,1.48,0.53,✔️,0.0,False,GPU +16,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),5218.0,+0/-0,0.427,17.33,17.24,13.063,7.54,0.047,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),5193.0,+0/-0,0.411,17.56,17.52,13.396,4423.24,29.2,✔️,0.0,False,GPU +18,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),5142.0,+0/-0,0.393,18.0,17.84,13.739,17.7,0.086,✔️,0.0,False,CPU +19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),4927.0,+0/-0,0.359,19.67,19.62,14.412,4423.24,2.237,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),4852.0,+0/-0,0.353,20.11,20.02,14.523,2.82,1.073,✔️,0.0,False,GPU +21,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),3270.0,+0/-0,0.255,22.11,22.11,16.353,20.98,0.329,✔️,0.0,False,CPU +22,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),2605.0,+0/-0,0.138,24.22,24.17,18.457,4638.91,343.215,✔️,0.0,False,GPU +23,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),2547.0,+0/-0,0.119,24.56,24.51,18.777,20.98,0.027,✔️,0.0,False,CPU +24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),2547.0,+0/-0,0.121,24.56,24.47,18.732,2389.31,0.267,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),2528.0,+0/-0,0.115,24.67,24.61,18.841,2389.31,0.023,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1766.0,+0/-0,0.007,27.44,27.43,20.674,14.87,1.855,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1584.0,+0/-0,0.0,28.33,28.3,20.901,1993.14,0.452,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1400.0,+0/-0,0.0,29.44,29.37,21.65,1993.14,0.059,✔️,0.0,False,GPU +29,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1237.0,+0/-0,0.0,30.67,30.6,22.097,4638.91,43.781,✔️,0.0,False,GPU +30,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1195.0,+0/-0,0.0,31.0,30.87,22.475,40.54,38.741,✔️,0.0,False,GPU +31,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1126.0,+0/-0,0.0,31.56,31.48,22.424,9.87,9.237,➖,0.0,False,GPU +32,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+0/-0,0.0,32.56,32.38,23.341,0.18,0.008,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),828.0,+0/-0,0.0,33.78,33.75,23.568,1294.93,0.417,✔️,0.0,False,GPU +34,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),639.0,+0/-0,0.0,34.89,34.86,24.01,1294.93,0.06,✔️,0.0,False,GPU +35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),572.0,+0/-0,0.0,35.22,35.18,24.919,0.18,0.04,✔️,0.0,False,CPU +36,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),-247.0,+0/-0,0.0,37.11,37.11,28.274,1863.58,6.302,✔️,0.0,False,GPU +37,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),-746.0,+0/-0,0.0,38.56,38.54,30.79,1863.58,1.131,✔️,0.0,False,GPU +38,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),-800.0,+0/-0,0.0,38.78,38.75,30.768,3.96,0.025,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),-1014.0,+0/-0,0.0,39.67,39.66,32.321,6.85,0.088,✔️,0.0,False,GPU +40,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),-1449.0,+0/-0,0.0,40.89,40.89,33.496,66.06,1.792,➖,0.0,False,GPU +41,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),-2053.0,+0/-0,0.0,42.0,42.0,37.017,7.55,0.149,✔️,0.0,False,GPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),-2631.0,+0/-0,0.0,43.11,43.11,41.044,569.0,0.17,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),-2953.0,+0/-0,0.0,43.89,43.89,41.644,569.0,1.658,✔️,0.0,False,CPU +44,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),-3647.0,+0/-0,0.0,45.11,45.11,50.207,0.07,0.015,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),-4085.0,+0/-0,0.0,46.22,46.21,51.289,2.86,0.085,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),-4259.0,+0/-0,0.0,46.78,46.77,51.687,30.41,0.089,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),-4819.0,+0/-0,0.0,48.44,48.44,54.976,30.41,0.015,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),-5018.0,+0/-0,0.0,49.33,49.32,55.365,0.13,0.014,✔️,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),-5107.0,+0/-0,0.0,49.78,49.72,55.242,3.09,0.997,✔️,0.0,False,GPU +50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),-5312.0,+0/-0,0.0,50.78,50.77,56.49,0.11,0.023,➖,0.0,False,GPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),-5506.0,+0/-0,0.0,51.56,51.54,57.332,76.27,3.054,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),-6498.0,+0/-0,0.0,53.0,53.0,70.796,76.27,0.37,✔️,0.0,False,CPU +53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-7417.0,+0/-0,0.0,54.0,54.0,82.671,0.14,0.136,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/winrate_matrix.png.zip index 5f2e52f8a499c7a5bd2253453b4f8c57fcfd7478..bacf864e33622ad70d285c8da827ed361604b8a5 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b402a5925b593bd073ebb55be666125b164b9f9e37e9e3ee50468e6e22be98c1 -size 1124587 +oid sha256:99404da097948664c156941ba89bd8842fe2061855f5eee00ffa59735e819703 +size 1208223 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 88d24611b00895d840d4189e4331086b797fbfe9..9cdd295939b7d6e5bd91b990255bd41dbad558c9 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:54c9680cdaaa0a1a71e0332aca1b99376a8af55a0fea3df04a39f8b83f95f03e -size 429560 +oid sha256:7134041b91eba54921c5ca10c8df96a0ef7b27d0cb00b1000eb84735db4de64e +size 444213 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip index d96c884a60f7ff9e1c3e2e24900edb29426d3560..0284e830634458bc27cf11875d0d9d5ccf08e91f 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f409d541e29be69f132cc8808d06d2c0ad9f6e4183abe290a548ee1a24798dab -size 917138 +oid sha256:79ee48b45b2d38f16542a44d893f4c97eaba053af94a06064958bb602946b7fa +size 913833 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip index cbb9e2bd9afd3f5c1213c0a4c51db365c140c3c7..3aed2870b248448f208e179c617d9ac84d90aa39 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:77f2d62b3b704530de5c1e8c319ef204be0bf686d6bb745780f4727861494132 -size 116913 +oid sha256:5ef82651479480b417878205f36a5ad79e1701252a8c0bcd53f4932369315374 +size 122674 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/website_leaderboard.csv index 4e7a5ce23546742c2917640f5292677bb556db44..2bbeab06f7f92f9e2a2757df07aeae115b9aa34a 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/website_leaderboard.csv @@ -1,54 +1,55 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1780.0,+190/-78,10.304,0.821,6.49,3.79,1154.67,7.879,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1755.0,+239/-99,7.171,0.812,7.18,3.12,2742.37,1.218,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1736.0,+281/-137,6.592,0.793,7.75,2.66,2742.37,7.945,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1648.0,+320/-180,8.027,0.703,10.83,3.96,11.21,0.845,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1613.0,+210/-135,11.362,0.58,12.31,6.11,4471.59,21.354,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1607.0,+176/-118,10.368,0.554,12.54,4.81,6.22,1.092,➖,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1586.0,+139/-72,12.584,0.498,13.48,7.68,4471.59,3.76,✔️,0.0,False,GPU -7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1547.0,+151/-75,16.178,0.456,15.33,9.15,5929.46,1.73,✔️,0.0,False,GPU -8,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1538.0,+225/-90,16.193,0.37,15.8,7.22,6548.58,2.898,✔️,0.0,False,CPU -9,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1500.0,+152/-80,17.039,0.399,17.71,9.33,5929.46,0.337,✔️,0.0,False,GPU -10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1485.0,+193/-157,18.473,0.422,18.51,7.3,4631.38,190.647,✔️,0.0,False,GPU -11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1481.0,+104/-65,15.318,0.293,18.73,15.09,3272.91,1.185,✔️,0.0,False,CPU -12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1474.0,+153/-80,17.306,0.298,19.1,14.57,1502.45,6.33,✔️,0.0,False,CPU -13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1469.0,+142/-72,17.496,0.274,19.4,15.11,1502.45,1.367,✔️,0.0,False,CPU -14,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1454.0,+136/-100,18.046,0.328,20.23,15.27,684.78,4.816,➖,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1450.0,+99/-57,16.053,0.224,20.42,16.47,3272.91,0.21,✔️,0.0,False,CPU -16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1447.0,+112/-130,18.168,0.308,20.59,13.61,2097.62,3.142,✔️,0.0,False,GPU -17,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1441.0,+204/-163,20.039,0.384,20.91,8.16,4631.38,30.351,✔️,0.0,False,GPU -18,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1432.0,+139/-101,16.443,0.197,21.42,13.4,7359.32,9.026,✔️,0.0,False,GPU -19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1429.0,+124/-86,17.647,0.232,21.59,16.53,2551.65,1.021,✔️,0.0,False,CPU -20,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1429.0,+84/-52,16.711,0.167,21.61,10.61,7359.32,0.596,✔️,0.0,False,GPU -21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1418.0,+138/-86,17.847,0.22,22.21,16.65,2551.65,4.527,✔️,0.0,False,CPU -22,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1411.0,+177/-111,19.041,0.278,22.59,14.24,19.65,0.236,✔️,0.0,False,GPU -23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1358.0,+80/-78,20.132,0.132,25.65,21.96,63121.14,0.827,✔️,0.0,False,CPU -24,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1354.0,+196/-181,22.31,0.241,25.83,12.52,717.53,1.628,✔️,0.0,False,CPU -25,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1347.0,+181/-110,20.717,0.118,26.24,18.68,4142.25,2.105,✔️,0.0,False,CPU -26,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1346.0,+232/-170,21.631,0.252,26.32,11.2,2775.63,12.592,✔️,0.0,False,CPU -27,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1338.0,+83/-58,19.786,0.094,26.77,23.03,64.18,0.255,✔️,0.0,False,CPU -28,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1334.0,+100/-110,20.501,0.191,26.97,15.29,2097.62,0.515,✔️,0.0,False,GPU -29,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1318.0,+195/-204,22.379,0.249,27.92,12.51,755.05,2.155,✔️,0.0,False,CPU -30,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1308.0,+86/-80,20.964,0.076,28.47,23.93,63121.14,0.173,✔️,0.0,False,CPU -31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1308.0,+227/-212,23.486,0.252,28.5,7.67,46.25,45.74,✔️,0.0,False,GPU -32,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1304.0,+202/-208,23.665,0.213,28.68,11.66,717.53,0.339,✔️,0.0,False,CPU -33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1294.0,+144/-116,22.606,0.111,29.24,22.8,20.91,0.126,✔️,0.0,False,CPU -34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1287.0,+213/-154,22.991,0.136,29.67,16.06,2775.63,1.431,✔️,0.0,False,CPU -35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1278.0,+174/-162,22.23,0.154,30.14,23.14,5.32,0.431,✔️,0.0,False,CPU -36,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1270.0,+136/-128,23.361,0.105,30.56,20.74,4142.25,0.287,✔️,0.0,False,CPU -37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1260.0,+190/-218,23.912,0.175,31.12,10.31,755.05,0.317,✔️,0.0,False,CPU -38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1239.0,+52/-47,21.716,0.0,32.26,30.21,5.48,0.599,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1224.0,+112/-120,25.903,0.035,33.05,26.03,20.96,0.482,✔️,0.0,False,GPU -40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1213.0,+75/-81,24.273,0.0,33.64,30.7,19.63,7.686,✔️,0.0,False,GPU -41,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1159.0,+107/-157,25.552,0.061,36.35,30.53,10.84,2.189,✔️,0.0,False,GPU -42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1087.0,+76/-138,28.351,0.0,39.56,30.71,14.75,0.337,✔️,0.0,False,CPU -43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1079.0,+217/-372,36.721,0.046,39.91,20.86,173.37,1.638,✔️,0.0,False,CPU -44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1050.0,+107/-233,38.3,0.0,41.06,38.81,905.14,0.875,✔️,0.0,False,CPU -45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1016.0,+105/-262,38.97,0.0,42.37,39.96,905.14,0.175,✔️,0.0,False,CPU -46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+128/-228,37.253,0.0,42.93,39.25,0.85,0.148,✔️,0.0,False,CPU -47,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),997.0,+163/-196,32.224,0.0,43.04,40.46,8.97,0.743,✔️,0.0,False,CPU -48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),943.0,+161/-236,39.279,0.0,44.77,41.62,0.78,0.147,✔️,0.0,False,CPU -49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),904.0,+45/-136,37.24,0.0,45.89,45.69,5.83,1.167,➖,0.0,False,GPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),897.0,+264/-569,46.585,0.03,46.07,35.58,173.37,0.147,✔️,0.0,False,CPU -51,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),786.0,+98/-282,45.98,0.0,48.56,48.16,4.72,0.188,✔️,0.0,False,CPU -52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),268.0,+117/-657,68.787,0.0,52.74,52.72,0.57,0.067,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1764.0,+174/-72,0.81,6.78,4.04,11.277,1154.67,7.879,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1742.0,+216/-92,0.798,7.4,3.26,8.446,2742.37,1.218,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1723.0,+255/-129,0.78,7.98,2.82,7.869,2742.37,7.945,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1638.0,+311/-171,0.688,11.09,4.43,9.408,11.21,0.845,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1601.0,+190/-126,0.568,12.67,6.65,12.456,4471.59,21.354,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1596.0,+161/-111,0.54,12.89,5.09,11.552,6.22,1.092,➖,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1574.0,+131/-70,0.488,13.88,8.37,13.559,4471.59,3.76,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1536.0,+144/-72,0.45,15.76,9.5,16.872,5929.46,1.73,✔️,0.0,False,GPU +8,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1526.0,+213/-87,0.371,16.28,7.58,16.751,6548.58,2.898,✔️,0.0,False,CPU +9,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1490.0,+147/-78,0.393,18.18,9.58,17.673,5929.46,0.337,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1474.0,+192/-155,0.415,19.06,7.42,19.175,4631.38,190.647,✔️,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1471.0,+95/-60,0.282,19.23,15.8,16.236,3272.91,1.185,✔️,0.0,False,CPU +12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1465.0,+157/-78,0.299,19.54,15.03,17.824,1502.45,6.33,✔️,0.0,False,CPU +13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1460.0,+137/-70,0.273,19.86,15.65,18.026,1502.45,1.367,✔️,0.0,False,CPU +14,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1444.0,+126/-99,0.324,20.77,15.8,18.679,684.78,4.816,➖,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1440.0,+86/-50,0.213,20.95,17.26,16.965,3272.91,0.21,✔️,0.0,False,CPU +16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1438.0,+111/-125,0.299,21.11,13.99,19.003,2097.62,3.142,✔️,0.0,False,GPU +17,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1431.0,+205/-161,0.38,21.47,8.27,20.624,4631.38,30.351,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1424.0,+137/-97,0.196,21.92,13.83,17.119,7359.32,9.026,✔️,0.0,False,GPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1421.0,+111/-79,0.227,22.06,17.09,18.298,2551.65,1.021,✔️,0.0,False,CPU +20,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1420.0,+89/-53,0.167,22.11,10.77,17.314,7359.32,0.596,✔️,0.0,False,GPU +21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1410.0,+126/-82,0.216,22.69,17.17,18.472,2551.65,4.527,✔️,0.0,False,CPU +22,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1402.0,+166/-108,0.277,23.15,14.83,19.531,19.65,0.236,✔️,0.0,False,GPU +23,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1396.0,+275/-233,0.458,23.49,4.93,17.032,22.56,1.484,➖,0.0,False,GPU +24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1350.0,+81/-75,0.128,26.23,22.79,20.66,63121.14,0.827,✔️,0.0,False,CPU +25,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1347.0,+197/-182,0.237,26.4,12.64,23.009,717.53,1.628,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1339.0,+224/-167,0.252,26.9,11.52,21.961,2775.63,12.592,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1338.0,+171/-109,0.114,26.91,19.1,21.152,4142.25,2.105,✔️,0.0,False,CPU +28,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1331.0,+80/-58,0.092,27.36,24.06,20.245,64.18,0.255,✔️,0.0,False,CPU +29,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1327.0,+100/-111,0.182,27.57,15.66,21.29,2097.62,0.515,✔️,0.0,False,GPU +30,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1312.0,+197/-205,0.242,28.5,12.88,23.234,755.05,2.155,✔️,0.0,False,CPU +31,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1302.0,+84/-77,0.072,29.06,24.55,21.503,63121.14,0.173,✔️,0.0,False,CPU +32,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1302.0,+230/-217,0.253,29.07,7.7,23.798,46.25,45.74,✔️,0.0,False,GPU +33,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1299.0,+201/-202,0.209,29.25,11.76,24.304,717.53,0.339,✔️,0.0,False,CPU +34,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1288.0,+143/-113,0.107,29.87,23.2,23.039,20.91,0.126,✔️,0.0,False,CPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1279.0,+211/-157,0.135,30.35,16.42,23.295,2775.63,1.431,✔️,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1273.0,+172/-161,0.152,30.73,23.52,22.58,5.32,0.431,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1264.0,+136/-129,0.106,31.24,21.05,23.695,4142.25,0.287,✔️,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1256.0,+188/-216,0.171,31.7,10.74,24.764,755.05,0.317,✔️,0.0,False,CPU +39,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1235.0,+48/-46,0.0,32.83,30.79,22.177,5.48,0.599,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1218.0,+114/-123,0.036,33.75,26.47,26.337,20.96,0.482,✔️,0.0,False,GPU +41,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1206.0,+72/-82,0.0,34.41,31.53,24.682,19.63,7.686,✔️,0.0,False,GPU +42,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1157.0,+104/-149,0.053,36.96,31.5,26.234,10.84,2.189,✔️,0.0,False,GPU +43,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1086.0,+70/-131,0.0,40.26,31.5,28.766,14.75,0.337,✔️,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1080.0,+215/-330,0.047,40.51,20.98,36.991,173.37,1.638,✔️,0.0,False,CPU +45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1046.0,+103/-215,0.0,41.93,39.78,38.74,905.14,0.875,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1012.0,+102/-240,0.0,43.23,40.94,39.299,905.14,0.175,✔️,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+128/-222,0.0,43.68,39.93,37.638,0.85,0.148,✔️,0.0,False,CPU +48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),994.0,+160/-186,0.0,43.88,41.4,32.443,8.97,0.743,✔️,0.0,False,CPU +49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),945.0,+158/-229,0.0,45.51,42.21,39.564,0.78,0.147,✔️,0.0,False,CPU +50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),908.0,+42/-136,0.0,46.62,46.4,37.448,5.83,1.167,➖,0.0,False,GPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),901.0,+261/-500,0.03,46.8,35.95,46.791,173.37,0.147,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),789.0,+88/-246,0.0,49.43,49.04,46.193,4.72,0.188,✔️,0.0,False,CPU +53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),267.0,+114/-488,0.0,53.74,53.72,68.914,0.57,0.067,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/winrate_matrix.png.zip index acefb7f7c6a3194cc68db9a220d43ac134de9d52..6c8fac8e06cbcced87722b7219c31fc2d691bd63 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:85c7aa4c385a4748735564be0568f539d0d67e5239f3e6e80b8a548e9b66d69b -size 2155304 +oid sha256:20f36eb4f4c68e890c6be19a86bf88a10e8461a8ba2143b07536bc54ff19f5f2 +size 2339653 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 7f9727b6ba52ecc7a8f58d68658694be95bb94ea..e57fa389ce5edd6473128dd276576494e38774b4 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:adc3070ee3202521b270613c1b0868ee11a3bd34faf5b3003d3a6b6a479ba922 -size 470825 +oid sha256:72cafc8d7d00b09777d03db9fec77b99bd35d14516da5f586f3ec6b21d755ca0 +size 492686 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip index e0dc53880351c7747d9f4530aa31aee7e0058a6e..8741fe748c176aead714d85bad71eff655a7975d 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bc4890a933182e29e072d239ee00b8f3fa5a480ed611c6d2df3f1871baea20c2 -size 993086 +oid sha256:eae7b2bbe9ba59363926d8a31c36e6994814d236e6203931d94bbfbc2c1c54d6 +size 973613 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip index 6de5a9cb85033d171d50d972944fd7b37b86842d..384d51bc0a308b42e52962a84715f091de932cce 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fa8ef0077128aadf5752c93607dd57ad048d5d6b39a36abb7dd6e1a78d353097 -size 122677 +oid sha256:bf58b74f4f4d8ea26407a55a45c8646d2e4b42031722e59ecafef37743074e0b +size 127420 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv index 76f7eaabc270279279db328ea3355f3fc07d6416..cb1dd5bd2c4eb37897ba9984e8e2bff79c0831e5 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1876.0,+270/-100,7.591,0.87,6.14,2.55,2710.27,7.889,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1852.0,+240/-92,8.338,0.851,6.75,3.29,2710.27,1.076,✔️,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1838.0,+207/-77,12.046,0.833,7.12,4.7,1374.26,12.843,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1799.0,+306/-137,8.785,0.807,8.24,3.94,10.23,0.844,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1788.0,+296/-112,12.979,0.791,8.58,4.37,3585.96,32.815,✔️,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1708.0,+168/-44,12.954,0.653,11.4,6.17,4150.25,19.131,✔️,0.0,False,GPU -6,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1688.0,+166/-61,11.831,0.628,12.2,5.26,5.85,0.91,➖,0.0,False,GPU -7,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1680.0,+197/-107,15.607,0.622,12.51,7.31,205.23,4.488,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1633.0,+220/-107,15.646,0.562,14.52,6.55,3585.96,1.167,✔️,0.0,False,GPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1630.0,+135/-43,14.49,0.553,14.68,8.59,4150.25,3.423,✔️,0.0,False,GPU -10,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1592.0,+203/-102,13.798,0.476,16.45,9.67,6.47,0.427,✔️,0.0,False,GPU -11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1575.0,+284/-135,18.07,0.414,17.28,7.5,7307.18,4.205,✔️,0.0,False,CPU -12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1549.0,+239/-110,18.854,0.412,18.59,10.08,5885.87,1.632,✔️,0.0,False,GPU -13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1513.0,+108/-46,17.651,0.303,20.47,17.01,3104.39,1.095,✔️,0.0,False,CPU -14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1497.0,+197/-114,19.852,0.346,21.34,10.08,5885.87,0.299,✔️,0.0,False,GPU -15,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1481.0,+240/-202,21.539,0.377,22.22,8.34,4513.63,201.505,✔️,0.0,False,GPU -16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1474.0,+122/-64,18.788,0.203,22.61,14.46,7367.31,7.816,✔️,0.0,False,GPU -17,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1470.0,+95/-49,18.528,0.218,22.86,18.63,3104.39,0.18,✔️,0.0,False,CPU -18,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1468.0,+142/-63,20.186,0.236,22.95,18.11,2210.02,1.236,✔️,0.0,False,CPU -19,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1460.0,+224/-87,20.164,0.219,23.37,17.51,1377.29,6.652,✔️,0.0,False,CPU -20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1455.0,+147/-63,20.414,0.221,23.69,18.61,2210.02,7.723,✔️,0.0,False,CPU -21,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1450.0,+196/-70,20.379,0.188,23.97,18.81,1377.29,1.395,✔️,0.0,False,CPU -22,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1433.0,+202/-128,21.045,0.245,24.92,19.16,696.38,4.428,➖,0.0,False,GPU -23,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1431.0,+237/-220,23.355,0.338,25.04,8.66,4513.63,26.967,✔️,0.0,False,GPU -24,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1424.0,+226/-151,21.754,0.294,25.44,10.59,12.57,2.243,✔️,0.0,False,GPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1420.0,+212/-145,22.067,0.267,25.68,15.1,17.83,0.197,✔️,0.0,False,GPU -26,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1412.0,+144/-153,21.235,0.205,26.15,21.17,1880.81,2.744,✔️,0.0,False,GPU -27,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1410.0,+86/-72,19.439,0.092,26.26,14.18,7367.31,0.587,✔️,0.0,False,GPU -28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1381.0,+315/-274,24.53,0.276,27.89,11.42,2644.99,12.716,✔️,0.0,False,CPU -29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1347.0,+200/-164,23.884,0.121,29.89,21.48,4654.35,2.028,✔️,0.0,False,CPU -30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1340.0,+132/-106,23.387,0.076,30.27,25.3,36484.23,0.712,✔️,0.0,False,CPU -31,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1320.0,+134/-78,22.957,0.06,31.45,27.13,264.83,0.227,✔️,0.0,False,CPU -32,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1318.0,+331/-279,27.104,0.291,31.55,7.6,40.51,46.546,✔️,0.0,False,GPU -33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1289.0,+146/-150,23.905,0.099,33.19,27.21,1880.81,0.353,✔️,0.0,False,GPU -34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1289.0,+262/-248,26.079,0.134,33.2,18.09,2644.99,1.47,✔️,0.0,False,CPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1282.0,+107/-99,24.339,0.027,33.59,27.61,36484.23,0.127,✔️,0.0,False,CPU -36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1278.0,+176/-198,26.121,0.106,33.8,27.13,717.36,1.838,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1255.0,+115/-83,26.62,0.029,35.05,28.39,17.94,0.418,✔️,0.0,False,GPU -38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1247.0,+64/-58,24.705,0.0,35.52,33.72,4.72,0.714,✔️,0.0,False,CPU -39,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1239.0,+180/-243,26.195,0.119,35.94,22.24,737.91,2.289,✔️,0.0,False,CPU -40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1232.0,+70/-89,26.605,0.0,36.29,33.66,20.73,8.179,✔️,0.0,False,GPU -41,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1230.0,+181/-140,27.149,0.06,36.45,24.6,4654.35,0.223,✔️,0.0,False,CPU -42,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1223.0,+101/-89,26.38,0.0,36.81,34.51,14.75,0.108,✔️,0.0,False,CPU -43,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1212.0,+159/-217,27.709,0.065,37.39,29.45,717.36,0.326,✔️,0.0,False,CPU -44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1204.0,+196/-142,25.943,0.055,37.78,34.29,5.04,0.449,✔️,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1160.0,+160/-231,27.997,0.033,40.02,25.17,737.91,0.316,✔️,0.0,False,CPU -46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1156.0,+183/-247,29.106,0.067,40.22,33.93,9.81,2.345,✔️,0.0,False,GPU -47,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1069.0,+131/-286,36.78,0.0,44.15,42.1,892.95,0.895,✔️,0.0,False,CPU -48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1066.0,+132/-195,32.333,0.0,44.29,33.03,13.69,0.314,✔️,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1031.0,+119/-328,37.562,0.0,45.67,43.14,892.95,0.146,✔️,0.0,False,CPU -50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+170/-313,37.99,0.0,46.83,42.97,0.74,0.137,✔️,0.0,False,CPU -51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),985.0,+174/-259,34.643,0.0,47.35,44.55,8.51,0.828,✔️,0.0,False,CPU -52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),954.0,+212/-444,42.698,0.0,48.38,41.73,151.69,1.252,✔️,0.0,False,CPU -53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),932.0,+195/-360,40.173,0.0,49.06,45.92,0.69,0.137,✔️,0.0,False,CPU -54,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),851.0,+29/-142,42.312,0.0,51.27,51.09,5.05,1.013,➖,0.0,False,GPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),775.0,+103/-312,43.542,0.0,52.93,52.56,4.53,0.164,✔️,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),580.0,+156/-543,54.165,0.0,55.58,55.16,151.69,0.099,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),178.0,+157/-1027,69.817,0.0,57.75,57.73,0.52,0.056,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1854.0,+258/-94,0.853,6.4,2.7,9.067,2710.27,7.889,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1831.0,+239/-86,0.834,7.0,3.42,9.813,2710.27,1.076,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1815.0,+176/-79,0.82,7.46,4.91,13.168,1374.26,12.843,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1780.0,+274/-128,0.788,8.54,4.44,10.383,10.23,0.844,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1768.0,+293/-109,0.778,8.92,4.71,13.973,3585.96,32.815,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1689.0,+165/-40,0.638,11.83,6.68,14.217,4150.25,19.131,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1670.0,+167/-65,0.61,12.61,5.54,13.199,5.85,0.91,➖,0.0,False,GPU +7,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),1660.0,+188/-104,0.613,13.01,7.5,16.287,205.23,4.488,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1617.0,+208/-103,0.551,14.97,6.78,16.57,3585.96,1.167,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1613.0,+126/-44,0.539,15.14,9.34,15.614,4150.25,3.423,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1577.0,+200/-98,0.459,16.92,10.42,15.15,6.47,0.427,✔️,0.0,False,GPU +11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1560.0,+264/-128,0.413,17.8,7.84,18.707,7307.18,4.205,✔️,0.0,False,CPU +12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1535.0,+236/-106,0.404,19.09,10.39,19.65,5885.87,1.632,✔️,0.0,False,GPU +13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1499.0,+99/-40,0.29,21.05,17.75,18.709,3104.39,1.095,✔️,0.0,False,CPU +14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1484.0,+191/-110,0.339,21.88,10.31,20.579,5885.87,0.299,✔️,0.0,False,GPU +15,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1472.0,+432/-287,0.528,22.56,4.69,18.42,22.52,1.352,➖,0.0,False,GPU +16,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1467.0,+218/-200,0.367,22.87,8.48,22.345,4513.63,201.505,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1461.0,+122/-67,0.201,23.2,14.89,19.565,7367.31,7.816,✔️,0.0,False,GPU +18,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1456.0,+90/-49,0.205,23.49,19.54,19.58,3104.39,0.18,✔️,0.0,False,CPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1456.0,+134/-60,0.23,23.49,18.66,20.933,2210.02,1.236,✔️,0.0,False,CPU +20,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1450.0,+210/-85,0.217,23.88,17.99,20.755,1377.29,6.652,✔️,0.0,False,CPU +21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1443.0,+142/-61,0.215,24.25,19.16,21.131,2210.02,7.723,✔️,0.0,False,CPU +22,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1439.0,+184/-68,0.186,24.5,19.44,20.984,1377.29,1.395,✔️,0.0,False,CPU +23,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1421.0,+190/-125,0.239,25.55,19.86,21.77,696.38,4.428,➖,0.0,False,GPU +24,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1419.0,+226/-222,0.33,25.7,8.76,24.025,4513.63,26.967,✔️,0.0,False,GPU +25,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1412.0,+227/-154,0.286,26.11,10.68,22.404,12.57,2.243,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1408.0,+200/-143,0.265,26.33,15.63,22.627,17.83,0.197,✔️,0.0,False,GPU +27,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1401.0,+141/-150,0.192,26.76,21.89,22.196,1880.81,2.744,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1399.0,+85/-74,0.091,26.84,14.45,20.13,7367.31,0.587,✔️,0.0,False,GPU +29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1370.0,+296/-274,0.274,28.56,11.66,24.902,2644.99,12.716,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1335.0,+188/-163,0.119,30.67,21.95,24.38,4654.35,2.028,✔️,0.0,False,CPU +31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1331.0,+126/-102,0.074,30.94,26.31,23.99,36484.23,0.712,✔️,0.0,False,CPU +32,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1310.0,+123/-79,0.058,32.13,28.3,23.48,264.83,0.227,✔️,0.0,False,CPU +33,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1309.0,+329/-286,0.292,32.21,7.62,27.455,40.51,46.546,✔️,0.0,False,GPU +34,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1280.0,+137/-143,0.088,33.89,28.16,24.812,1880.81,0.353,✔️,0.0,False,GPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1279.0,+255/-249,0.131,33.96,18.43,26.421,2644.99,1.47,✔️,0.0,False,CPU +36,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1274.0,+101/-95,0.025,34.28,28.32,24.955,36484.23,0.127,✔️,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1271.0,+173/-199,0.1,34.45,27.64,26.924,717.36,1.838,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1249.0,+115/-84,0.03,35.71,28.82,27.113,17.94,0.418,✔️,0.0,False,GPU +39,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1240.0,+62/-58,0.0,36.18,34.36,25.23,4.72,0.714,✔️,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1233.0,+177/-242,0.11,36.61,23.48,27.18,737.91,2.289,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1224.0,+66/-87,0.0,37.07,34.49,27.069,20.73,8.179,✔️,0.0,False,GPU +42,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1221.0,+172/-142,0.061,37.25,24.99,27.526,4654.35,0.223,✔️,0.0,False,CPU +43,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1216.0,+97/-83,0.0,37.55,35.3,26.871,14.75,0.108,✔️,0.0,False,CPU +44,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1206.0,+156/-215,0.061,38.05,30.01,28.442,717.36,0.326,✔️,0.0,False,CPU +45,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1198.0,+182/-146,0.054,38.47,35.02,26.339,5.04,0.449,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1156.0,+156/-228,0.028,40.7,28.24,28.978,737.91,0.316,✔️,0.0,False,CPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1151.0,+171/-232,0.059,40.94,34.96,29.89,9.81,2.345,✔️,0.0,False,GPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1064.0,+121/-249,0.0,44.99,43.05,37.281,892.95,0.895,✔️,0.0,False,CPU +49,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1061.0,+120/-198,0.0,45.11,33.8,32.806,13.69,0.314,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1027.0,+107/-286,0.0,46.51,44.12,37.933,892.95,0.146,✔️,0.0,False,CPU +51,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+166/-304,0.0,47.55,43.63,38.428,0.74,0.137,✔️,0.0,False,CPU +52,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),983.0,+169/-237,0.0,48.16,45.47,34.887,8.51,0.828,✔️,0.0,False,CPU +53,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),957.0,+207/-446,0.0,49.08,42.18,43.003,151.69,1.252,✔️,0.0,False,CPU +54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),935.0,+190/-378,0.0,49.76,46.49,40.494,0.69,0.137,✔️,0.0,False,CPU +55,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),852.0,+29/-148,0.0,52.11,51.92,42.542,5.05,1.013,➖,0.0,False,GPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),779.0,+98/-284,0.0,53.77,53.43,43.779,4.53,0.164,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),595.0,+131/-452,0.0,56.43,56.03,54.395,151.69,0.099,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),180.0,+144/-928,0.0,58.75,58.73,69.958,0.52,0.056,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip index f9369f578812394cc77fb5dc2be4a0855d94677c..644a112073485d0b8d1cdaa673b7800b140fec11 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fd0f60c3e4b688ef0737f83dcf74f06811cce4837bb6e781879bcdec34214d2d -size 2656649 +oid sha256:aa198fae34d291d70c6822ffc2ac39f15cbf01f266749e9bb68925cff67fbd80 +size 2665550 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index ebd6a4e73a6ef2725059aacb7bef29e9c80d7f6c..916aaf93f8491cc85753482c5bdc38b9a7f4b53f 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2dde6e64557305b14e1790b0dc1baf513553178052b4ec48416b28e00c400cae -size 426338 +oid sha256:f1d7e3da53ce7f0136a9d0b8a862806ec95f63db1c7a9214e2f3cf8fef86cc21 +size 440365 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip index 0ed3362c7a2ececff5e5e14cf5f5773b1ae4b692..73c53586714f202a2ceb3ff8fdacb08b4b968f50 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fad1246a0a408de16105ba444d7223cc1fa3968f2f552abcd95e669d774adad5 -size 955147 +oid sha256:1e36803644c9604cd43744bfcf02a70f42840b041e40c71c37ff77ba4d4ce779 +size 972826 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_all/tuning-impact-elo.png.zip index d29f80872247c959aba5ab6242db991c353f6c39..e7eb547dd32e85a6224ad6a23d0df7ee06c87220 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:72aa314fa5566052afb622b54aa1eee415f9c752941946ebd4bcb98eda09e086 -size 117379 +oid sha256:d8902f920aa99ed681018684cf38208476f34e90e9996e73c235a6471a32232b +size 119967 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_regression/datasets_all/website_leaderboard.csv index 1c8baffa1660d8d9fa27d71b937f731283ea2a14..305a12689e01f9f9d7ee0a1a79cece80ca57e8f3 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_all/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_regression/datasets_all/website_leaderboard.csv @@ -1,52 +1,53 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1816.0,+189/-125,1.486,0.812,6.26,2.56,1709.05,8.122,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1783.0,+136/-84,2.397,0.791,7.1,2.64,4786.55,239.537,✔️,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1760.0,+117/-75,2.908,0.696,7.73,3.92,649.34,5.869,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1730.0,+107/-72,2.848,0.729,8.63,4.27,4786.55,38.501,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1728.0,+113/-66,3.196,0.677,8.69,5.41,3995.01,10.051,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1689.0,+174/-116,2.246,0.697,9.94,4.34,1709.05,0.812,✔️,0.0,False,GPU -6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1675.0,+108/-89,4.64,0.567,10.44,6.37,1866.07,6.07,✔️,0.0,False,CPU -7,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1632.0,+135/-71,3.791,0.601,11.98,4.77,46.62,39.212,✔️,0.0,False,GPU -8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1606.0,+109/-79,4.177,0.517,12.99,7.92,3995.01,0.844,✔️,0.0,False,GPU -9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1600.0,+132/-117,5.528,0.506,13.21,5.41,3779.52,7.69,✔️,0.0,False,GPU -10,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1600.0,+168/-119,3.758,0.557,13.21,7.34,7.04,0.508,✔️,0.0,False,GPU -11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1535.0,+106/-61,6.221,0.369,15.79,9.82,3552.96,0.966,✔️,0.0,False,CPU -12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1527.0,+93/-80,6.683,0.361,16.1,12.59,700.15,9.322,✔️,0.0,False,CPU -13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1516.0,+120/-91,5.711,0.379,16.58,10.22,714.5,1.376,✔️,0.0,False,GPU -14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1509.0,+110/-76,6.341,0.348,16.87,9.48,3552.96,0.105,✔️,0.0,False,CPU -15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1496.0,+140/-90,5.114,0.349,17.46,12.61,4158.29,1.41,✔️,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1458.0,+87/-74,7.275,0.274,19.17,15.01,700.15,0.968,✔️,0.0,False,CPU -17,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1447.0,+191/-193,13.147,0.423,19.66,5.5,5.2,0.755,➖,0.0,False,GPU -18,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1442.0,+56/-45,7.217,0.255,19.92,17.26,834.93,2.614,✔️,0.0,False,CPU -19,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1426.0,+95/-84,6.387,0.262,20.63,16.37,714.5,0.099,✔️,0.0,False,GPU -20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1420.0,+60/-49,7.365,0.221,20.92,17.74,834.93,0.388,✔️,0.0,False,CPU -21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1419.0,+116/-86,7.527,0.265,20.96,12.86,3779.52,0.396,✔️,0.0,False,GPU -22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1409.0,+128/-104,6.071,0.25,21.44,14.82,4158.29,0.167,✔️,0.0,False,GPU -23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1404.0,+106/-103,7.922,0.262,21.68,15.62,10.89,0.092,✔️,0.0,False,CPU -24,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1307.0,+92/-77,9.067,0.118,26.06,20.62,15.5,0.299,✔️,0.0,False,GPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1304.0,+146/-119,7.789,0.161,26.23,20.14,13.32,0.131,✔️,0.0,False,GPU -26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1275.0,+127/-103,9.279,0.116,27.53,19.36,4608.59,1.233,✔️,0.0,False,CPU -27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1262.0,+132/-92,8.803,0.087,28.11,21.77,8.9,1.645,✔️,0.0,False,GPU -28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1234.0,+125/-104,11.617,0.107,29.43,21.54,158.22,0.844,✔️,0.0,False,CPU -29,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1226.0,+40/-39,9.708,0.005,29.76,28.4,2.11,0.275,✔️,0.0,False,CPU -30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1211.0,+115/-100,10.071,0.081,30.44,22.91,4608.59,0.097,✔️,0.0,False,CPU -31,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1206.0,+111/-97,11.83,0.082,30.68,20.81,158.22,0.151,✔️,0.0,False,CPU -32,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1204.0,+84/-82,10.368,0.061,30.79,27.39,2.24,0.242,✔️,0.0,False,CPU -33,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1179.0,+133/-168,12.864,0.117,31.89,23.15,2929.85,0.294,✔️,0.0,False,CPU -34,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1172.0,+114/-119,12.134,0.039,32.19,24.29,2.45,0.743,✔️,0.0,False,GPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1170.0,+70/-88,12.388,0.035,32.28,29.71,515.73,0.771,✔️,0.0,False,CPU -36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1133.0,+66/-94,12.876,0.025,33.8,31.32,515.73,0.124,✔️,0.0,False,CPU -37,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1129.0,+136/-171,13.402,0.088,33.95,24.71,2929.85,0.03,✔️,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1084.0,+104/-126,13.769,0.014,35.69,31.52,0.47,0.055,✔️,0.0,False,CPU -39,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1060.0,+128/-181,14.338,0.049,36.6,31.21,8.47,0.035,✔️,0.0,False,CPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1057.0,+101/-120,13.746,0.02,36.71,34.25,540.06,2.672,✔️,0.0,False,CPU -41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1043.0,+84/-102,13.502,0.0,37.2,35.18,20.48,0.08,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1000.0,+97/-105,14.233,0.01,38.71,37.01,540.06,0.325,✔️,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+54/-79,14.393,0.0,38.71,37.38,0.53,0.062,✔️,0.0,False,CPU -44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),912.0,+150/-180,19.563,0.007,41.42,36.02,92.55,0.897,✔️,0.0,False,CPU -45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),869.0,+106/-170,18.494,0.0,42.56,41.65,2.6,0.392,✔️,0.0,False,CPU -46,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),806.0,+150/-188,21.987,0.0,43.99,39.8,92.55,0.052,✔️,0.0,False,CPU -47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),672.0,+199/-297,29.17,0.0,46.39,39.43,0.19,0.04,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),506.0,+138/-315,36.388,0.0,48.39,48.21,193.95,0.173,✔️,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),459.0,+162/-378,36.561,0.0,48.83,48.63,193.95,0.074,✔️,0.0,False,CPU -50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),248.0,+141/-390,38.996,0.0,50.31,50.29,0.95,0.097,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1803.0,+185/-125,0.811,6.39,2.58,1.5,1709.05,8.122,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1772.0,+137/-82,0.785,7.22,2.67,2.411,4786.55,239.537,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1747.0,+122/-76,0.693,7.9,3.96,2.922,649.34,5.869,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1719.0,+111/-72,0.722,8.77,4.35,2.862,4786.55,38.501,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1717.0,+113/-64,0.669,8.81,5.48,3.21,3995.01,10.051,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1677.0,+178/-116,0.692,10.14,4.36,2.26,1709.05,0.812,✔️,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1663.0,+108/-89,0.557,10.65,6.43,4.654,1866.07,6.07,✔️,0.0,False,CPU +7,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1622.0,+129/-70,0.591,12.18,4.87,3.806,46.62,39.212,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1597.0,+108/-79,0.508,13.15,8.0,4.19,3995.01,0.844,✔️,0.0,False,GPU +9,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1590.0,+164/-117,0.551,13.43,7.41,3.773,7.04,0.508,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1588.0,+130/-118,0.502,13.5,5.47,5.542,3779.52,7.69,✔️,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1525.0,+103/-60,0.356,16.07,9.92,6.235,3552.96,0.966,✔️,0.0,False,CPU +12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1516.0,+96/-80,0.349,16.46,12.78,6.697,700.15,9.322,✔️,0.0,False,CPU +13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1506.0,+119/-89,0.374,16.88,10.51,5.725,714.5,1.376,✔️,0.0,False,GPU +14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1500.0,+107/-74,0.334,17.15,9.57,6.355,3552.96,0.105,✔️,0.0,False,CPU +15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1487.0,+140/-88,0.34,17.76,12.77,5.128,4158.29,1.41,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1449.0,+87/-71,0.262,19.54,15.19,7.289,700.15,0.968,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1441.0,+187/-187,0.417,19.89,5.57,13.161,5.2,0.755,➖,0.0,False,GPU +18,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1433.0,+58/-44,0.24,20.29,17.66,7.231,834.93,2.614,✔️,0.0,False,CPU +19,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1419.0,+94/-83,0.258,20.96,16.78,6.401,714.5,0.099,✔️,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1413.0,+61/-49,0.207,21.28,18.04,7.379,834.93,0.388,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1411.0,+112/-86,0.266,21.35,12.99,7.54,3779.52,0.396,✔️,0.0,False,GPU +22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1402.0,+126/-102,0.239,21.81,14.98,6.084,4158.29,0.167,✔️,0.0,False,GPU +23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1394.0,+103/-103,0.25,22.17,15.84,7.936,10.89,0.092,✔️,0.0,False,CPU +24,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1301.0,+90/-74,0.119,26.55,21.3,9.08,15.5,0.299,✔️,0.0,False,GPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1300.0,+141/-120,0.15,26.62,20.39,7.802,13.32,0.131,✔️,0.0,False,GPU +26,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1300.0,+145/-139,0.224,26.64,11.92,9.111,20.22,6.621,➖,0.0,False,GPU +27,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1271.0,+126/-102,0.108,28.0,19.57,9.293,4608.59,1.233,✔️,0.0,False,CPU +28,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1258.0,+128/-91,0.087,28.61,22.86,8.817,8.9,1.645,✔️,0.0,False,GPU +29,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1230.0,+119/-103,0.104,29.93,22.21,11.631,158.22,0.844,✔️,0.0,False,CPU +30,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1222.0,+42/-41,0.005,30.32,28.92,9.722,2.11,0.275,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1209.0,+111/-102,0.073,30.92,23.15,10.084,4608.59,0.097,✔️,0.0,False,CPU +32,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1204.0,+106/-96,0.08,31.19,21.48,11.844,158.22,0.151,✔️,0.0,False,CPU +33,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1198.0,+82/-82,0.062,31.42,27.85,10.382,2.24,0.242,✔️,0.0,False,CPU +34,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1173.0,+129/-162,0.109,32.58,23.72,12.878,2929.85,0.294,✔️,0.0,False,CPU +35,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1170.0,+113/-116,0.037,32.73,25.84,12.148,2.45,0.743,✔️,0.0,False,GPU +36,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1167.0,+71/-93,0.033,32.84,30.38,12.402,515.73,0.771,✔️,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1132.0,+66/-96,0.024,34.37,31.91,12.89,515.73,0.124,✔️,0.0,False,CPU +38,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1124.0,+133/-165,0.085,34.69,25.15,13.415,2929.85,0.03,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1081.0,+102/-124,0.011,36.4,32.43,13.783,0.47,0.055,✔️,0.0,False,CPU +40,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1055.0,+128/-181,0.046,37.42,31.89,14.352,8.47,0.035,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1052.0,+100/-120,0.02,37.52,34.93,13.759,540.06,2.672,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1042.0,+85/-105,0.0,37.91,35.79,13.516,20.48,0.08,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+53/-82,0.0,39.4,38.14,14.407,0.53,0.062,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),996.0,+97/-107,0.01,39.55,37.75,14.246,540.06,0.325,✔️,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),909.0,+148/-173,0.006,42.29,36.86,19.577,92.55,0.897,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),866.0,+108/-171,0.0,43.45,42.48,18.508,2.6,0.392,✔️,0.0,False,CPU +47,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),800.0,+148/-186,0.0,44.98,41.02,22.0,92.55,0.052,✔️,0.0,False,CPU +48,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),666.0,+197/-294,0.0,47.39,40.32,29.183,0.19,0.04,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),500.0,+138/-312,0.0,49.39,49.21,36.401,193.95,0.173,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),453.0,+160/-379,0.0,49.83,49.63,36.574,193.95,0.074,✔️,0.0,False,CPU +51,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),242.0,+139/-388,0.0,51.31,51.29,39.009,0.95,0.097,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_all/winrate_matrix.png.zip index 2c24c72725d42805b0c06c332f9394c206f218d8..e9df6a4920b30fd7a3e0e836a104a7821fc68815 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e93756f8142d92fec7354da521d9403a7c3f48839093c7e45ae10c72ba8a1ccf -size 1808653 +oid sha256:ab2097df80896ec0ae87a8ce4b48a6bcce76558eb20d9bf8484ed90baa6b3444 +size 2000997 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index f95d19af568ee2c0f3fb785ab82ca906c0224888..d3ef163716438469cd7b8bd9c06a79a858938b2f 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5b07713b4a66db34a4d074a568daff532d3fd43db354f700b23dd24f5874218f -size 429005 +oid sha256:d7e1ffbbe2d7754df0645dcd42c85fa559d1a851a7aee30d407f8a0e88489538 +size 442026 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip index 34e86756eebe9efba772006f1afadb29d833c57f..0e8fa78ac8512a8166c872fd27bf9b4e0e27aacf 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a9539158468ed4b66e71b28e2b8859bfa9e2b32fc149422010e9ea334685f276 -size 928150 +oid sha256:a0f9643c052a8ac465b5a0f2586fd6a136377db726826e818203143f804114b7 +size 922963 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_medium/tuning-impact-elo.png.zip index f019a4f0a6279abf4f388f9aaf3fba6802fdb016..677bb8b538e69b95cc07cd2ccb87b6de00e1c2fb 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:429741eeb2a12d29145d231294aab5a683f0da69b9b15c8c7b264aebf63ee32c -size 119720 +oid sha256:95f51f1c03fdf2edff019de30747f8626b877a075d1bef3b0e4abe51c6b231ed +size 122391 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_regression/datasets_medium/website_leaderboard.csv index e3cd8978a55c78ce453557b9d24fccd2ef76e30c..e1c802cedd223c5af92024526bfa387c2d5925ef 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_medium/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_regression/datasets_medium/website_leaderboard.csv @@ -1,52 +1,53 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),2069.0,+1045/-120,1.209,0.862,4.4,2.67,735.58,9.557,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1971.0,+783/-126,1.696,0.802,6.31,4.21,735.58,0.714,✔️,0.0,False,GPU -2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1922.0,+681/-41,2.304,0.713,7.47,5.68,2526.28,3.717,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1917.0,+1210/-123,1.293,0.782,7.6,2.16,4786.55,361.217,✔️,0.0,False,GPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1909.0,+515/-11,3.331,0.661,7.8,4.92,473.01,8.73,✔️,0.0,False,CPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1900.0,+797/-114,2.7,0.688,8.04,5.61,2028.29,1.675,✔️,0.0,False,GPU -6,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1884.0,+532/-21,3.488,0.644,8.47,4.35,430.4,7.174,✔️,0.0,False,GPU -7,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1850.0,+1175/-128,1.707,0.717,9.42,4.22,4786.55,41.599,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1802.0,+856/-218,2.636,0.564,10.87,7.04,1.6,0.508,✔️,0.0,False,GPU -9,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1753.0,+1086/-143,2.355,0.591,12.42,4.95,45.77,39.212,✔️,0.0,False,GPU -10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1721.0,+794/-134,3.741,0.535,13.47,8.53,2028.29,0.081,✔️,0.0,False,GPU -11,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1684.0,+503/-95,4.288,0.474,14.71,9.45,425.61,0.615,✔️,0.0,False,GPU -12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1671.0,+376/-133,5.936,0.388,15.18,11.21,222.05,11.236,✔️,0.0,False,CPU -13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1650.0,+347/-105,6.099,0.356,15.93,8.83,1658.41,0.742,✔️,0.0,False,CPU -14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1623.0,+616/-142,4.108,0.443,16.89,13.4,2526.28,0.166,✔️,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1619.0,+361/-178,6.197,0.35,17.04,7.26,1658.41,0.059,✔️,0.0,False,CPU -16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1618.0,+331/-121,6.24,0.326,17.09,13.63,222.05,0.968,✔️,0.0,False,CPU -17,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1602.0,+522/-214,5.117,0.341,17.64,14.24,3440.87,0.619,✔️,0.0,False,GPU -18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1555.0,+327/-104,5.243,0.296,19.4,17.23,425.61,0.04,✔️,0.0,False,GPU -19,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1545.0,+667/-210,4.564,0.305,19.76,5.04,2.06,0.755,➖,0.0,False,GPU -20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1533.0,+425/-345,6.882,0.326,20.22,14.49,5.49,0.037,✔️,0.0,False,CPU -21,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1526.0,+421/-182,5.96,0.24,20.44,18.38,3440.87,0.052,✔️,0.0,False,GPU -22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1509.0,+268/-136,6.83,0.217,21.09,19.43,282.13,1.74,✔️,0.0,False,CPU -23,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1506.0,+647/-156,5.272,0.285,21.18,18.43,6.88,0.094,✔️,0.0,False,GPU -24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1497.0,+267/-142,6.953,0.197,21.51,19.3,282.13,0.388,✔️,0.0,False,CPU -25,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1360.0,+317/-308,8.067,0.159,26.42,21.95,0.6,0.08,✔️,0.0,False,CPU -26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1330.0,+330/-211,7.827,0.091,27.49,25.06,5.03,0.025,✔️,0.0,False,GPU -27,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1305.0,+240/-126,8.79,0.011,28.33,27.32,2.28,0.13,✔️,0.0,False,GPU -28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1301.0,+209/-114,8.242,0.061,28.49,26.91,4348.61,0.528,✔️,0.0,False,CPU -29,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1292.0,+283/-160,8.309,0.013,28.78,27.91,0.55,0.233,✔️,0.0,False,CPU -30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1219.0,+204/-297,9.231,0.038,31.24,29.53,4348.61,0.034,✔️,0.0,False,CPU -31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1199.0,+231/-316,9.7,0.0,31.89,30.73,6.75,0.235,✔️,0.0,False,GPU -32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1176.0,+359/-419,10.264,0.022,32.62,30.66,35.67,0.529,✔️,0.0,False,CPU -33,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1156.0,+87/-156,9.993,0.0,33.27,33.0,79.08,0.546,✔️,0.0,False,CPU -34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1144.0,+260/-394,10.361,0.006,33.64,32.41,35.67,0.05,✔️,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1127.0,+112/-248,10.297,0.0,34.16,33.79,79.08,0.057,✔️,0.0,False,CPU -36,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1102.0,+387/-661,13.832,0.143,34.91,29.95,2044.56,0.128,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1037.0,+193/-452,13.252,0.001,36.8,35.28,15.08,0.028,✔️,0.0,False,CPU -38,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1029.0,+364/-709,14.402,0.114,37.02,33.51,2044.56,0.01,✔️,0.0,False,CPU -39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1026.0,+358/-676,12.06,0.006,37.11,35.07,0.17,0.049,✔️,0.0,False,CPU -40,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+37/-248,11.446,0.0,37.82,37.67,0.43,0.053,✔️,0.0,False,CPU -41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),976.0,+370/-780,15.083,0.102,38.47,35.09,3.66,0.011,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),848.0,+60/-540,14.323,0.0,41.4,41.16,318.98,0.827,✔️,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),794.0,+54/-529,14.781,0.0,42.44,42.26,318.98,0.114,✔️,0.0,False,CPU -44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),789.0,+112/-640,18.087,0.0,42.53,42.33,142.58,1.184,✔️,0.0,False,CPU -45,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),664.0,+83/-704,20.643,0.0,44.53,44.43,142.58,0.064,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),607.0,+116/-707,18.384,0.0,45.27,45.07,1.36,0.111,✔️,0.0,False,CPU -47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),349.0,+100/-1185,28.1,0.0,47.51,47.41,0.06,0.034,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),81.0,+131/-1638,39.828,0.0,48.92,48.92,44.3,0.048,✔️,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),-114.0,+182/-2167,40.21,0.0,49.77,49.76,44.3,0.011,✔️,0.0,False,CPU -50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),-414.0,+224/-2121,43.378,0.0,50.8,50.8,0.13,0.018,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),2047.0,+764/-106,0.863,4.4,2.67,1.209,735.58,9.557,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1950.0,+666/-113,0.804,6.31,4.21,1.696,735.58,0.714,✔️,0.0,False,GPU +2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1901.0,+626/-36,0.714,7.47,5.68,2.304,2526.28,3.717,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1896.0,+870/-123,0.784,7.6,2.16,1.293,4786.55,361.217,✔️,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1884.0,+405/-19,0.661,7.91,4.94,3.331,473.01,8.73,✔️,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1878.0,+626/-101,0.69,8.04,5.61,2.7,2028.29,1.675,✔️,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1856.0,+413/-22,0.644,8.64,4.37,3.488,430.4,7.174,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1827.0,+835/-112,0.72,9.49,4.23,1.707,4786.55,41.599,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1781.0,+682/-204,0.563,10.87,7.04,2.636,1.6,0.508,✔️,0.0,False,GPU +9,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1728.0,+756/-126,0.596,12.6,4.97,2.355,45.77,39.212,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1702.0,+631/-121,0.537,13.47,8.53,3.741,2028.29,0.081,✔️,0.0,False,GPU +11,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1660.0,+414/-86,0.477,14.91,9.51,4.288,425.61,0.615,✔️,0.0,False,GPU +12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1647.0,+417/-129,0.388,15.38,11.27,5.936,222.05,11.236,✔️,0.0,False,CPU +13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1626.0,+343/-86,0.356,16.13,8.87,6.099,1658.41,0.742,✔️,0.0,False,CPU +14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1602.0,+520/-127,0.446,17.0,13.53,4.108,2526.28,0.166,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1595.0,+380/-166,0.35,17.24,7.29,6.197,1658.41,0.059,✔️,0.0,False,CPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1594.0,+350/-124,0.327,17.29,13.7,6.24,222.05,0.968,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1580.0,+488/-198,0.341,17.82,14.46,5.117,3440.87,0.619,✔️,0.0,False,GPU +18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1533.0,+333/-88,0.3,19.6,17.39,5.243,425.61,0.04,✔️,0.0,False,GPU +19,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1529.0,+582/-198,0.312,19.76,5.04,4.564,2.06,0.755,➖,0.0,False,GPU +20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1507.0,+389/-310,0.326,20.6,14.61,6.882,5.49,0.037,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1506.0,+419/-165,0.24,20.64,18.61,5.96,3440.87,0.052,✔️,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1488.0,+296/-123,0.218,21.29,19.54,6.83,282.13,1.74,✔️,0.0,False,CPU +23,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1488.0,+498/-141,0.29,21.31,18.73,5.272,6.88,0.094,✔️,0.0,False,GPU +24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1477.0,+282/-127,0.2,21.71,19.41,6.953,282.13,0.388,✔️,0.0,False,CPU +25,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1340.0,+331/-237,0.161,26.8,22.15,8.067,0.6,0.08,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1316.0,+308/-192,0.09,27.69,25.31,7.827,5.03,0.025,✔️,0.0,False,GPU +27,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1294.0,+235/-101,0.011,28.53,27.45,8.79,2.28,0.13,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1289.0,+216/-86,0.061,28.69,27.06,8.242,4348.61,0.528,✔️,0.0,False,CPU +29,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1280.0,+286/-140,0.013,29.02,28.13,8.309,0.55,0.233,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1212.0,+206/-194,0.038,31.44,29.68,9.231,4348.61,0.034,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1189.0,+234/-185,0.001,32.24,31.03,9.7,6.75,0.235,✔️,0.0,False,GPU +32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1172.0,+372/-356,0.022,32.84,30.78,10.264,35.67,0.529,✔️,0.0,False,CPU +33,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1153.0,+93/-122,0.0,33.49,33.19,9.993,79.08,0.546,✔️,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1141.0,+280/-307,0.007,33.87,32.55,10.361,35.67,0.05,✔️,0.0,False,CPU +35,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1140.0,+414/-404,0.117,33.91,25.88,9.879,8.79,7.637,➖,0.0,False,GPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1126.0,+116/-182,0.0,34.38,33.98,10.297,79.08,0.057,✔️,0.0,False,CPU +37,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1086.0,+378/-683,0.145,35.67,30.41,13.832,2044.56,0.128,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1032.0,+193/-356,0.001,37.31,35.66,13.252,15.08,0.028,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1018.0,+370/-475,0.007,37.71,35.47,12.06,0.17,0.049,✔️,0.0,False,CPU +40,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1015.0,+356/-736,0.117,37.82,34.06,14.402,2044.56,0.01,✔️,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+53/-251,0.0,38.24,38.08,11.446,0.43,0.053,✔️,0.0,False,CPU +42,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),963.0,+363/-777,0.106,39.27,35.64,15.083,3.66,0.011,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),841.0,+67/-394,0.0,42.22,41.93,14.323,318.98,0.827,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),786.0,+61/-473,0.0,43.31,43.1,14.781,318.98,0.114,✔️,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),785.0,+97/-596,0.0,43.33,43.12,18.087,142.58,1.184,✔️,0.0,False,CPU +46,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),651.0,+66/-607,0.0,45.53,45.43,20.643,142.58,0.064,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),602.0,+108/-646,0.0,46.18,45.96,18.384,1.36,0.111,✔️,0.0,False,CPU +48,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),338.0,+94/-1186,0.0,48.51,48.42,28.1,0.06,0.034,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),70.0,+111/-1680,0.0,49.92,49.92,39.828,44.3,0.048,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),-125.0,+167/-2054,0.0,50.77,50.76,40.21,44.3,0.011,✔️,0.0,False,CPU +51,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),-424.0,+210/-2019,0.0,51.8,51.8,43.378,0.13,0.018,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_medium/winrate_matrix.png.zip index 9d25dc272f49cad02c500630df25fc825b19ec5b..693f26fa1ea5a3d4370f165cdc06a8ddbb328755 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:39c31dc82bf30b40fa16056158f67331b71d259ff03d8542b3daf22c2e2525f7 -size 1489576 +oid sha256:4b75ada03cbd5b0c71f47d1d8c192f56adff0bf05417b578f3232591b63b7a33 +size 1623604 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 57b9af2d66157c437c441f6646c3478865d8c35b..c26226a7938fbc69719dd73f6540f8333ef21882 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b7a1c2e8a752e20698083d943b23847e97c8af7502aa32b2186bb8b618b558d9 -size 419998 +oid sha256:7f494b0bdeeb926b02d8214e3bad74df67609b2d7227ca616645e1434bbc6db7 +size 433520 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip index 03b80186f5922ec3141d06d0af9925b4ab73050a..293b2ca6bab6bc3f2da4f8b6c36aadb9c2daf5cf 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b15da6955704a5aa6bcf2fe7dc27a7815a2b740e153b1b11992630ef054a1745 -size 987516 +oid sha256:8c375ea40d75d75b145d3e3e5f9f8713cffa36613a2bc367c1abff43c305f3db +size 982857 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_small/tuning-impact-elo.png.zip index de0568ba1ec97563f9fa9c335932ee17f7548c90..c21b596cadace56d8b548750198f324be6ae2ad4 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:15d1c1846761635bbeba3ed64ca4f16a8c89ff90668ceeef435e5580802d0087 -size 114452 +oid sha256:1d4b221331e3cc56eea176d778c8e7516a0317d961dd577dd99dcb1d92d5246f +size 122123 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_regression/datasets_small/website_leaderboard.csv index 339952b7b479372c4ee9c422bc2e94c85f1c595c..7f0b6c4c112c2db1c39659ad83448cb972add2bb 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_small/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_regression/datasets_small/website_leaderboard.csv @@ -1,52 +1,53 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1763.0,+145/-63,3.087,0.797,6.79,3.07,4822.0,140.81,✔️,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1746.0,+229/-93,2.545,0.729,7.27,3.69,709.58,5.864,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1740.0,+293/-144,1.659,0.782,7.42,2.5,2306.87,7.231,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1716.0,+119/-54,3.561,0.737,8.13,4.31,4822.0,22.324,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1686.0,+133/-56,3.505,0.67,9.09,5.29,4558.61,33.853,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1612.0,+126/-52,4.689,0.606,11.7,4.66,48.13,42.001,✔️,0.0,False,GPU -6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1602.0,+145/-76,5.458,0.509,12.08,7.82,5805.42,4.31,✔️,0.0,False,CPU -7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1599.0,+238/-134,2.59,0.631,12.2,4.42,2306.87,0.902,✔️,0.0,False,GPU -8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1587.0,+131/-72,4.449,0.506,12.69,7.58,4558.61,2.234,✔️,0.0,False,GPU -9,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1539.0,+225/-156,4.46,0.553,14.68,7.53,7.67,0.547,✔️,0.0,False,GPU -10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1516.0,+140/-75,6.298,0.377,15.7,10.55,4430.73,1.103,✔️,0.0,False,CPU -11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1494.0,+121/-71,7.15,0.345,16.68,13.64,1203.24,7.106,✔️,0.0,False,CPU -12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1492.0,+143/-72,6.43,0.347,16.76,11.72,4430.73,0.179,✔️,0.0,False,CPU -13,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1491.0,+180/-93,7.543,0.376,16.79,5.25,8977.36,8.017,✔️,0.0,False,GPU -14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1479.0,+203/-101,5.112,0.355,17.34,11.77,4438.37,1.981,✔️,0.0,False,GPU -15,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1471.0,+125/-94,6.601,0.321,17.74,10.77,2039.04,2.144,✔️,0.0,False,GPU -16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1440.0,+102/-35,7.459,0.278,19.19,16.13,1265.82,3.15,✔️,0.0,False,CPU -17,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1431.0,+292/-243,18.512,0.497,19.6,5.84,6.44,0.709,➖,0.0,False,GPU -18,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1413.0,+102/-71,7.922,0.242,20.47,16.02,1203.24,0.956,✔️,0.0,False,CPU -19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1412.0,+96/-39,7.622,0.237,20.55,16.89,1265.82,0.476,✔️,0.0,False,CPU -20,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1394.0,+135/-83,7.101,0.241,21.4,15.88,2039.04,0.109,✔️,0.0,False,GPU -21,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1381.0,+203/-117,6.14,0.257,22.06,13.23,4438.37,0.231,✔️,0.0,False,GPU -22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1370.0,+154/-99,8.572,0.222,22.59,16.41,12.31,0.204,✔️,0.0,False,CPU -23,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1352.0,+157/-92,9.663,0.154,23.51,12.54,8977.36,0.421,✔️,0.0,False,GPU -24,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1313.0,+187/-132,7.765,0.205,25.44,17.93,22.14,0.179,✔️,0.0,False,GPU -25,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1307.0,+151/-99,8.242,0.142,25.74,18.41,21.51,8.356,✔️,0.0,False,GPU -26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1284.0,+166/-166,9.927,0.15,26.92,16.47,5150.31,1.592,✔️,0.0,False,CPU -27,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1273.0,+116/-149,12.463,0.159,27.44,18.16,664.2,1.363,✔️,0.0,False,CPU -28,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1245.0,+114/-152,12.747,0.13,28.82,17.01,664.2,0.274,✔️,0.0,False,CPU -29,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1239.0,+68/-67,11.438,0.013,29.12,22.27,26.36,0.302,✔️,0.0,False,GPU -30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1222.0,+166/-151,10.595,0.107,29.94,20.1,5150.31,0.136,✔️,0.0,False,CPU -31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1220.0,+186/-176,12.259,0.101,30.01,20.27,4333.58,0.422,✔️,0.0,False,CPU -32,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1213.0,+58/-45,10.582,0.0,30.38,28.71,4.3,0.318,✔️,0.0,False,CPU -33,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1185.0,+94/-140,13.884,0.057,31.67,27.97,675.17,1.541,✔️,0.0,False,CPU -34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1177.0,+185/-183,12.776,0.072,32.03,21.23,4333.58,0.037,✔️,0.0,False,CPU -35,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1144.0,+53/-50,11.807,0.0,33.52,32.42,4.06,0.299,✔️,0.0,False,CPU -36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1143.0,+98/-144,14.489,0.041,33.57,29.95,675.17,0.206,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1138.0,+99/-130,13.385,0.033,33.78,31.0,2214.9,7.795,✔️,0.0,False,CPU -38,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1120.0,+153/-223,14.224,0.056,34.61,22.72,10.2,2.718,✔️,0.0,False,GPU -39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1115.0,+97/-173,14.837,0.018,34.79,29.64,0.5,0.116,✔️,0.0,False,CPU -40,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1100.0,+152/-182,13.873,0.016,35.44,29.19,16.01,0.041,✔️,0.0,False,CPU -41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1077.0,+95/-132,13.89,0.016,36.38,34.34,2214.9,0.838,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1050.0,+118/-144,13.659,0.0,37.45,35.11,22.39,0.134,✔️,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+87/-145,16.235,0.0,39.27,37.2,0.63,0.12,✔️,0.0,False,CPU -44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),956.0,+163/-363,20.486,0.011,40.72,32.95,88.43,0.512,✔️,0.0,False,CPU -45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),952.0,+90/-160,18.564,0.0,40.87,39.76,6.97,0.818,✔️,0.0,False,CPU -46,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),852.0,+166/-381,22.826,0.0,43.65,37.37,88.43,0.051,✔️,0.0,False,CPU -47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),756.0,+184/-548,29.839,0.0,45.69,35.68,0.42,0.047,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),600.0,+123/-419,34.237,0.0,48.06,47.79,777.57,0.312,✔️,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),585.0,+130/-439,34.281,0.0,48.25,47.95,777.57,0.108,✔️,0.0,False,CPU -50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),399.0,+87/-397,36.257,0.0,50.0,49.97,3.91,0.126,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1757.0,+147/-66,0.785,6.98,3.13,3.11,4822.0,140.81,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1740.0,+230/-91,0.724,7.44,3.75,2.568,709.58,5.864,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1734.0,+290/-141,0.778,7.63,2.52,1.682,2306.87,7.231,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1711.0,+122/-56,0.724,8.32,4.43,3.584,4822.0,22.324,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1681.0,+128/-57,0.656,9.28,5.4,3.528,4558.61,33.853,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1609.0,+127/-52,0.589,11.92,4.81,4.712,48.13,42.001,✔️,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1598.0,+140/-79,0.492,12.36,7.92,5.48,5805.42,4.31,✔️,0.0,False,CPU +7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1594.0,+234/-135,0.622,12.54,4.46,2.613,2306.87,0.902,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1583.0,+129/-74,0.489,12.96,7.7,4.472,4558.61,2.234,✔️,0.0,False,GPU +9,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1535.0,+225/-156,0.543,15.04,7.67,4.483,7.67,0.547,✔️,0.0,False,GPU +10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1513.0,+138/-75,0.355,16.03,10.72,6.32,4430.73,1.103,✔️,0.0,False,CPU +11,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1490.0,+137/-72,0.324,17.1,11.89,6.453,4430.73,0.179,✔️,0.0,False,CPU +12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1489.0,+127/-72,0.324,17.14,13.95,7.172,1203.24,7.106,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1486.0,+180/-94,0.369,17.26,5.34,7.566,8977.36,8.017,✔️,0.0,False,GPU +14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1477.0,+201/-101,0.34,17.72,11.9,5.134,4438.37,1.981,✔️,0.0,False,GPU +15,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1469.0,+125/-94,0.309,18.11,11.25,6.624,2039.04,2.144,✔️,0.0,False,GPU +16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1437.0,+98/-37,0.254,19.67,16.66,7.482,1265.82,3.15,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1430.0,+288/-239,0.483,19.98,5.95,18.535,6.44,0.709,➖,0.0,False,GPU +18,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1411.0,+105/-70,0.222,20.95,16.29,7.945,1203.24,0.956,✔️,0.0,False,CPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1410.0,+94/-40,0.212,21.01,17.27,7.645,1265.82,0.476,✔️,0.0,False,CPU +20,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1394.0,+133/-82,0.231,21.81,16.42,7.124,2039.04,0.109,✔️,0.0,False,GPU +21,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1388.0,+148/-158,0.292,22.1,8.92,8.631,22.46,1.039,➖,0.0,False,GPU +22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1379.0,+202/-119,0.238,22.54,13.36,6.162,4438.37,0.231,✔️,0.0,False,GPU +23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1367.0,+151/-98,0.202,23.16,16.72,8.595,12.31,0.204,✔️,0.0,False,CPU +24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1350.0,+156/-95,0.153,24.07,12.67,9.686,8977.36,0.421,✔️,0.0,False,GPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1313.0,+185/-132,0.188,25.95,18.18,7.787,22.14,0.179,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1306.0,+152/-101,0.14,26.34,19.63,8.265,21.51,8.356,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1282.0,+160/-169,0.136,27.57,16.69,9.949,5150.31,1.592,✔️,0.0,False,CPU +28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1271.0,+116/-146,0.155,28.11,18.92,12.486,664.2,1.363,✔️,0.0,False,CPU +29,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1244.0,+113/-149,0.126,29.51,17.72,12.77,664.2,0.274,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1237.0,+70/-68,0.012,29.83,23.31,11.461,26.36,0.302,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1222.0,+162/-153,0.094,30.59,20.35,10.618,5150.31,0.136,✔️,0.0,False,CPU +32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1221.0,+182/-173,0.087,30.65,20.85,12.281,4333.58,0.422,✔️,0.0,False,CPU +33,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1211.0,+59/-45,0.0,31.13,29.43,10.605,4.3,0.318,✔️,0.0,False,CPU +34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1184.0,+92/-139,0.054,32.44,28.85,13.907,675.17,1.541,✔️,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1178.0,+183/-182,0.065,32.73,21.62,12.799,4333.58,0.037,✔️,0.0,False,CPU +36,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1143.0,+50/-53,0.0,34.31,33.2,11.829,4.06,0.299,✔️,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1142.0,+94/-144,0.039,34.37,30.74,14.512,675.17,0.206,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1137.0,+99/-134,0.033,34.58,31.62,13.407,2214.9,7.795,✔️,0.0,False,CPU +39,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1120.0,+151/-227,0.054,35.35,24.93,14.247,10.2,2.718,✔️,0.0,False,GPU +40,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1115.0,+94/-177,0.014,35.58,30.77,14.86,0.5,0.116,✔️,0.0,False,CPU +41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1099.0,+151/-175,0.009,36.27,29.92,13.895,16.01,0.041,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1077.0,+96/-137,0.016,37.19,35.02,13.912,2214.9,0.838,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1050.0,+120/-143,0.0,38.28,35.88,13.681,22.39,0.134,✔️,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+83/-142,0.0,40.13,38.18,16.258,0.63,0.12,✔️,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),955.0,+162/-366,0.009,41.64,33.8,20.508,88.43,0.512,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),952.0,+97/-166,0.0,41.74,40.56,18.585,6.97,0.818,✔️,0.0,False,CPU +47,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),849.0,+165/-377,0.0,44.64,38.67,22.848,88.43,0.051,✔️,0.0,False,CPU +48,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),754.0,+183/-544,0.0,46.68,36.5,29.86,0.42,0.047,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),598.0,+122/-415,0.0,49.05,48.78,34.258,777.57,0.312,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),583.0,+130/-435,0.0,49.24,48.95,34.302,777.57,0.108,✔️,0.0,False,CPU +51,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),398.0,+86/-393,0.0,51.0,50.97,36.278,3.91,0.126,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_small/winrate_matrix.png.zip index 2544357581e947bc6313f2e2cf324392626a2573..6d646cfa1839b7da9eed1686c8691a2bd1300e44 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e2b974c4b9319e180d327d4bd1394e9d7a2dca40e7d490bef851fd8160c88fa4 -size 1820517 +oid sha256:471b9bfaccb66634ea7a3dc18c672d266cac6f48baf2dae6b93f0c75784e306c +size 1999068 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 193e38beb5962e8a2f37f35248b722d29fe2b5b0..e632d490e8de0710a11209241e540044cd5dc465 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:47c984a7d938d82dc253552c27707fd6428166cc13dd6a707963b8f3b77cc143 -size 453334 +oid sha256:26ae92637f00f54addd2da77117ca332f9371575cf501455ab61fdfb4ed172b0 +size 467971 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip index 9a649cfb79d98d6d41fdd709070dc1742163e2ea..3690320e48819631aa04fad03d9a7ddaa70f9ce1 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:db3e37f00a756464344b395ee4405cf9a773cb38023a6e91c3851aa7e9454a3c -size 1019357 +oid sha256:a637d422b9d687d878c662f33dc1931c69f69a40248b5e31b55ea24535879564 +size 1032206 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip index b196b2f3428f4c67513f7a563f2a8eb53525cea3..01aafdc788f4a087ab1e169d4feca21be88105d1 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a2a4d31d903566c633629b4f526b0de6ceec8aa949eb0946d3e0135d95f909d4 -size 119094 +oid sha256:1586b633bf5e8f2e00c2098b05bc7954543d548027b476de468b6904af72087a +size 123745 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/website_leaderboard.csv index 0e833c45528f5d3f913975bf8a03140d92d16394..b9ed4ea397cbb4f9d924fb29633ff04017be57a4 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/website_leaderboard.csv @@ -1,56 +1,57 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1738.0,+227/-92,2.676,0.724,8.07,4.14,669.92,5.859,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1729.0,+168/-38,3.492,0.767,8.36,3.52,4663.16,132.911,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1723.0,+332/-141,1.797,0.764,8.56,2.6,2166.18,6.341,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1699.0,+128/-34,3.903,0.721,9.38,4.73,4663.16,18.295,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1682.0,+131/-57,3.736,0.676,10.01,5.56,4823.88,33.539,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1626.0,+176/-56,4.903,0.627,12.17,4.57,47.35,39.185,✔️,0.0,False,GPU -6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1607.0,+195/-116,3.411,0.601,13.01,5.36,6577.99,56.795,✔️,0.0,False,GPU -7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1592.0,+287/-164,2.725,0.619,13.66,4.39,2166.18,0.812,✔️,0.0,False,GPU -8,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1568.0,+116/-54,6.052,0.451,14.73,9.53,6530.9,4.363,✔️,0.0,False,CPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1565.0,+155/-64,4.84,0.462,14.89,8.5,4823.88,1.952,✔️,0.0,False,GPU -10,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1557.0,+236/-169,4.606,0.583,15.25,7.94,7.15,0.501,✔️,0.0,False,GPU -11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1540.0,+116/-42,6.578,0.392,16.08,10.73,3558.14,0.966,✔️,0.0,False,CPU -12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1520.0,+129/-52,6.697,0.354,17.06,12.43,3558.14,0.105,✔️,0.0,False,CPU -13,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1503.0,+250/-242,19.511,0.546,17.88,5.52,5.94,0.652,➖,0.0,False,GPU -14,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1492.0,+191/-91,6.52,0.36,18.42,6.11,99.14,2.48,✔️,0.0,False,GPU -15,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1476.0,+131/-107,7.027,0.309,19.23,11.16,2045.39,1.388,✔️,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1475.0,+112/-68,7.822,0.303,19.3,15.45,1003.11,4.891,✔️,0.0,False,CPU -17,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1459.0,+217/-133,5.499,0.328,20.15,13.53,4158.29,1.946,✔️,0.0,False,GPU -18,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1440.0,+231/-181,5.382,0.391,21.16,9.88,6577.99,0.498,✔️,0.0,False,GPU -19,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1437.0,+138/-67,8.679,0.266,21.28,11.17,9308.22,7.885,✔️,0.0,False,GPU -20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1428.0,+88/-37,8.1,0.229,21.77,18.45,933.42,2.759,✔️,0.0,False,CPU -21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1412.0,+127/-98,7.406,0.246,22.61,16.94,2045.39,0.104,✔️,0.0,False,GPU -22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1401.0,+91/-46,8.265,0.192,23.23,19.21,933.42,0.388,✔️,0.0,False,CPU -23,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1399.0,+114/-70,8.659,0.201,23.35,18.34,1003.11,0.894,✔️,0.0,False,CPU -24,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1385.0,+140/-87,9.129,0.219,24.07,16.92,10.91,0.197,✔️,0.0,False,CPU -25,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1362.0,+218/-156,6.622,0.219,25.36,14.47,4158.29,0.208,✔️,0.0,False,GPU -26,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1354.0,+232/-206,7.982,0.284,25.77,13.92,9.1,0.85,✔️,0.0,False,GPU -27,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1299.0,+96/-60,10.902,0.073,28.81,20.91,9308.22,0.416,✔️,0.0,False,GPU -28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1293.0,+194/-163,13.262,0.171,29.15,18.85,623.85,1.144,✔️,0.0,False,CPU -29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1292.0,+215/-168,8.427,0.168,29.2,20.9,15.01,0.154,✔️,0.0,False,GPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1287.0,+170/-106,8.943,0.118,29.44,22.47,23.19,8.523,✔️,0.0,False,GPU -31,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1272.0,+170/-162,13.412,0.135,30.3,17.5,623.85,0.291,✔️,0.0,False,CPU -32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1250.0,+149/-168,12.707,0.091,31.47,21.51,2929.85,0.41,✔️,0.0,False,CPU -33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1228.0,+90/-90,12.445,0.015,32.61,24.17,27.82,0.299,✔️,0.0,False,GPU -34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1219.0,+163/-152,11.15,0.07,33.04,27.68,5692.02,1.354,✔️,0.0,False,CPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1204.0,+166/-180,13.286,0.074,33.82,22.52,2929.85,0.037,✔️,0.0,False,CPU -36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1202.0,+47/-44,11.45,0.0,33.94,32.18,4.57,0.275,✔️,0.0,False,CPU -37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1198.0,+126/-178,14.807,0.06,34.14,29.95,633.57,1.512,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1187.0,+188/-175,11.641,0.077,34.7,22.5,5692.02,0.131,✔️,0.0,False,CPU -39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1161.0,+120/-186,15.352,0.042,35.98,31.92,633.57,0.248,✔️,0.0,False,CPU -40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1142.0,+154/-178,14.894,0.055,36.87,25.15,10.27,3.409,✔️,0.0,False,GPU -41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1142.0,+46/-60,12.598,0.0,36.87,35.63,3.79,0.311,✔️,0.0,False,CPU -42,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1131.0,+130/-155,14.18,0.001,37.4,30.44,11.19,0.038,✔️,0.0,False,CPU -43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1128.0,+134/-196,15.73,0.002,37.53,32.07,0.47,0.117,✔️,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1085.0,+62/-94,14.849,0.0,39.46,38.31,2248.07,7.53,✔️,0.0,False,CPU -45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1025.0,+68/-114,15.336,0.0,41.94,41.02,2248.07,0.89,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1013.0,+116/-188,14.924,0.0,42.38,40.15,24.31,0.134,✔️,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+109/-207,17.325,0.0,42.87,40.52,0.53,0.118,✔️,0.0,False,CPU -48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),982.0,+155/-419,21.384,0.0,43.54,34.61,84.3,0.438,✔️,0.0,False,CPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),902.0,+53/-155,20.488,0.0,46.1,45.46,7.09,0.843,✔️,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),877.0,+150/-428,23.664,0.0,46.82,39.94,84.3,0.05,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),801.0,+167/-577,30.315,0.0,48.7,36.63,0.39,0.044,✔️,0.0,False,CPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),607.0,+111/-430,36.055,0.0,52.0,51.7,779.18,0.247,✔️,0.0,False,CPU -53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),594.0,+126/-470,36.078,0.0,52.17,51.85,779.18,0.089,✔️,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),421.0,+89/-422,37.463,0.0,53.9,53.88,3.97,0.122,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1739.0,+227/-91,0.719,8.27,4.19,2.698,669.92,5.859,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1729.0,+168/-39,0.762,8.57,3.59,3.514,4663.16,132.911,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1723.0,+330/-141,0.759,8.8,2.61,1.819,2166.18,6.341,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1700.0,+126/-35,0.714,9.59,4.88,3.925,4663.16,18.295,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1683.0,+128/-58,0.668,10.23,5.68,3.758,4823.88,33.539,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1629.0,+173/-55,0.621,12.42,4.68,4.925,47.35,39.185,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1608.0,+192/-116,0.588,13.34,5.52,3.433,6577.99,56.795,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1592.0,+290/-166,0.611,14.05,4.42,2.746,2166.18,0.812,✔️,0.0,False,GPU +8,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1570.0,+118/-56,0.44,15.05,9.66,6.073,6530.9,4.363,✔️,0.0,False,CPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1567.0,+155/-64,0.453,15.2,8.65,4.861,4823.88,1.952,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1558.0,+240/-170,0.574,15.66,8.09,4.628,7.15,0.501,✔️,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1541.0,+117/-41,0.378,16.46,10.88,6.6,3558.14,0.966,✔️,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1522.0,+130/-54,0.34,17.44,12.59,6.719,3558.14,0.105,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1506.0,+244/-239,0.539,18.25,5.6,19.533,5.94,0.652,➖,0.0,False,GPU +14,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),1495.0,+192/-92,0.358,18.84,6.31,6.542,99.14,2.48,✔️,0.0,False,GPU +15,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1479.0,+131/-107,0.301,19.65,11.67,7.049,2045.39,1.388,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1476.0,+114/-68,0.293,19.83,15.82,7.844,1003.11,4.891,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1462.0,+214/-134,0.321,20.58,13.69,5.521,4158.29,1.946,✔️,0.0,False,GPU +18,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1456.0,+100/-90,0.321,20.87,8.82,8.181,22.35,1.037,➖,0.0,False,GPU +19,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1440.0,+229/-182,0.385,21.72,10.03,5.404,6577.99,0.498,✔️,0.0,False,GPU +20,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1439.0,+137/-69,0.26,21.82,11.61,8.701,9308.22,7.885,✔️,0.0,False,GPU +21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1430.0,+85/-37,0.214,22.31,18.92,8.121,933.42,2.759,✔️,0.0,False,CPU +22,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1416.0,+125/-102,0.243,23.08,17.37,7.428,2045.39,0.104,✔️,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1403.0,+91/-46,0.176,23.76,19.64,8.287,933.42,0.388,✔️,0.0,False,CPU +24,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1401.0,+111/-70,0.187,23.9,18.64,8.681,1003.11,0.894,✔️,0.0,False,CPU +25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1386.0,+140/-88,0.206,24.73,17.21,9.151,10.91,0.197,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1365.0,+215/-157,0.213,25.91,14.61,6.643,4158.29,0.208,✔️,0.0,False,GPU +27,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1356.0,+232/-206,0.28,26.4,14.39,8.004,9.1,0.85,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1302.0,+96/-64,0.072,29.46,21.24,10.923,9308.22,0.416,✔️,0.0,False,GPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1296.0,+216/-171,0.162,29.79,21.18,8.448,15.01,0.154,✔️,0.0,False,GPU +30,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1294.0,+195/-165,0.164,29.92,19.42,13.284,623.85,1.144,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1290.0,+172/-110,0.117,30.13,23.29,8.964,23.19,8.523,✔️,0.0,False,GPU +32,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1272.0,+171/-164,0.128,31.09,18.24,13.434,623.85,0.291,✔️,0.0,False,CPU +33,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1252.0,+148/-168,0.086,32.21,22.04,12.728,2929.85,0.41,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1229.0,+91/-91,0.014,33.43,25.31,12.467,27.82,0.299,✔️,0.0,False,GPU +35,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1222.0,+164/-151,0.062,33.78,28.25,11.172,5692.02,1.354,✔️,0.0,False,CPU +36,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1206.0,+166/-180,0.07,34.62,22.92,13.308,2929.85,0.037,✔️,0.0,False,CPU +37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1202.0,+45/-45,0.0,34.8,33.0,11.471,4.57,0.275,✔️,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1198.0,+126/-179,0.053,35.02,30.86,14.829,633.57,1.512,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1190.0,+188/-176,0.07,35.44,22.79,11.662,5692.02,0.131,✔️,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1161.0,+121/-186,0.036,36.89,32.75,15.374,633.57,0.248,✔️,0.0,False,CPU +41,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1144.0,+154/-182,0.053,37.71,26.57,14.916,10.27,3.409,✔️,0.0,False,GPU +42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1143.0,+46/-61,0.0,37.77,36.5,12.62,3.79,0.311,✔️,0.0,False,CPU +43,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1132.0,+130/-156,0.0,38.27,31.11,14.201,11.19,0.038,✔️,0.0,False,CPU +44,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1129.0,+133/-199,0.001,38.43,33.18,15.752,0.47,0.117,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1086.0,+62/-96,0.0,40.38,39.21,14.87,2248.07,7.53,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1026.0,+70/-116,0.0,42.87,41.95,15.356,2248.07,0.89,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1014.0,+116/-188,0.0,43.33,41.08,14.945,24.31,0.134,✔️,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+109/-209,0.0,43.85,41.61,17.347,0.53,0.118,✔️,0.0,False,CPU +49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),983.0,+154/-421,0.0,44.47,35.41,21.405,84.3,0.438,✔️,0.0,False,CPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),903.0,+53/-156,0.0,47.09,46.46,20.509,7.09,0.843,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),878.0,+150/-428,0.0,47.8,40.99,23.685,84.3,0.05,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),802.0,+167/-576,0.0,49.69,37.37,30.336,0.39,0.044,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),608.0,+112/-431,0.0,53.0,52.7,36.075,779.18,0.247,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),594.0,+126/-471,0.0,53.16,52.84,36.098,779.18,0.089,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),422.0,+88/-423,0.0,54.9,54.88,37.483,3.97,0.122,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip index d0addc0af4c5786fb1c548f75151d364832a8fd2..bcfbabddc0f3947dce7028fbd342767f149026cf 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:570288d387692cf01750596fe7f47174e67fd37ca2168c1acdcda8843b018f30 -size 2122484 +oid sha256:ea1e96a37b050a7760b28f879bde0e28078e08e4823b037173d8a53a0aa2820a +size 2285400 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index 3ffc470fdf4fbfd1ca5453887890a79f8f12d0f7..5871c955883375bce0b78ef6a70ad0c769e2c795 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:158a7b8fcee64d8e1dfc1f68ee03f6f20d4cbd3ea6cbf7de5b1cef28e1cfb94c -size 406340 +oid sha256:a0f6063b3d5293e7aab3a38e7bd2a235ad3322a2370df6a9ffdb170de456ff21 +size 420226 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_all/pareto_n_configs_imp.png.zip index 262857be434bf759d0e2a7164bd467d7caddb593..04099d247867a5a770ac75d97936708f5c0ab67d 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bf031fc1a26f2b9c1d2e0c68a667655db3e31f124f9edc1612ff1cd4dff35d91 -size 943947 +oid sha256:f9e73a0b918f0a1151ba1fb21c404f854a3d644ae42913c85a19b2ecf32afefa +size 947904 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_all/tuning-impact-elo.png.zip index 1b25a32e189700e1b86e578ffd060124e4f41498..c14d5801c4f16d40c22feb82531826911c58ed49 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f0dca378cfa632370cda1dccb30509925c960ae1b327ad172ec8223a93904f7f -size 108706 +oid sha256:e2f65b67f39256e7fe67260494a273624209a4174e387aa9e5beababe7afa198 +size 113838 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_all/datasets_all/website_leaderboard.csv index 86b69b95a6d8e562f990a96c7bc3486a170ac05b..b1003434ccd068d7e1355d78cfcb4e461b70b8ee 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_all/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_all/datasets_all/website_leaderboard.csv @@ -1,51 +1,52 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1652.0,+102/-67,3.671,0.686,7.61,2.54,2059.94,9.785,✔️,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1647.0,+88/-77,5.78,0.664,7.77,3.65,556.15,6.31,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1603.0,+92/-71,5.787,0.614,9.21,3.81,2059.94,1.03,✔️,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1583.0,+63/-68,7.556,0.569,9.92,4.53,1754.94,1.767,✔️,0.0,False,CPU -4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1562.0,+83/-55,6.429,0.558,10.74,4.89,5.71,0.611,✔️,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1546.0,+58/-54,8.669,0.509,11.35,6.92,2791.97,13.886,✔️,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1462.0,+55/-60,10.194,0.416,15.05,7.54,2791.97,0.373,✔️,0.0,False,GPU -7,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1461.0,+74/-64,9.079,0.448,15.12,4.62,6154.73,386.167,✔️,0.0,False,GPU -8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1444.0,+42/-42,11.616,0.318,15.93,12.3,416.56,2.236,✔️,0.0,False,CPU -9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1441.0,+75/-47,10.656,0.376,16.06,8.26,3133.91,1.273,✔️,0.0,False,GPU -10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1417.0,+57/-51,11.2,0.325,17.27,11.49,1665.53,0.559,✔️,0.0,False,CPU -11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1416.0,+79/-61,10.724,0.4,17.34,5.58,6154.73,39.452,✔️,0.0,False,GPU -12,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1413.0,+85/-64,11.505,0.385,17.49,7.38,4618.5,7.737,✔️,0.0,False,GPU -13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1392.0,+51/-47,11.63,0.293,18.55,12.16,1665.53,0.065,✔️,0.0,False,CPU -14,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1390.0,+42/-50,12.245,0.265,18.62,12.58,700.96,1.439,✔️,0.0,False,CPU -15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1375.0,+56/-49,12.525,0.249,19.43,14.97,416.56,0.381,✔️,0.0,False,CPU -16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1371.0,+73/-56,11.592,0.294,19.64,11.95,3133.91,0.13,✔️,0.0,False,GPU -17,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1370.0,+61/-61,12.118,0.296,19.69,10.44,4618.5,0.47,✔️,0.0,False,GPU -18,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1369.0,+42/-45,12.212,0.252,19.73,11.47,6.7,0.088,✔️,0.0,False,CPU -19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1364.0,+46/-44,12.431,0.24,19.98,12.06,700.96,0.213,✔️,0.0,False,CPU -20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1363.0,+58/-46,12.395,0.284,20.04,11.85,866.11,2.007,✔️,0.0,False,GPU -21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1311.0,+58/-49,13.767,0.207,22.85,12.6,866.11,0.097,✔️,0.0,False,GPU -22,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1305.0,+76/-71,13.42,0.283,23.15,8.71,45.42,39.406,✔️,0.0,False,GPU -23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1301.0,+56/-53,14.0,0.212,23.39,16.03,11.56,0.127,✔️,0.0,False,GPU -24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1292.0,+53/-56,14.955,0.191,23.88,14.66,2961.52,0.482,✔️,0.0,False,CPU -25,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1272.0,+47/-54,13.751,0.148,24.92,19.48,2832.8,1.801,✔️,0.0,False,CPU -26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1272.0,+47/-53,14.37,0.146,24.92,17.63,10.44,1.714,✔️,0.0,False,GPU -27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1235.0,+65/-59,15.791,0.146,26.94,14.02,2961.52,0.048,✔️,0.0,False,CPU -28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1226.0,+62/-53,16.621,0.125,27.43,15.34,13.74,0.316,✔️,0.0,False,GPU -29,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1210.0,+58/-59,17.039,0.113,28.25,20.79,191.44,0.76,✔️,0.0,False,CPU -30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1210.0,+59/-63,16.737,0.13,28.27,15.82,7.66,0.046,✔️,0.0,False,CPU -31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1205.0,+49/-65,15.617,0.117,28.49,22.59,2832.8,0.112,✔️,0.0,False,CPU -32,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1195.0,+61/-57,15.665,0.108,29.04,17.06,2.06,0.122,✔️,0.0,False,CPU -33,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1187.0,+67/-72,17.388,0.119,29.46,21.04,594.95,4.651,✔️,0.0,False,CPU -34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1174.0,+70/-72,18.088,0.115,30.11,18.25,191.44,0.101,✔️,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1169.0,+61/-69,17.981,0.084,30.35,22.42,377.08,0.747,✔️,0.0,False,CPU -36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1160.0,+58/-50,16.457,0.066,30.85,27.1,2.2,0.171,✔️,0.0,False,CPU -37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1125.0,+53/-54,18.796,0.045,32.55,26.69,377.08,0.091,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1109.0,+66/-85,19.015,0.081,33.35,21.78,594.95,0.337,✔️,0.0,False,CPU -39,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1033.0,+72/-74,23.637,0.058,36.76,25.12,3.14,0.741,✔️,0.0,False,GPU -40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1027.0,+59/-69,20.862,0.026,37.0,33.69,8.96,0.129,✔️,0.0,False,CPU -41,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+69/-66,23.662,0.014,38.12,30.41,0.43,0.053,✔️,0.0,False,CPU -42,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),972.0,+68/-86,25.503,0.023,39.2,35.14,129.1,1.627,✔️,0.0,False,CPU -43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),970.0,+77/-89,25.257,0.014,39.27,35.71,0.26,0.054,✔️,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),969.0,+73/-80,22.958,0.029,39.31,36.05,3.12,0.312,✔️,0.0,False,CPU -45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),908.0,+84/-111,32.143,0.022,41.43,22.85,240.73,0.308,✔️,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),871.0,+82/-112,32.729,0.017,42.56,30.24,240.73,0.068,✔️,0.0,False,CPU -47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),808.0,+88/-128,35.299,0.01,44.25,41.39,1.23,0.115,✔️,0.0,False,CPU -48,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),804.0,+77/-92,31.641,0.01,44.33,42.72,129.1,0.103,✔️,0.0,False,CPU -49,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),601.0,+82/-142,44.249,0.0,48.0,47.66,0.19,0.037,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1648.0,+101/-68,0.684,7.73,2.55,3.695,2059.94,9.785,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1640.0,+90/-77,0.661,7.97,3.67,5.805,556.15,6.31,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1599.0,+89/-69,0.613,9.36,3.87,5.812,2059.94,1.03,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1577.0,+62/-67,0.565,10.16,4.58,7.58,1754.94,1.767,✔️,0.0,False,CPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1558.0,+81/-54,0.558,10.89,5.0,6.453,5.71,0.611,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1542.0,+56/-50,0.506,11.55,7.09,8.695,2791.97,13.886,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1458.0,+56/-60,0.412,15.36,8.21,10.22,2791.97,0.373,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1457.0,+71/-62,0.447,15.39,4.65,9.102,6154.73,386.167,✔️,0.0,False,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1440.0,+42/-43,0.317,16.26,12.44,11.64,416.56,2.236,✔️,0.0,False,CPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1438.0,+74/-46,0.371,16.37,8.38,10.681,3133.91,1.273,✔️,0.0,False,GPU +10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1414.0,+55/-49,0.321,17.55,11.89,11.225,1665.53,0.559,✔️,0.0,False,CPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1412.0,+79/-59,0.399,17.7,5.63,10.746,6154.73,39.452,✔️,0.0,False,GPU +12,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1408.0,+84/-65,0.383,17.86,7.41,11.528,4618.5,7.737,✔️,0.0,False,GPU +13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1389.0,+48/-48,0.289,18.88,12.29,11.655,1665.53,0.065,✔️,0.0,False,CPU +14,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1387.0,+42/-50,0.265,19.01,12.78,12.27,700.96,1.439,✔️,0.0,False,CPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1372.0,+55/-49,0.248,19.8,15.13,12.549,416.56,0.381,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1368.0,+62/-60,0.296,20.02,10.53,12.141,4618.5,0.47,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1368.0,+70/-56,0.29,20.03,12.12,11.617,3133.91,0.13,✔️,0.0,False,GPU +18,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1366.0,+42/-45,0.248,20.12,12.03,12.238,6.7,0.088,✔️,0.0,False,CPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1361.0,+43/-44,0.24,20.37,12.27,12.455,700.96,0.213,✔️,0.0,False,CPU +20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1360.0,+59/-47,0.282,20.45,12.03,12.418,866.11,2.007,✔️,0.0,False,GPU +21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1309.0,+59/-48,0.207,23.25,12.8,13.79,866.11,0.097,✔️,0.0,False,GPU +22,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1302.0,+77/-72,0.284,23.62,8.78,13.443,45.42,39.406,✔️,0.0,False,GPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1298.0,+55/-53,0.214,23.84,16.27,14.024,11.56,0.127,✔️,0.0,False,GPU +24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1290.0,+51/-56,0.191,24.29,14.84,14.98,2961.52,0.482,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1270.0,+47/-55,0.147,25.41,19.76,13.774,2832.8,1.801,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1270.0,+46/-52,0.143,25.43,18.13,14.395,10.44,1.714,✔️,0.0,False,GPU +27,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1263.0,+73/-67,0.227,25.78,10.54,15.394,13.95,2.081,➖,0.0,False,GPU +28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1233.0,+64/-60,0.146,27.45,14.19,15.815,2961.52,0.048,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1223.0,+62/-52,0.121,27.98,15.52,16.643,13.74,0.316,✔️,0.0,False,GPU +30,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1209.0,+58/-60,0.111,28.76,21.11,17.061,191.44,0.76,✔️,0.0,False,CPU +31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1207.0,+60/-60,0.131,28.84,16.26,16.763,7.66,0.046,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1204.0,+49/-63,0.115,29.04,23.02,15.64,2832.8,0.112,✔️,0.0,False,CPU +33,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1194.0,+60/-59,0.11,29.57,17.21,15.689,2.06,0.122,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1184.0,+67/-70,0.115,30.11,21.84,17.411,594.95,4.651,✔️,0.0,False,CPU +35,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1174.0,+69/-69,0.114,30.66,18.55,18.11,191.44,0.101,✔️,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1168.0,+59/-68,0.082,30.92,23.31,18.005,377.08,0.747,✔️,0.0,False,CPU +37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1158.0,+58/-50,0.068,31.46,27.56,16.48,2.2,0.171,✔️,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1125.0,+52/-53,0.044,33.14,27.7,18.82,377.08,0.091,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1106.0,+64/-83,0.077,34.08,22.35,19.036,594.95,0.337,✔️,0.0,False,CPU +40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1033.0,+71/-73,0.055,37.49,25.54,23.66,3.14,0.741,✔️,0.0,False,GPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1027.0,+59/-68,0.022,37.75,34.35,20.886,8.96,0.129,✔️,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+68/-64,0.014,38.86,32.49,23.686,0.43,0.053,✔️,0.0,False,CPU +43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),971.0,+66/-83,0.023,40.0,35.8,25.526,129.1,1.627,✔️,0.0,False,CPU +44,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),969.0,+76/-86,0.015,40.08,36.5,25.279,0.26,0.054,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),967.0,+75/-78,0.026,40.16,36.95,22.98,3.12,0.312,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),906.0,+82/-112,0.022,42.33,23.13,32.165,240.73,0.308,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),870.0,+80/-113,0.017,43.44,30.68,32.751,240.73,0.068,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),807.0,+88/-125,0.01,45.16,42.21,35.321,1.23,0.115,✔️,0.0,False,CPU +49,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),803.0,+75/-92,0.01,45.25,43.58,31.663,129.1,0.103,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),598.0,+80/-142,0.0,49.0,48.67,44.271,0.19,0.037,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_all/winrate_matrix.png.zip index b002816800ce81041958baff94e10bec68935dbd..2eb397cbd7d8eb158d905420a9778ae36c58828f 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4cb3d20a2e52fbd76fabbb31a203eea2c789f512a53d83479839b98b271f9791 -size 1650238 +oid sha256:22861b76e22b9c26b4c97c5b6bb89d8fe29bbc17f6fab39f75db72136f5ff954 +size 1853998 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index 46bb31c6634cf0d2c6367c022ff741eb8dd98757..90e8ac13d6d3dd6a37af0067053614aefe71fed3 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8f8117dc0a5ca7eb0157549bc2f7bff0aca0941e31b5af2d0057cab5528f3f3d -size 412803 +oid sha256:6935b0f2da35395af95a983bc2328d7082ef05368b17b0536c70ff0674d8cf01 +size 427642 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip index bb24c85f6035ad84630095f727c8378ed273d0bd..f67597949b575932848fa5458bc6ecabee056436 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9401766c7b58398a4a7f0529b22fe32d5e2afe2e790d248ba4b58153a16d832e -size 960289 +oid sha256:03f2209e1d12588a772e021435f5765504af9427ed39eda5d7f092a19e78328f +size 950519 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_medium/tuning-impact-elo.png.zip index c5d7f75c0b1611fb4f84507987bd6b1e887a5da9..e58e08203c76d5e17c42b40153dc783ec751dd64 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bfda76b2aa6770e7f801ae9784dcaadfa1140dbcb322797c287feb138ee71bb1 -size 110829 +oid sha256:1f6bf096cb41e7be5bd5e55125c8596e7853ee6f8fa83c89b6f0051b7fd4d5a8 +size 117434 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_all/datasets_medium/website_leaderboard.csv index b58532d50918e26fe5b2c33c220618212f95184d..ddf4911b14474a05ce1e2b6f8e8388495865d009 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_medium/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_all/datasets_medium/website_leaderboard.csv @@ -1,51 +1,52 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1903.0,+146/-69,2.092,0.802,4.47,2.35,432.14,4.214,✔️,0.0,False,CPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1861.0,+154/-67,2.561,0.768,5.27,2.85,302.05,2.868,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1746.0,+184/-126,3.897,0.659,8.13,2.86,739.1,11.65,✔️,0.0,False,GPU -3,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1705.0,+184/-133,4.31,0.633,9.4,4.51,2522.58,6.234,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1694.0,+166/-114,5.33,0.606,9.73,4.35,739.1,1.134,✔️,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1643.0,+104/-92,5.912,0.521,11.53,7.94,1714.29,1.743,✔️,0.0,False,GPU -6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1600.0,+93/-65,7.706,0.422,13.2,9.88,129.25,2.621,✔️,0.0,False,CPU -7,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1596.0,+112/-75,6.757,0.446,13.33,8.51,763.16,0.246,✔️,0.0,False,CPU -8,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1566.0,+103/-69,6.886,0.421,14.57,8.16,763.16,0.04,✔️,0.0,False,CPU -9,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1564.0,+147/-120,7.852,0.434,14.67,10.09,2522.58,0.318,✔️,0.0,False,GPU -10,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1560.0,+96/-65,7.44,0.371,14.8,10.43,289.35,0.706,✔️,0.0,False,CPU -11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1552.0,+161/-99,7.607,0.397,15.13,6.74,1993.57,0.581,✔️,0.0,False,GPU -12,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1543.0,+109/-91,8.332,0.352,15.53,11.78,129.25,0.117,✔️,0.0,False,CPU -13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1532.0,+138/-116,7.939,0.396,16.0,9.61,1714.29,0.079,✔️,0.0,False,GPU -14,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1529.0,+164/-124,6.506,0.4,16.13,8.01,1.91,0.668,✔️,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1524.0,+111/-86,7.158,0.393,16.37,7.52,2.93,0.045,✔️,0.0,False,CPU -16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1502.0,+106/-85,7.674,0.332,17.33,12.89,289.35,0.08,✔️,0.0,False,CPU -17,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1475.0,+149/-129,8.076,0.341,18.53,5.34,4532.14,387.927,✔️,0.0,False,GPU -18,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1465.0,+149/-93,8.336,0.296,19.0,10.82,1993.57,0.06,✔️,0.0,False,GPU -19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1423.0,+116/-118,9.191,0.283,20.93,14.58,559.64,0.88,✔️,0.0,False,GPU -20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1411.0,+152/-149,10.666,0.299,21.5,8.17,4532.14,39.452,✔️,0.0,False,GPU -21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1382.0,+104/-97,11.539,0.179,22.87,17.76,559.64,0.051,✔️,0.0,False,GPU -22,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1378.0,+131/-134,9.53,0.235,23.07,8.97,0.48,0.047,✔️,0.0,False,CPU -23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1375.0,+127/-114,10.925,0.191,23.2,19.49,5.42,0.078,✔️,0.0,False,GPU -24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1346.0,+80/-69,10.049,0.125,24.6,21.11,1803.23,1.199,✔️,0.0,False,CPU -25,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1331.0,+130/-148,13.131,0.21,25.33,17.48,1535.55,0.182,✔️,0.0,False,CPU -26,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1290.0,+130/-142,13.594,0.188,27.23,13.1,41.5,39.095,✔️,0.0,False,GPU -27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1288.0,+81/-70,12.893,0.055,27.33,25.1,6.54,0.294,✔️,0.0,False,GPU -28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1276.0,+95/-102,11.609,0.099,27.87,23.83,9.07,0.224,✔️,0.0,False,GPU -29,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1239.0,+83/-96,12.534,0.057,29.6,27.08,1803.23,0.051,✔️,0.0,False,CPU -30,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1236.0,+103/-99,12.859,0.048,29.73,24.05,65.73,0.23,✔️,0.0,False,CPU -31,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1222.0,+98/-130,12.599,0.044,30.33,27.5,36.73,0.359,✔️,0.0,False,CPU -32,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1218.0,+80/-93,12.025,0.03,30.53,28.9,0.34,0.07,✔️,0.0,False,CPU -33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1216.0,+135/-138,14.424,0.121,30.6,24.81,1535.55,0.013,✔️,0.0,False,CPU -34,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1204.0,+113/-137,15.52,0.109,31.13,27.61,2.99,0.016,✔️,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1203.0,+93/-86,13.195,0.037,31.17,27.59,65.73,0.025,✔️,0.0,False,CPU -36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1189.0,+115/-139,15.027,0.063,31.8,26.56,354.91,1.636,✔️,0.0,False,CPU -37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1148.0,+95/-121,14.978,0.021,33.53,31.16,36.73,0.038,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1074.0,+77/-88,16.346,0.005,36.47,35.3,2.58,0.039,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1049.0,+90/-96,17.593,0.011,37.4,35.95,354.91,0.094,✔️,0.0,False,CPU -40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1005.0,+164/-230,25.288,0.017,38.93,33.65,2.55,0.157,✔️,0.0,False,GPU -41,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+73/-113,18.839,0.0,39.1,38.22,0.24,0.022,✔️,0.0,False,CPU -42,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),977.0,+96/-131,22.57,0.011,39.87,37.93,150.58,4.952,✔️,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),933.0,+106/-153,21.097,0.015,41.2,39.26,1.49,0.113,✔️,0.0,False,CPU -44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),914.0,+167/-227,32.011,0.008,41.77,38.82,43.96,0.081,✔️,0.0,False,CPU -45,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),908.0,+108/-152,22.415,0.001,41.93,40.6,0.13,0.022,✔️,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),856.0,+156/-254,32.774,0.0,43.3,41.29,43.96,0.017,✔️,0.0,False,CPU -47,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),787.0,+59/-118,29.026,0.0,44.87,44.64,150.58,0.546,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),716.0,+107/-199,34.827,0.0,46.2,45.88,0.2,0.02,✔️,0.0,False,CPU -49,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),540.0,+101/-189,42.676,0.0,48.47,48.38,0.1,0.068,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1896.0,+152/-65,0.803,4.53,2.36,2.092,432.14,4.214,✔️,0.0,False,CPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1854.0,+161/-68,0.77,5.33,2.86,2.561,302.05,2.868,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1741.0,+184/-128,0.661,8.2,2.86,3.897,739.1,11.65,✔️,0.0,False,GPU +3,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1700.0,+181/-134,0.636,9.47,4.51,4.31,2522.58,6.234,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1690.0,+166/-115,0.609,9.8,4.36,5.33,739.1,1.134,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1641.0,+103/-90,0.527,11.53,7.94,5.912,1714.29,1.743,✔️,0.0,False,GPU +6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1595.0,+95/-68,0.426,13.33,9.91,7.706,129.25,2.621,✔️,0.0,False,CPU +7,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1593.0,+111/-74,0.45,13.4,8.53,6.757,763.16,0.246,✔️,0.0,False,CPU +8,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1563.0,+103/-66,0.424,14.63,8.16,6.886,763.16,0.04,✔️,0.0,False,CPU +9,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1559.0,+150/-118,0.439,14.8,10.17,7.852,2522.58,0.318,✔️,0.0,False,GPU +10,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1558.0,+97/-64,0.377,14.87,10.44,7.44,289.35,0.706,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1548.0,+158/-97,0.399,15.27,6.77,7.607,1993.57,0.581,✔️,0.0,False,GPU +12,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1539.0,+108/-90,0.357,15.67,11.81,8.332,129.25,0.117,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1528.0,+135/-117,0.402,16.13,9.62,7.939,1714.29,0.079,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1527.0,+159/-123,0.404,16.2,8.02,6.506,1.91,0.668,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1520.0,+113/-87,0.397,16.5,7.53,7.158,2.93,0.045,✔️,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1499.0,+105/-84,0.338,17.47,12.95,7.674,289.35,0.08,✔️,0.0,False,CPU +17,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1471.0,+153/-130,0.346,18.73,5.35,8.076,4532.14,387.927,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1462.0,+149/-91,0.3,19.13,10.85,8.336,1993.57,0.06,✔️,0.0,False,GPU +19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1418.0,+113/-114,0.291,21.2,14.67,9.191,559.64,0.88,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1407.0,+152/-149,0.304,21.77,8.18,10.666,4532.14,39.452,✔️,0.0,False,GPU +21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1381.0,+104/-97,0.187,23.0,17.83,11.539,559.64,0.051,✔️,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1376.0,+126/-134,0.241,23.27,8.98,9.53,0.48,0.047,✔️,0.0,False,CPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1371.0,+128/-113,0.194,23.47,19.62,10.925,5.42,0.078,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1345.0,+80/-68,0.126,24.73,21.24,10.049,1803.23,1.199,✔️,0.0,False,CPU +25,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1328.0,+129/-146,0.217,25.6,17.53,13.131,1535.55,0.182,✔️,0.0,False,CPU +26,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1286.0,+131/-142,0.196,27.63,13.19,13.594,41.5,39.095,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1285.0,+82/-72,0.058,27.67,25.28,12.893,6.54,0.294,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1272.0,+96/-101,0.108,28.27,24.26,11.609,9.07,0.224,✔️,0.0,False,GPU +29,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1238.0,+83/-95,0.059,29.87,27.25,12.534,1803.23,0.051,✔️,0.0,False,CPU +30,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1235.0,+103/-97,0.047,30.0,24.23,12.859,65.73,0.23,✔️,0.0,False,CPU +31,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1220.0,+99/-128,0.044,30.73,27.74,12.599,36.73,0.359,✔️,0.0,False,CPU +32,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1215.0,+79/-96,0.031,30.93,29.17,12.025,0.34,0.07,✔️,0.0,False,CPU +33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1215.0,+134/-136,0.13,30.93,24.94,14.424,1535.55,0.013,✔️,0.0,False,CPU +34,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1204.0,+93/-85,0.037,31.43,27.75,13.195,65.73,0.025,✔️,0.0,False,CPU +35,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1202.0,+112/-137,0.116,31.53,27.78,15.52,2.99,0.016,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1186.0,+114/-142,0.064,32.27,26.76,15.027,354.91,1.636,✔️,0.0,False,CPU +37,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1163.0,+112/-109,0.057,33.27,27.71,15.281,11.42,10.384,➖,0.0,False,GPU +38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1145.0,+97/-121,0.022,34.07,31.52,14.978,36.73,0.038,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1072.0,+81/-90,0.006,37.07,35.78,16.346,2.58,0.039,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1044.0,+93/-97,0.011,38.13,36.57,17.593,354.91,0.094,✔️,0.0,False,CPU +41,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1003.0,+164/-228,0.018,39.6,34.34,25.288,2.55,0.157,✔️,0.0,False,GPU +42,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+71/-114,0.0,39.7,38.73,18.839,0.24,0.022,✔️,0.0,False,CPU +43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),969.0,+95/-125,0.01,40.73,38.89,22.57,150.58,4.952,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),927.0,+104/-157,0.015,42.07,39.98,21.097,1.49,0.113,✔️,0.0,False,CPU +45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),908.0,+170/-227,0.007,42.63,39.55,32.011,43.96,0.081,✔️,0.0,False,CPU +46,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),904.0,+108/-149,0.002,42.73,41.27,22.415,0.13,0.022,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),851.0,+158/-250,0.0,44.17,42.04,32.774,43.96,0.017,✔️,0.0,False,CPU +48,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),777.0,+56/-117,0.0,45.87,45.65,29.026,150.58,0.546,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),706.0,+106/-200,0.0,47.2,46.89,34.827,0.2,0.02,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),531.0,+101/-190,0.0,49.47,49.38,42.676,0.1,0.068,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_medium/winrate_matrix.png.zip index 89a3b924987d574001bdf460591415c02b131f74..f36233b92ceae3b4cebf33968e44f2ca185c6ce9 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5410cbb32d9e2aeff5bd82f889e9720c1d3ee926c6143e1e0ecf6bcf4bd72dad -size 1388476 +oid sha256:d37543304db9c6d1dad33087f28ba1e68757ad95cdb9360a14f3a4c2061954f9 +size 1524005 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 4117bbaa6d00050a56d8f38cefac37f0af46531e..016bae72ad01bfbf364f5102f583331c88fbcc58 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:91d739cbb1cfc689afaaf966ca7c72018a0b6a84c671bd03df26a16f539d06e7 -size 407490 +oid sha256:3ba6dd0f281589918d6589601214d222e615c46f93c9c758fb51a117392b6a8c +size 420500 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_small/pareto_n_configs_imp.png.zip index a385570874e57edb00749df07790d76e2de6c4c7..91930c047422c080e0f05577492e1833e52d2eff 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:74ed447e0ee8f207330e742b9b5c03021122de8c1251f8bffd4ec72aa438243d -size 954131 +oid sha256:26c76ec1ef8df52e68700b4d4891c1e4a61b342cebec1ee985a244b4b3bbf82d +size 932119 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_small/tuning-impact-elo.png.zip index 9543e285311e61d10b13baa29e245ba81cd37256..6ad46c0d8e5a46c37cdefa7cda336ea06b0caff3 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:115bbdc8ff466f498833834d3eaaea57a96136b5abb5ec1631f1378ec2e150e6 -size 114869 +oid sha256:4cff7e0da8bec751ca9a499f9c3f5434facaa1cf96c902507520a022414b6cf0 +size 116559 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_all/datasets_small/website_leaderboard.csv index be00b71d8fa0b31f4a52dd0993ac310691f7e032..15669463eff8092b2cf1234b7c78b150aa05404a 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_small/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_all/datasets_small/website_leaderboard.csv @@ -1,52 +1,53 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1636.0,+133/-80,3.789,0.686,7.58,2.52,2319.48,8.957,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1601.0,+120/-70,6.609,0.613,8.71,4.3,6.58,0.586,✔️,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1586.0,+93/-71,7.334,0.607,9.25,4.24,657.21,7.989,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1583.0,+118/-73,6.191,0.605,9.35,3.72,2319.48,0.942,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1521.0,+89/-61,10.035,0.484,11.78,6.89,3855.22,24.935,✔️,0.0,False,GPU -5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1501.0,+59/-54,10.041,0.459,12.64,7.65,3351.09,1.56,✔️,0.0,False,CPU -6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1468.0,+97/-73,9.692,0.488,14.14,4.45,8059.77,364.288,✔️,0.0,False,GPU -7,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1456.0,+93/-77,12.275,0.474,14.71,5.45,5.11,0.624,➖,0.0,False,GPU -8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1446.0,+76/-75,11.353,0.402,15.21,7.8,3855.22,0.954,✔️,0.0,False,GPU -9,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1426.0,+92/-83,10.94,0.438,16.17,5.13,8059.77,39.456,✔️,0.0,False,GPU -10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1410.0,+74/-63,12.139,0.35,17.03,9.34,3629.74,1.48,✔️,0.0,False,GPU -11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1395.0,+49/-43,13.451,0.264,17.76,14.34,833.03,2.182,✔️,0.0,False,CPU -12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1361.0,+67/-55,13.265,0.258,19.58,14.1,2448.79,0.814,✔️,0.0,False,CPU -13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1347.0,+74/-65,13.932,0.271,20.36,12.18,1346.37,2.149,✔️,0.0,False,GPU -14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1343.0,+73/-70,13.161,0.276,20.57,12.83,3629.74,0.189,✔️,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1337.0,+60/-54,13.815,0.23,20.9,15.67,2448.79,0.078,✔️,0.0,False,CPU -16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1336.0,+60/-43,14.459,0.207,20.96,14.19,886.51,1.982,✔️,0.0,False,CPU -17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1325.0,+71/-76,14.708,0.272,21.61,10.38,6341.76,8.556,✔️,0.0,False,GPU -18,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1322.0,+52/-49,14.478,0.198,21.78,17.8,833.03,0.435,✔️,0.0,False,CPU -19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1321.0,+62/-49,14.619,0.193,21.83,12.01,886.51,0.318,✔️,0.0,False,CPU -20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1320.0,+50/-45,14.531,0.181,21.85,16.14,8.39,0.139,✔️,0.0,False,CPU -21,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1317.0,+87/-85,13.538,0.321,22.03,7.85,51.78,39.927,✔️,0.0,False,GPU -22,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1309.0,+68/-67,14.095,0.231,22.5,10.84,6341.76,0.501,✔️,0.0,False,GPU -23,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1288.0,+68/-61,14.901,0.208,23.68,11.95,1346.37,0.103,✔️,0.0,False,GPU -24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1282.0,+74/-56,15.925,0.175,24.03,14.51,3249.24,0.548,✔️,0.0,False,CPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1279.0,+71/-62,15.488,0.215,24.17,15.22,13.71,0.149,✔️,0.0,False,GPU -26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1270.0,+62/-68,15.198,0.169,24.69,16.32,16.6,4.299,✔️,0.0,False,GPU -27,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1250.0,+50/-60,15.496,0.153,25.81,19.29,3693.32,2.14,✔️,0.0,False,CPU -28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1244.0,+64/-60,16.566,0.15,26.17,12.07,3249.24,0.059,✔️,0.0,False,CPU -29,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1213.0,+77/-61,17.458,0.127,27.92,14.0,9.93,0.052,✔️,0.0,False,CPU -30,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1212.0,+73/-78,18.902,0.125,27.97,13.54,14.9,0.323,✔️,0.0,False,GPU -31,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1208.0,+82/-76,19.083,0.138,28.17,19.51,497.58,1.147,✔️,0.0,False,CPU -32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1196.0,+64/-71,17.098,0.137,28.83,21.74,3693.32,0.131,✔️,0.0,False,CPU -33,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1188.0,+83/-87,18.568,0.136,29.26,20.25,1666.78,7.827,✔️,0.0,False,CPU -34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1185.0,+86/-87,19.577,0.15,29.46,16.02,497.58,0.163,✔️,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1148.0,+76/-74,20.32,0.093,31.44,23.1,575.62,1.155,✔️,0.0,False,CPU -36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1140.0,+56/-61,18.504,0.079,31.85,27.22,3.63,0.218,✔️,0.0,False,CPU -37,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1130.0,+56/-64,18.427,0.052,32.36,28.08,3.1,0.207,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1128.0,+71/-93,19.793,0.106,32.47,20.92,1666.78,0.561,✔️,0.0,False,CPU -39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1099.0,+66/-74,21.333,0.046,33.99,27.62,575.62,0.147,✔️,0.0,False,CPU -40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1041.0,+93/-104,23.137,0.07,36.75,23.22,6.09,0.951,✔️,0.0,False,GPU -41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1011.0,+60/-89,22.946,0.028,38.11,33.87,12.68,0.155,✔️,0.0,False,CPU -42,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+72/-86,25.864,0.02,38.57,28.53,0.51,0.085,✔️,0.0,False,CPU -43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),988.0,+78/-86,26.623,0.018,39.08,35.07,0.43,0.081,✔️,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),979.0,+57/-96,23.922,0.03,39.44,35.96,5.83,0.571,✔️,0.0,False,CPU -45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),970.0,+65/-114,26.92,0.028,39.81,34.88,116.3,1.243,✔️,0.0,False,CPU -46,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),904.0,+86/-105,32.392,0.028,42.24,19.72,520.62,0.491,✔️,0.0,False,CPU -47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),874.0,+89/-114,32.903,0.024,43.19,27.61,520.62,0.097,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),834.0,+102/-146,35.691,0.014,44.39,40.65,2.59,0.129,✔️,0.0,False,CPU -49,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),808.0,+77/-124,32.92,0.014,45.08,43.04,116.3,0.075,✔️,0.0,False,CPU -50,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),616.0,+96/-191,45.084,0.0,48.78,48.35,0.37,0.035,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1633.0,+134/-84,0.687,7.72,2.53,3.789,2319.48,8.957,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1598.0,+113/-68,0.614,8.9,4.42,6.609,6.58,0.586,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1581.0,+96/-71,0.605,9.5,4.28,7.334,657.21,7.989,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1580.0,+115/-72,0.606,9.54,3.78,6.191,2319.48,0.942,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1517.0,+82/-60,0.484,12.06,7.07,10.035,3855.22,24.935,✔️,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1497.0,+56/-53,0.459,12.94,7.83,10.041,3351.09,1.56,✔️,0.0,False,CPU +6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1465.0,+97/-74,0.486,14.44,4.49,9.692,8059.77,364.288,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1454.0,+93/-76,0.472,14.99,5.58,12.275,5.11,0.624,➖,0.0,False,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1442.0,+76/-74,0.402,15.6,8.18,11.353,3855.22,0.954,✔️,0.0,False,GPU +9,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1423.0,+92/-82,0.435,16.56,5.19,10.94,8059.77,39.456,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1407.0,+72/-62,0.348,17.42,9.5,12.139,3629.74,1.48,✔️,0.0,False,GPU +11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1393.0,+48/-42,0.265,18.18,14.56,13.451,833.03,2.182,✔️,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1361.0,+63/-54,0.256,19.94,14.83,13.265,2448.79,0.814,✔️,0.0,False,CPU +13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1345.0,+72/-66,0.272,20.83,12.39,13.932,1346.37,2.149,✔️,0.0,False,GPU +14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1341.0,+72/-69,0.277,21.07,13.05,13.161,3629.74,0.189,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1336.0,+61/-52,0.227,21.35,15.92,13.815,2448.79,0.078,✔️,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1333.0,+59/-42,0.209,21.49,14.51,14.459,886.51,1.982,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1322.0,+71/-76,0.271,22.11,10.46,14.708,6341.76,8.556,✔️,0.0,False,GPU +18,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1320.0,+52/-48,0.2,22.25,18.07,14.478,833.03,0.435,✔️,0.0,False,CPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1318.0,+61/-49,0.194,22.33,12.27,14.619,886.51,0.318,✔️,0.0,False,CPU +20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1318.0,+48/-46,0.181,22.35,17.02,14.531,8.39,0.139,✔️,0.0,False,CPU +21,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1315.0,+87/-82,0.321,22.53,7.91,13.538,51.78,39.927,✔️,0.0,False,GPU +22,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1308.0,+68/-66,0.234,22.92,10.92,14.095,6341.76,0.501,✔️,0.0,False,GPU +23,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1300.0,+77/-82,0.286,23.39,9.72,15.62,16.54,1.255,➖,0.0,False,GPU +24,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1286.0,+68/-61,0.211,24.18,12.15,14.901,1346.37,0.103,✔️,0.0,False,GPU +25,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1281.0,+73/-55,0.173,24.5,14.71,15.925,3249.24,0.548,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1278.0,+72/-62,0.218,24.69,15.48,15.488,13.71,0.149,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1268.0,+61/-66,0.169,25.28,16.82,15.198,16.6,4.299,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1247.0,+49/-62,0.155,26.44,19.61,15.496,3693.32,2.14,✔️,0.0,False,CPU +29,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1242.0,+64/-60,0.149,26.75,12.22,16.566,3249.24,0.059,✔️,0.0,False,CPU +30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1211.0,+77/-60,0.127,28.56,14.37,17.458,9.93,0.052,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1210.0,+73/-78,0.124,28.58,13.67,18.902,14.9,0.323,✔️,0.0,False,GPU +32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1208.0,+82/-75,0.137,28.72,19.81,19.083,497.58,1.147,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1194.0,+62/-72,0.14,29.5,22.23,17.098,3693.32,0.131,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1186.0,+81/-86,0.136,29.99,21.14,18.568,1666.78,7.827,✔️,0.0,False,CPU +35,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1185.0,+83/-87,0.15,30.01,16.26,19.577,497.58,0.163,✔️,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1147.0,+74/-74,0.093,32.14,24.1,20.32,575.62,1.155,✔️,0.0,False,CPU +37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1139.0,+57/-62,0.082,32.54,27.73,18.504,3.63,0.218,✔️,0.0,False,CPU +38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1130.0,+56/-65,0.054,33.03,28.56,18.427,3.1,0.207,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1127.0,+71/-93,0.104,33.19,21.53,19.793,1666.78,0.561,✔️,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1098.0,+65/-75,0.045,34.71,28.85,21.333,575.62,0.147,✔️,0.0,False,CPU +41,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1042.0,+92/-104,0.069,37.5,23.56,23.137,6.09,0.951,✔️,0.0,False,GPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1010.0,+60/-89,0.028,38.92,34.59,22.946,12.68,0.155,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+71/-82,0.02,39.38,31.03,25.864,0.51,0.085,✔️,0.0,False,CPU +44,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),988.0,+76/-87,0.019,39.89,35.88,26.623,0.43,0.081,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),979.0,+59/-96,0.029,40.28,36.89,23.922,5.83,0.571,✔️,0.0,False,CPU +46,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),971.0,+64/-112,0.028,40.58,35.44,26.92,116.3,1.243,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),903.0,+85/-104,0.028,43.15,19.92,32.392,520.62,0.491,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),874.0,+88/-111,0.024,44.08,27.96,32.903,520.62,0.097,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),836.0,+102/-151,0.014,45.25,41.4,35.691,2.59,0.129,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),809.0,+75/-122,0.014,45.97,43.86,32.92,116.3,0.075,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),614.0,+94/-188,0.0,49.78,49.36,45.084,0.37,0.035,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_small/winrate_matrix.png.zip index 02bd67635acd2e4c7c57d502dc7c597c71a22ca8..3d735929d5f690ae089f23dcedb4a4f73ee10f5f 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9c7038bc67071760bfefd8b80f28a534cb77859a0a2eeef694039b91e6d32f9e -size 1755538 +oid sha256:355db9df47f0417983bb282c2574bb8e7bb9cce309b71130c30815c3f82042aa +size 1920486 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 2ec82b7644e0db3c85d9532612fc3d110c5a259d..fa683086c69cee1b1843bcfdc24651a6c26a4c9c 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6a037e55d2eae743df3a904bd20f113f91fdb39d32a6d1d5f0c30a0136554a44 -size 436370 +oid sha256:45387927267a262fb3028ee59031a6d95096bcf901799e61af4f1515b2470cc0 +size 450877 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip index 0a2cb9ba8e4277a715fe6a20888f119dffa551ac..111b0d3d8e723c9961ee2cde25f328df8db8f9a0 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:51da469fe975914a0c5714e7de394c3db21c44c9989feabd9bc000e08512d4d6 -size 1010232 +oid sha256:876783c437abfee73678df6f0ecbb468b5f7462120c744678f2be6eaa559497a +size 990324 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip index 8f448b80c2ee6ae2f02fddece319d0f512964ebe..3664c99ad96a5dd46ce746ef0afa251009c23ca8 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b5ef270574c931cdda27084bbff9f8968185d2eeac86a7447adb4d2b2d68395d -size 116602 +oid sha256:e446b67adadcf8a0c4e85051790c49d01ba25b4f31cefbd2d764eb3c02262498 +size 122713 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/website_leaderboard.csv index 2b2509f0606eb4c0fc2bea5440bc1b08a39c4d47..55ebff7ca869bcd338492efa552965ddd93e1463 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/website_leaderboard.csv @@ -1,56 +1,57 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1662.0,+123/-83,4.083,0.681,7.94,2.68,2200.35,8.314,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1641.0,+86/-70,6.963,0.624,8.62,4.92,6.21,0.561,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1608.0,+112/-89,6.6,0.599,9.85,4.13,2200.35,0.882,✔️,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1581.0,+100/-81,7.89,0.579,10.88,4.62,625.63,7.662,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1544.0,+104/-79,9.548,0.512,12.47,5.25,3650.56,52.333,✔️,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1542.0,+73/-57,10.583,0.484,12.55,7.17,3565.03,23.942,✔️,0.0,False,GPU -6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1494.0,+64/-52,10.816,0.422,14.85,8.48,3035.75,1.498,✔️,0.0,False,CPU -7,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1492.0,+85/-71,10.796,0.426,14.95,7.23,137.76,3.046,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1478.0,+93/-77,12.913,0.462,15.65,5.54,4.66,0.611,➖,0.0,False,GPU -9,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1469.0,+101/-74,10.352,0.458,16.12,4.82,7821.64,382.96,✔️,0.0,False,GPU -10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1462.0,+66/-66,11.945,0.396,16.47,8.59,3565.03,0.706,✔️,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1430.0,+110/-75,11.612,0.415,18.24,5.58,7821.64,39.414,✔️,0.0,False,GPU -12,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1419.0,+91/-76,12.117,0.355,18.83,10.17,3650.56,0.543,✔️,0.0,False,GPU -13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1405.0,+86/-73,12.789,0.325,19.64,10.49,3407.57,1.449,✔️,0.0,False,GPU -14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1376.0,+57/-44,14.548,0.204,21.32,17.79,780.12,2.129,✔️,0.0,False,CPU -15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1366.0,+66/-62,14.21,0.232,21.92,15.71,2234.83,0.653,✔️,0.0,False,CPU -16,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1356.0,+87/-62,13.444,0.324,22.52,8.57,4.97,0.545,✔️,0.0,False,GPU -17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1336.0,+83/-72,13.869,0.248,23.68,15.28,3407.57,0.173,✔️,0.0,False,GPU -18,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1334.0,+105/-86,14.157,0.327,23.82,7.82,51.52,40.033,✔️,0.0,False,GPU -19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1333.0,+73/-58,14.964,0.222,23.85,16.69,1216.65,2.007,✔️,0.0,False,GPU -20,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1333.0,+59/-51,14.839,0.194,23.85,18.67,2234.83,0.071,✔️,0.0,False,CPU -21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1327.0,+50/-54,15.598,0.165,24.2,19.28,848.99,1.866,✔️,0.0,False,CPU -22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1326.0,+95/-77,15.646,0.244,24.3,16.0,6308.39,7.737,✔️,0.0,False,GPU -23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1320.0,+58/-47,15.572,0.16,24.62,17.69,8.13,0.126,✔️,0.0,False,CPU -24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1317.0,+78/-84,14.8,0.217,24.82,12.21,6308.39,0.477,✔️,0.0,False,GPU -25,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1313.0,+60/-51,15.742,0.151,25.06,13.64,848.99,0.298,✔️,0.0,False,CPU -26,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1304.0,+72/-62,15.595,0.151,25.61,20.96,780.12,0.381,✔️,0.0,False,CPU -27,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1294.0,+73/-68,16.739,0.176,26.21,15.54,2961.52,0.521,✔️,0.0,False,CPU -28,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1290.0,+65/-57,15.72,0.164,26.45,16.92,16.29,4.082,✔️,0.0,False,GPU -29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1284.0,+80/-82,16.14,0.207,26.82,17.19,13.3,0.146,✔️,0.0,False,GPU -30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1274.0,+70/-67,15.938,0.157,27.44,19.34,1216.65,0.097,✔️,0.0,False,GPU -31,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1250.0,+74/-66,17.471,0.142,28.85,12.92,2961.52,0.054,✔️,0.0,False,CPU -32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1250.0,+68/-64,16.216,0.137,28.88,22.23,3460.09,2.057,✔️,0.0,False,CPU -33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1224.0,+80/-78,18.085,0.116,30.42,14.17,9.77,0.046,✔️,0.0,False,CPU -34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1223.0,+69/-63,19.271,0.113,30.52,13.73,14.5,0.3,✔️,0.0,False,GPU -35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1203.0,+87/-86,19.33,0.138,31.68,21.35,1453.07,6.678,✔️,0.0,False,CPU -36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1196.0,+73/-84,20.448,0.103,32.11,24.77,418.39,0.972,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1183.0,+69/-66,17.821,0.1,32.85,26.26,3460.09,0.121,✔️,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1174.0,+74/-87,20.873,0.126,33.33,19.28,418.39,0.147,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1141.0,+80/-84,20.613,0.104,35.24,21.86,1453.07,0.637,✔️,0.0,False,CPU -40,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1134.0,+78/-99,21.785,0.067,35.61,26.54,563.93,0.997,✔️,0.0,False,CPU -41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1123.0,+53/-61,19.728,0.028,36.21,33.38,2.86,0.19,✔️,0.0,False,CPU -42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1112.0,+53/-68,19.956,0.037,36.83,33.62,3.28,0.179,✔️,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1091.0,+71/-90,22.688,0.042,37.89,31.53,563.93,0.149,✔️,0.0,False,CPU -44,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1048.0,+93/-107,24.424,0.073,40.09,23.76,5.55,0.867,✔️,0.0,False,GPU -45,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1011.0,+78/-85,23.923,0.028,41.82,36.56,9.39,0.15,✔️,0.0,False,CPU -46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+86/-110,26.668,0.021,42.29,32.32,0.48,0.074,✔️,0.0,False,CPU -47,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),988.0,+92/-122,27.3,0.02,42.79,38.51,0.38,0.075,✔️,0.0,False,CPU -48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),984.0,+82/-97,24.547,0.027,42.97,39.0,5.2,0.522,✔️,0.0,False,CPU -49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),958.0,+74/-86,28.135,0.007,44.06,41.64,101.2,1.135,✔️,0.0,False,CPU -50,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),919.0,+86/-112,32.177,0.03,45.55,19.46,497.63,0.481,✔️,0.0,False,CPU -51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),891.0,+85/-107,32.668,0.025,46.5,27.98,497.63,0.084,✔️,0.0,False,CPU -52,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),855.0,+97/-122,34.952,0.012,47.67,43.57,2.47,0.121,✔️,0.0,False,CPU -53,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),779.0,+85/-116,34.443,0.0,49.73,48.99,101.2,0.065,✔️,0.0,False,CPU -54,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),631.0,+109/-168,45.322,0.0,52.58,52.11,0.28,0.026,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1659.0,+124/-83,0.681,8.09,2.68,4.083,2200.35,8.314,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1637.0,+88/-70,0.625,8.83,5.08,6.963,6.21,0.561,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1604.0,+111/-90,0.599,10.06,4.18,6.6,2200.35,0.882,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1577.0,+100/-80,0.576,11.15,4.65,7.89,625.63,7.662,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1540.0,+102/-78,0.513,12.77,5.31,9.548,3650.56,52.333,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1539.0,+70/-55,0.482,12.85,7.33,10.583,3565.03,23.942,✔️,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1491.0,+61/-50,0.421,15.18,8.65,10.816,3035.75,1.498,✔️,0.0,False,CPU +7,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),1488.0,+84/-71,0.425,15.35,7.31,10.796,137.76,3.046,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1476.0,+92/-77,0.461,15.95,5.64,12.913,4.66,0.611,➖,0.0,False,GPU +9,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1466.0,+101/-76,0.457,16.45,4.86,10.352,7821.64,382.96,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1459.0,+67/-64,0.393,16.86,9.05,11.945,3565.03,0.706,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1427.0,+110/-74,0.414,18.64,5.63,11.612,7821.64,39.414,✔️,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1416.0,+89/-76,0.353,19.24,10.4,12.117,3650.56,0.543,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1403.0,+84/-72,0.323,20.03,10.67,12.789,3407.57,1.449,✔️,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1374.0,+57/-45,0.205,21.77,18.05,14.548,780.12,2.129,✔️,0.0,False,CPU +15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1364.0,+66/-58,0.228,22.32,16.6,14.21,2234.83,0.653,✔️,0.0,False,CPU +16,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1353.0,+86/-62,0.322,23.0,8.84,13.444,4.97,0.545,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1334.0,+82/-71,0.249,24.2,15.56,13.869,3407.57,0.173,✔️,0.0,False,GPU +18,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1331.0,+58/-48,0.192,24.33,18.96,14.839,2234.83,0.071,✔️,0.0,False,CPU +19,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1331.0,+105/-85,0.327,24.33,7.87,14.157,51.52,40.033,✔️,0.0,False,GPU +20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1331.0,+73/-57,0.222,24.36,16.98,14.964,1216.65,2.007,✔️,0.0,False,GPU +21,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1328.0,+82/-89,0.28,24.53,10.22,16.208,13.95,1.208,➖,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1324.0,+49/-54,0.164,24.77,19.75,15.598,848.99,1.866,✔️,0.0,False,CPU +23,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1324.0,+94/-76,0.245,24.82,16.14,15.646,6308.39,7.737,✔️,0.0,False,GPU +24,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1318.0,+59/-45,0.161,25.17,18.79,15.572,8.13,0.126,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1317.0,+78/-84,0.22,25.24,12.29,14.8,6308.39,0.477,✔️,0.0,False,GPU +26,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1311.0,+60/-51,0.151,25.61,13.89,15.742,848.99,0.298,✔️,0.0,False,CPU +27,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1302.0,+71/-62,0.153,26.12,21.25,15.595,780.12,0.381,✔️,0.0,False,CPU +28,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1293.0,+72/-67,0.175,26.7,15.73,16.739,2961.52,0.521,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1287.0,+65/-57,0.165,27.06,17.41,15.72,16.29,4.082,✔️,0.0,False,GPU +30,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1282.0,+79/-82,0.209,27.36,17.44,16.14,13.3,0.146,✔️,0.0,False,GPU +31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1272.0,+70/-67,0.159,27.98,19.72,15.938,1216.65,0.097,✔️,0.0,False,GPU +32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1249.0,+74/-65,0.14,29.45,13.07,17.471,2961.52,0.054,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1247.0,+68/-64,0.139,29.55,22.59,16.216,3460.09,2.057,✔️,0.0,False,CPU +34,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1222.0,+78/-79,0.116,31.09,14.55,18.085,9.77,0.046,✔️,0.0,False,CPU +35,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1222.0,+69/-63,0.112,31.12,13.84,19.271,14.5,0.3,✔️,0.0,False,GPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1200.0,+86/-88,0.137,32.44,22.24,19.33,1453.07,6.678,✔️,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1195.0,+72/-81,0.101,32.71,25.17,20.448,418.39,0.972,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1181.0,+67/-65,0.103,33.55,26.9,17.821,3460.09,0.121,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1175.0,+75/-87,0.124,33.91,19.54,20.873,418.39,0.147,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1139.0,+81/-83,0.102,36.0,22.39,20.613,1453.07,0.637,✔️,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1133.0,+77/-94,0.066,36.36,27.61,21.785,563.93,0.997,✔️,0.0,False,CPU +42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1122.0,+55/-61,0.03,36.94,33.98,19.728,2.86,0.19,✔️,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1110.0,+54/-67,0.039,37.59,34.3,19.956,3.28,0.179,✔️,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1091.0,+69/-85,0.04,38.65,32.69,22.688,563.93,0.149,✔️,0.0,False,CPU +45,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1047.0,+90/-107,0.07,40.88,24.07,24.424,5.55,0.867,✔️,0.0,False,GPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1010.0,+76/-83,0.028,42.67,37.31,23.923,9.39,0.15,✔️,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+86/-110,0.022,43.11,34.47,26.668,0.48,0.074,✔️,0.0,False,CPU +48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),989.0,+92/-119,0.019,43.61,39.31,27.3,0.38,0.075,✔️,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),984.0,+78/-97,0.027,43.82,39.97,24.547,5.2,0.522,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),959.0,+74/-87,0.007,44.85,42.33,28.135,101.2,1.135,✔️,0.0,False,CPU +51,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),918.0,+86/-113,0.03,46.45,19.62,32.177,497.63,0.481,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),892.0,+84/-104,0.026,47.38,28.29,32.668,497.63,0.084,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),857.0,+100/-122,0.012,48.52,44.3,34.952,2.47,0.121,✔️,0.0,False,CPU +54,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),780.0,+83/-110,0.0,50.64,49.92,34.443,101.2,0.065,✔️,0.0,False,CPU +55,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),629.0,+108/-167,0.0,53.58,53.12,45.322,0.28,0.026,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/winrate_matrix.png.zip index 49707a685280d951e2ec78cd9a74c70ced604a01..440941bda9a889841711f898704927fe8b087c9b 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:df24977c336ef0261fe7105a917f6bf8e476e9e01cd601feaef0b2bac86afa73 -size 1975031 +oid sha256:8e4020843a13e7ef7ae5b5c25400a9e56654d6cc012738f57abcb9f1733846df +size 2150790 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index 72bef4906d784f510ed721540d7edc808bc4580e..f6bd68e86d18366884f2f6995d0b148354cdee59 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5161b0a70ebf831535ad94964ddcfa48e5fe7c31b44599d9ca5fc246d7294076 -size 440584 +oid sha256:09003622c6997a2ca0d0d22e7b2ca3fc770036abd8c92cdecc88d4ebe2b36650 +size 453500 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip index 9f318a441520c5cc6d9a47623ceb1d771498256a..8dd757b0170d1c6a333da0b4a9b17dd9fe5a9742 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:81ee7058d1801d257cfb925194c06aa707778878d949e6178589079565d79741 -size 955001 +oid sha256:98948b757036f255426ca757c96865d55515d0c63268b2af5544ddeba8baa2a7 +size 940463 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_all/tuning-impact-elo.png.zip index 7d437bae9291427f982b44fd6377fce6f9224435..d546ebfe5b9b4e1fe72cf6b22e030dfa9733e621 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:89f490d57e59d9ea4a2e5a4c010a537ef9c49409be22c5e7066c16501974ceef -size 122415 +oid sha256:dc42e4e412db992e9c04f3a1af9c6e1ced4dbe3a7c0656b95e15957e72e35aed +size 125725 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_binary/datasets_all/website_leaderboard.csv index f2ea4ed05810176238cf6cfee34acb7f6d2a241b..bb1ecd64ca62618dcd283e3e2b9eb95b00a5060c 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_all/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_all/website_leaderboard.csv @@ -1,53 +1,54 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1593.0,+118/-94,5.769,0.652,9.15,3.43,442.6,6.737,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1580.0,+109/-77,5.263,0.61,9.6,3.24,1877.76,10.745,✔️,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1580.0,+106/-77,7.063,0.598,9.63,4.44,1231.21,1.36,✔️,0.0,False,CPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1564.0,+103/-72,6.622,0.58,10.23,4.57,4.79,0.708,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1550.0,+111/-66,6.293,0.574,10.77,4.42,1877.76,1.035,✔️,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1528.0,+75/-64,8.807,0.507,11.7,7.56,2721.85,9.311,✔️,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1446.0,+86/-72,10.77,0.42,15.52,6.33,2721.85,0.265,✔️,0.0,False,GPU -7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1437.0,+90/-66,10.645,0.415,16.0,6.81,2180.12,1.135,✔️,0.0,False,GPU -8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1414.0,+62/-51,12.259,0.31,17.22,12.93,328.64,0.774,✔️,0.0,False,CPU -9,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1390.0,+80/-71,12.822,0.311,18.52,10.69,462.92,0.613,✔️,0.0,False,CPU -10,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1389.0,+65/-54,11.84,0.315,18.53,9.21,3.84,0.068,✔️,0.0,False,CPU -11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1388.0,+81/-62,9.185,0.349,18.6,6.75,5890.6,409.986,✔️,0.0,False,GPU -12,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1387.0,+104/-82,11.692,0.342,18.65,8.68,3436.74,0.405,✔️,0.0,False,GPU -13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1375.0,+74/-75,11.971,0.324,19.3,11.38,1043.89,0.478,✔️,0.0,False,CPU -14,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1373.0,+104/-101,11.837,0.372,19.42,7.62,3436.74,8.478,✔️,0.0,False,GPU -15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1363.0,+94/-75,11.784,0.328,19.98,10.69,2180.12,0.117,✔️,0.0,False,GPU -16,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1351.0,+62/-60,12.509,0.287,20.67,13.8,1043.89,0.041,✔️,0.0,False,CPU -17,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1350.0,+77/-81,13.208,0.265,20.7,9.89,462.92,0.108,✔️,0.0,False,CPU -18,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1350.0,+102/-71,14.058,0.278,20.73,11.47,248.47,1.7,➖,0.0,False,GPU -19,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1343.0,+84/-84,11.401,0.305,21.12,7.47,5890.6,39.565,✔️,0.0,False,GPU -20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1338.0,+62/-58,13.439,0.238,21.4,17.03,328.64,0.089,✔️,0.0,False,CPU -21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1332.0,+76/-56,14.224,0.23,21.7,13.18,1579.5,0.428,✔️,0.0,False,CPU -22,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1316.0,+78/-87,13.107,0.245,22.63,13.61,559.77,1.513,✔️,0.0,False,GPU -23,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1315.0,+56/-53,13.331,0.189,22.7,15.2,9.04,1.162,✔️,0.0,False,GPU -24,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1301.0,+94/-79,14.33,0.245,23.5,15.1,8.0,0.124,✔️,0.0,False,GPU -25,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1287.0,+72/-59,13.514,0.187,24.33,17.4,2206.58,2.301,✔️,0.0,False,CPU -26,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1272.0,+67/-73,14.761,0.177,25.18,17.14,559.77,0.086,✔️,0.0,False,GPU -27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1269.0,+68/-58,14.999,0.158,25.33,12.79,1579.5,0.05,✔️,0.0,False,CPU -28,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1266.0,+73/-64,15.996,0.167,25.53,12.12,4.79,0.045,✔️,0.0,False,CPU -29,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1231.0,+74/-95,14.605,0.189,27.52,10.76,47.89,39.446,✔️,0.0,False,GPU -30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1227.0,+75/-90,16.538,0.154,27.75,19.43,561.29,4.462,✔️,0.0,False,CPU -31,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1209.0,+88/-82,16.117,0.145,28.77,13.58,1.42,0.118,✔️,0.0,False,CPU -32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1204.0,+76/-65,15.932,0.14,29.1,22.97,2206.58,0.107,✔️,0.0,False,CPU -33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1195.0,+66/-70,16.944,0.12,29.6,12.96,11.15,0.312,✔️,0.0,False,GPU -34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1173.0,+71/-64,17.891,0.086,30.82,25.14,122.9,0.583,✔️,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1144.0,+70/-84,19.294,0.064,32.37,26.56,171.61,0.552,✔️,0.0,False,CPU -36,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1142.0,+83/-115,18.653,0.111,32.5,18.23,561.29,0.236,✔️,0.0,False,CPU -37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1140.0,+67/-71,17.125,0.074,32.62,28.52,0.93,0.094,✔️,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1131.0,+76/-90,19.504,0.093,33.07,19.56,122.9,0.072,✔️,0.0,False,CPU -39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1108.0,+66/-74,19.946,0.022,34.28,30.56,171.61,0.053,✔️,0.0,False,CPU -40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1041.0,+74/-76,21.201,0.044,37.6,32.81,4.97,0.092,✔️,0.0,False,CPU -41,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1013.0,+111/-133,26.33,0.08,38.87,29.06,0.81,0.115,➖,0.0,False,GPU -42,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+77/-100,24.112,0.025,39.45,27.55,0.29,0.031,✔️,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1000.0,+76/-108,21.671,0.042,39.47,35.71,2.79,0.259,✔️,0.0,False,CPU -44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),984.0,+68/-95,24.306,0.006,40.13,38.17,140.66,3.209,✔️,0.0,False,CPU -45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),974.0,+97/-117,27.936,0.038,40.53,17.35,155.53,0.277,✔️,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),938.0,+95/-130,28.597,0.029,41.95,25.15,155.53,0.05,✔️,0.0,False,CPU -47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),938.0,+92/-121,26.856,0.047,41.97,36.6,2.71,0.392,✔️,0.0,False,GPU -48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),932.0,+96/-116,25.941,0.017,42.17,37.91,0.18,0.035,✔️,0.0,False,CPU -49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),902.0,+102/-109,30.347,0.017,43.27,39.53,0.8,0.088,✔️,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),816.0,+76/-102,30.586,0.0,45.87,45.13,140.66,0.188,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),600.0,+97/-211,43.693,0.0,50.0,49.65,0.15,0.024,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1590.0,+119/-93,0.648,9.35,3.43,5.81,442.6,6.737,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1579.0,+111/-77,0.609,9.77,3.26,5.304,1877.76,10.745,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1577.0,+108/-77,0.594,9.83,4.47,7.105,1231.21,1.36,✔️,0.0,False,CPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1564.0,+102/-71,0.581,10.37,4.61,6.664,4.79,0.708,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1550.0,+112/-67,0.573,10.93,4.43,6.335,1877.76,1.035,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1528.0,+76/-64,0.5,11.87,7.75,8.851,2721.85,9.311,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1445.0,+85/-69,0.413,15.82,7.11,10.814,2721.85,0.265,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1436.0,+87/-65,0.407,16.3,6.89,10.688,2180.12,1.135,✔️,0.0,False,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1412.0,+61/-52,0.311,17.55,13.05,12.299,328.64,0.774,✔️,0.0,False,CPU +9,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1389.0,+66/-53,0.312,18.83,9.75,11.883,3.84,0.068,✔️,0.0,False,CPU +10,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1389.0,+80/-70,0.313,18.85,10.8,12.863,462.92,0.613,✔️,0.0,False,CPU +11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1388.0,+82/-64,0.349,18.9,6.77,9.224,5890.6,409.986,✔️,0.0,False,GPU +12,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1387.0,+103/-82,0.342,18.95,8.73,11.731,3436.74,0.405,✔️,0.0,False,GPU +13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1376.0,+74/-74,0.321,19.53,11.86,12.014,1043.89,0.478,✔️,0.0,False,CPU +14,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1372.0,+103/-100,0.37,19.82,7.66,11.877,3436.74,8.478,✔️,0.0,False,GPU +15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1362.0,+95/-74,0.321,20.38,10.84,11.826,2180.12,0.117,✔️,0.0,False,GPU +16,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1351.0,+61/-58,0.285,20.97,13.89,12.55,1043.89,0.041,✔️,0.0,False,CPU +17,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1350.0,+76/-81,0.268,21.07,10.02,13.249,462.92,0.108,✔️,0.0,False,CPU +18,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1350.0,+100/-71,0.279,21.07,11.61,14.097,248.47,1.7,➖,0.0,False,GPU +19,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1341.0,+86/-82,0.303,21.55,7.56,11.439,5890.6,39.565,✔️,0.0,False,GPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1338.0,+63/-58,0.239,21.77,17.15,13.479,328.64,0.089,✔️,0.0,False,CPU +21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1333.0,+77/-54,0.229,22.03,13.33,14.266,1579.5,0.428,✔️,0.0,False,CPU +22,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1316.0,+76/-88,0.242,23.03,13.77,13.146,559.77,1.513,✔️,0.0,False,GPU +23,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1314.0,+57/-52,0.186,23.13,15.58,13.373,9.04,1.162,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1301.0,+94/-79,0.247,23.93,15.31,14.371,8.0,0.124,✔️,0.0,False,GPU +25,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1286.0,+71/-59,0.185,24.8,17.62,13.555,2206.58,2.301,✔️,0.0,False,CPU +26,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1273.0,+66/-70,0.174,25.55,17.34,14.801,559.77,0.086,✔️,0.0,False,GPU +27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1271.0,+68/-57,0.158,25.7,12.91,15.04,1579.5,0.05,✔️,0.0,False,CPU +28,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1266.0,+73/-64,0.166,26.0,12.49,16.039,4.79,0.045,✔️,0.0,False,CPU +29,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1249.0,+84/-83,0.202,27.0,10.67,16.519,13.07,3.169,➖,0.0,False,GPU +30,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1230.0,+75/-95,0.188,28.08,10.84,14.643,47.89,39.446,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1226.0,+75/-91,0.15,28.35,20.21,16.576,561.29,4.462,✔️,0.0,False,CPU +32,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1211.0,+88/-82,0.147,29.23,13.65,16.157,1.42,0.118,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1203.0,+75/-66,0.137,29.67,23.42,15.971,2206.58,0.107,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1195.0,+67/-67,0.113,30.17,13.05,16.981,11.15,0.312,✔️,0.0,False,GPU +35,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1173.0,+73/-64,0.084,31.38,25.47,17.929,122.9,0.583,✔️,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1146.0,+68/-82,0.063,32.93,27.11,19.335,171.61,0.552,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1141.0,+81/-116,0.106,33.2,18.63,18.689,561.29,0.236,✔️,0.0,False,CPU +38,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1140.0,+67/-72,0.077,33.22,28.89,17.164,0.93,0.094,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1132.0,+76/-91,0.093,33.7,19.75,19.542,122.9,0.072,✔️,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1110.0,+65/-70,0.022,34.88,31.35,19.987,171.61,0.053,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1042.0,+72/-74,0.038,38.33,33.43,21.241,4.97,0.092,✔️,0.0,False,CPU +42,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1012.0,+110/-134,0.074,39.73,29.98,26.368,0.81,0.115,➖,0.0,False,GPU +43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+76/-99,0.024,40.25,30.25,24.153,0.29,0.031,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),999.0,+75/-104,0.036,40.3,36.56,21.709,2.79,0.259,✔️,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),984.0,+68/-94,0.007,40.97,39.02,24.344,140.66,3.209,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),973.0,+97/-119,0.038,41.4,17.5,27.974,155.53,0.277,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),939.0,+94/-131,0.03,42.78,25.42,28.635,155.53,0.05,✔️,0.0,False,CPU +48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),937.0,+92/-120,0.042,42.83,37.6,26.894,2.71,0.392,✔️,0.0,False,GPU +49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),933.0,+95/-117,0.017,43.0,38.73,25.978,0.18,0.035,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),902.0,+100/-108,0.017,44.13,40.28,30.385,0.8,0.088,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),816.0,+76/-103,0.0,46.8,46.07,30.623,140.66,0.188,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),598.0,+96/-211,0.0,51.0,50.66,43.731,0.15,0.024,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_all/winrate_matrix.png.zip index 21dff1c633778c0e0c7f113a248dffa5c1e1689d..5fe8245d5bb65a427d13f9b31b37996cc46c26d0 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9aa6ca4ec171e2913de5d7aceca21dd3a1013adbd31555704cbdde2447c84609 -size 1822134 +oid sha256:e21a1627e5221de9524b66014751c8aed9650ee273f1b6ed50a46e11c02e092b +size 1970987 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index e62e84aaf26cdff335d76561e9cc4ad6cae1745a..e4bc6079f1df21eab03db3ddfc00b70b093cfe58 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:81450a2529565558aed95b36264f627abe0bcbe8edad4c195ea612e2d223dbee -size 431992 +oid sha256:3151d6568308356ba919acd7baf40723aa8e423dcfd7096022c2b0f97feaf8c9 +size 448322 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip index e3fd8a97ec3ca36d349ef97b2a93865c94b57a38..b7564f87ebe6ed687b4cae4e9f6e54832cf1586b 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:75842db9e1a6a82d9e096f53ca36205440c3a539473f83ea6cbfcc676747b09b -size 927206 +oid sha256:6ed1f691a8cd2d3ef29fa944a18e21e7ceb1631b7fc08f5b38975039fc2b838d +size 905682 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/tuning-impact-elo.png.zip index e7c104317bb1afa443c92566ac07bf58bacff8ec..7a5eea15364cc8ddb29d333ff649e992652edf1d 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9fbcf2445416dccaa6f8874e26a565a6487cdc33ec2372302e129543afc3c9f4 -size 120976 +oid sha256:ccf0f534f3488c2a45774bc586b067a6306d37cacf905c10345e05882d8b5a08 +size 124032 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/website_leaderboard.csv index dbc80d37b1bc450cc0369387db113d169aa7336c..dcb173fc925b36aa4bda7b94b4d6bbc5c4e93eba 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/website_leaderboard.csv @@ -1,54 +1,55 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",2079.0,+240/-68,1.537,0.866,3.33,2.13,310.55,1.824,✔️,0.0,False,CPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",2016.0,+291/-84,2.096,0.837,4.22,2.22,256.14,2.868,✔️,0.0,False,GPU -2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1765.0,+277/-164,4.453,0.621,10.56,3.82,2149.06,6.373,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1743.0,+238/-173,5.875,0.527,11.33,4.11,841.84,11.882,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1734.0,+140/-56,5.562,0.515,11.67,8.77,1655.56,2.483,✔️,0.0,False,GPU -5,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1725.0,+191/-108,7.345,0.458,12.0,8.28,114.48,0.404,✔️,0.0,False,CPU -6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1705.0,+217/-120,6.556,0.517,12.78,7.57,618.23,0.184,✔️,0.0,False,CPU -7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1688.0,+228/-141,7.872,0.45,13.44,7.6,841.84,1.263,✔️,0.0,False,GPU -8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1686.0,+225/-118,7.382,0.477,13.56,5.11,807.72,0.575,✔️,0.0,False,GPU -9,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1678.0,+210/-108,8.481,0.431,13.89,9.94,87.3,0.533,✔️,0.0,False,CPU -10,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1676.0,+166/-104,6.655,0.487,13.94,8.68,618.23,0.031,✔️,0.0,False,CPU -11,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1656.0,+188/-119,9.295,0.422,14.78,10.05,2149.06,0.318,✔️,0.0,False,GPU -12,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1653.0,+159/-92,6.571,0.458,14.94,5.57,2.12,0.043,✔️,0.0,False,CPU -13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1631.0,+170/-121,7.667,0.399,15.89,10.8,114.48,0.058,✔️,0.0,False,CPU -14,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1628.0,+183/-107,8.174,0.39,16.0,9.31,1655.56,0.086,✔️,0.0,False,GPU -15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1596.0,+183/-119,9.425,0.328,17.44,13.37,87.3,0.069,✔️,0.0,False,CPU -16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1594.0,+229/-161,8.03,0.393,17.56,8.34,807.72,0.07,✔️,0.0,False,GPU -17,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1562.0,+182/-145,7.783,0.326,19.0,8.97,2.32,0.889,✔️,0.0,False,GPU -18,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1548.0,+184/-161,9.48,0.295,19.67,9.57,9.96,4.847,✔️,0.0,False,GPU -19,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1488.0,+188/-198,10.588,0.248,22.56,18.43,5.77,0.078,✔️,0.0,False,GPU -20,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1475.0,+188/-199,10.147,0.267,23.22,6.55,0.48,0.047,✔️,0.0,False,CPU -21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1473.0,+155/-148,9.606,0.247,23.33,15.26,559.89,1.156,✔️,0.0,False,GPU -22,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1473.0,+120/-74,9.902,0.175,23.33,19.14,1128.19,1.674,✔️,0.0,False,CPU -23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1457.0,+181/-128,13.211,0.212,24.11,20.3,262.92,0.205,✔️,0.0,False,CPU -24,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1453.0,+174/-154,10.102,0.181,24.33,18.53,4560.06,421.968,✔️,0.0,False,GPU -25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1444.0,+165/-123,12.787,0.166,24.78,17.43,559.89,0.056,✔️,0.0,False,GPU -26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1424.0,+89/-76,11.863,0.079,25.78,23.59,6.55,0.295,✔️,0.0,False,GPU -27,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1390.0,+114/-157,12.364,0.11,27.44,22.3,368.88,2.507,✔️,0.0,False,CPU -28,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1376.0,+211/-169,13.801,0.137,28.17,17.73,4560.06,39.452,✔️,0.0,False,GPU -29,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1347.0,+190/-192,13.488,0.13,29.56,19.19,92.9,0.786,➖,0.0,False,GPU -30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1333.0,+181/-120,16.174,0.073,30.22,27.34,0.92,0.018,✔️,0.0,False,CPU -31,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1324.0,+203/-161,14.896,0.093,30.67,25.33,262.92,0.015,✔️,0.0,False,CPU -32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1317.0,+122/-138,13.481,0.079,31.0,27.5,1128.19,0.055,✔️,0.0,False,CPU -33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1305.0,+130/-166,13.452,0.036,31.56,28.19,40.45,0.331,✔️,0.0,False,CPU -34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1300.0,+57/-72,14.731,0.018,31.78,30.79,43.93,0.204,✔️,0.0,False,CPU -35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1271.0,+112/-155,12.558,0.042,33.11,30.95,0.34,0.052,✔️,0.0,False,CPU -36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1257.0,+53/-66,15.152,0.001,33.78,33.41,43.93,0.018,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1234.0,+108/-127,14.1,0.01,34.78,33.46,9.93,0.224,✔️,0.0,False,GPU -38,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1220.0,+141/-229,18.215,0.025,35.39,31.53,41.8,39.131,✔️,0.0,False,GPU -39,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1198.0,+131/-157,17.05,0.022,36.33,33.39,40.45,0.028,✔️,0.0,False,CPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1185.0,+110/-188,16.514,0.018,36.89,34.91,368.88,0.084,✔️,0.0,False,CPU -41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1144.0,+102/-133,16.247,0.01,38.56,37.02,1.81,0.048,✔️,0.0,False,CPU -42,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1107.0,+174/-278,25.559,0.016,39.94,35.92,43.96,0.099,✔️,0.0,False,CPU -43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1102.0,+121/-204,20.908,0.02,40.11,37.38,514.21,52.341,✔️,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1071.0,+139/-245,19.261,0.025,41.22,38.13,1.61,0.113,✔️,0.0,False,CPU -45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1057.0,+170/-263,26.294,0.002,41.72,38.93,43.96,0.024,✔️,0.0,False,CPU -46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+126/-288,22.447,0.0,43.56,42.51,0.11,0.012,✔️,0.0,False,CPU -47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),898.0,+241/-628,30.801,0.027,46.33,38.97,2.77,0.157,✔️,0.0,False,GPU -48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),884.0,+102/-218,28.037,0.0,46.67,46.31,0.22,0.022,✔️,0.0,False,CPU -49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),860.0,+106/-244,25.027,0.0,47.22,46.92,0.1,0.02,✔️,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),850.0,+64/-159,28.826,0.0,47.44,47.16,514.21,2.49,✔️,0.0,False,CPU -51,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),823.0,+118/-263,33.173,0.0,48.0,47.49,0.2,0.035,➖,0.0,False,GPU -52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),491.0,+97/-381,46.122,0.0,52.11,52.08,0.11,0.089,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",2082.0,+245/-69,0.869,3.33,2.13,1.537,310.55,1.824,✔️,0.0,False,CPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",2019.0,+291/-84,0.84,4.22,2.22,2.096,256.14,2.868,✔️,0.0,False,GPU +2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1765.0,+279/-167,0.623,10.67,3.82,4.453,2149.06,6.373,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1744.0,+244/-174,0.53,11.44,4.11,5.875,841.84,11.882,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1738.0,+140/-54,0.523,11.67,8.77,5.562,1655.56,2.483,✔️,0.0,False,GPU +5,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1729.0,+193/-106,0.465,12.0,8.28,7.345,114.48,0.404,✔️,0.0,False,CPU +6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1709.0,+217/-118,0.524,12.78,7.57,6.556,618.23,0.184,✔️,0.0,False,CPU +7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1690.0,+231/-142,0.454,13.56,7.61,7.872,841.84,1.263,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1687.0,+230/-118,0.48,13.67,5.12,7.382,807.72,0.575,✔️,0.0,False,GPU +9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1680.0,+164/-103,0.494,13.94,8.68,6.655,618.23,0.031,✔️,0.0,False,CPU +10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1679.0,+216/-114,0.437,14.0,9.96,8.481,87.3,0.533,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1658.0,+190/-123,0.426,14.89,10.06,9.295,2149.06,0.318,✔️,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1656.0,+157/-90,0.465,14.94,5.57,6.571,2.12,0.043,✔️,0.0,False,CPU +13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1632.0,+174/-117,0.404,16.0,10.85,7.667,114.48,0.058,✔️,0.0,False,CPU +14,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1630.0,+184/-110,0.399,16.11,9.32,8.174,1655.56,0.086,✔️,0.0,False,GPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1598.0,+189/-122,0.335,17.56,13.39,9.425,87.3,0.069,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1596.0,+233/-167,0.398,17.67,8.34,8.03,807.72,0.07,✔️,0.0,False,GPU +17,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1565.0,+184/-145,0.336,19.11,8.97,7.783,2.32,0.889,✔️,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1549.0,+185/-166,0.301,19.89,9.62,9.48,9.96,4.847,✔️,0.0,False,GPU +19,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1490.0,+193/-200,0.254,22.78,18.48,10.588,5.77,0.078,✔️,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1478.0,+190/-198,0.27,23.33,6.55,10.147,0.48,0.047,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1476.0,+115/-70,0.177,23.44,19.25,9.902,1128.19,1.674,✔️,0.0,False,CPU +22,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1474.0,+152/-152,0.249,23.56,15.3,9.606,559.89,1.156,✔️,0.0,False,GPU +23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1461.0,+183/-126,0.219,24.22,20.34,13.211,262.92,0.205,✔️,0.0,False,CPU +24,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1452.0,+177/-155,0.188,24.67,18.7,10.102,4560.06,421.968,✔️,0.0,False,GPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1450.0,+162/-120,0.167,24.78,17.43,12.787,559.89,0.056,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1426.0,+92/-76,0.09,26.0,23.71,11.863,6.55,0.295,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1393.0,+116/-158,0.111,27.67,22.36,12.364,368.88,2.507,✔️,0.0,False,CPU +28,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1374.0,+210/-170,0.14,28.61,17.82,13.801,4560.06,39.452,✔️,0.0,False,GPU +29,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1351.0,+190/-189,0.133,29.78,19.24,13.488,92.9,0.786,➖,0.0,False,GPU +30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1338.0,+184/-122,0.082,30.44,27.41,16.174,0.92,0.018,✔️,0.0,False,CPU +31,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1331.0,+202/-158,0.1,30.78,25.37,14.896,262.92,0.015,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1319.0,+124/-143,0.083,31.33,27.66,13.481,1128.19,0.055,✔️,0.0,False,CPU +33,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1303.0,+54/-71,0.017,32.11,31.18,14.731,43.93,0.204,✔️,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1303.0,+130/-170,0.038,32.11,28.53,13.452,40.45,0.331,✔️,0.0,False,CPU +35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1273.0,+116/-159,0.048,33.56,31.2,12.558,0.34,0.052,✔️,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1261.0,+54/-66,0.001,34.11,33.7,15.152,43.93,0.018,✔️,0.0,False,CPU +37,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1244.0,+126/-137,0.028,34.89,31.35,17.012,16.58,16.469,➖,0.0,False,GPU +38,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1234.0,+112/-132,0.015,35.33,33.85,14.1,9.93,0.224,✔️,0.0,False,GPU +39,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1220.0,+143/-231,0.027,35.94,31.86,18.215,41.8,39.131,✔️,0.0,False,GPU +40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1194.0,+133/-163,0.022,37.11,33.97,17.05,40.45,0.028,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1183.0,+113/-195,0.02,37.56,35.42,16.514,368.88,0.084,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1146.0,+105/-138,0.013,39.11,37.42,16.247,1.81,0.048,✔️,0.0,False,CPU +43,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1104.0,+175/-278,0.017,40.72,36.53,25.559,43.96,0.099,✔️,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1094.0,+120/-205,0.018,41.11,38.5,20.908,514.21,52.341,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1067.0,+142/-249,0.026,42.11,38.81,19.261,1.61,0.113,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1056.0,+172/-263,0.002,42.5,39.56,26.294,43.96,0.024,✔️,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+128/-282,0.0,44.33,43.22,22.447,0.11,0.012,✔️,0.0,False,CPU +48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),897.0,+236/-634,0.026,47.22,40.25,30.801,2.77,0.157,✔️,0.0,False,GPU +49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),878.0,+102/-219,0.0,47.67,47.32,28.037,0.22,0.022,✔️,0.0,False,CPU +50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),859.0,+112/-244,0.0,48.11,47.78,25.027,0.1,0.02,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),844.0,+63/-158,0.0,48.44,48.17,28.826,514.21,2.49,✔️,0.0,False,CPU +52,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),817.0,+118/-265,0.0,49.0,48.5,33.173,0.2,0.035,➖,0.0,False,GPU +53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),486.0,+98/-384,0.0,53.11,53.08,46.122,0.11,0.089,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/winrate_matrix.png.zip index 92c23bb6a2af41170a11a7bdef81503d0a91dd80..d1688ba2dfd5f9dd92e4b9b2a95f90ded18af857 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:92b6d384d79fa1d0954e106681952232f0c41c64ad197237057eb230808cff08 -size 1490980 +oid sha256:70721559b47e7eae835e7fdd4c5a6c01434ef0cb41d20a062cd2359974c0eac6 +size 1597666 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 0c82f0b250cbcadc9a3817fd76e5e8e2e4cfe4bc..ced7f7e32194556f9e5bb8a9b6fce5403ee60682 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:37b1aa80825290d522b308ae9b2f6505be4c33febf780a766f6718feaddc569e -size 466124 +oid sha256:19e3cce4ba58587e1325b6477e615621ebbf1fb9f846d801b0ef7f7ced5dbeba +size 482862 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip index 76bb300c22481f8c84ae52b812b9d848739bb8d0..d1526cc0f15478d49b787d5ac7cd40359aadcba3 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:289fe235e4e8107b3852e9f1ed2ccdf6b908d4df1e5951b4053e1c5dd9c8d3ad -size 957268 +oid sha256:77126cb6afebfb6bc6f0e5d146b2e4df24f372159a0538e4c1ac36ee3992f9c6 +size 939955 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_small/tuning-impact-elo.png.zip index ae8b7aef0fb173779ad38eed586885de6cc09eeb..e49089b406203411418e07e89861de83a6be1020 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a21daf9761de761552eb0b92ff1bcfaa697d7dd20a61ad5f7bd2e4cf0bc92270 -size 115824 +oid sha256:5dfdd1c61c97ba18bb9dec32225168eb097d51ef01cbd470c3d6bf7d65c0b707 +size 118533 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_binary/datasets_small/website_leaderboard.csv index af456e458133095b3cda755fc8cc6f847ce53b5a..12f1616da29fa82f99974f345d03aca4bccbb7a6 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_small/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_small/website_leaderboard.csv @@ -1,54 +1,55 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1635.0,+100/-58,6.489,0.669,6.71,3.81,5.53,0.55,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1560.0,+146/-88,5.365,0.628,9.14,3.21,2125.17,9.785,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1535.0,+151/-108,5.981,0.607,10.05,3.84,2125.17,0.882,✔️,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1494.0,+147/-96,7.707,0.552,11.76,4.59,613.33,8.472,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1480.0,+91/-88,10.57,0.469,12.38,7.57,2791.97,23.942,✔️,0.0,False,GPU -5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1470.0,+101/-81,9.79,0.465,12.81,9.12,1886.83,1.333,✔️,0.0,False,CPU -6,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1420.0,+121/-84,10.535,0.459,15.26,5.29,4.14,0.557,➖,0.0,False,GPU -7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1405.0,+110/-75,12.26,0.395,16.02,6.49,2791.97,0.544,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1386.0,+108/-84,9.125,0.413,17.05,5.48,6616.87,408.881,✔️,0.0,False,GPU -9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1373.0,+114/-86,12.409,0.361,17.76,8.13,2237.77,1.273,✔️,0.0,False,GPU -10,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1372.0,+126/-89,14.65,0.33,17.81,10.09,298.33,2.036,➖,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1350.0,+95/-92,10.701,0.371,19.0,6.14,6616.87,39.713,✔️,0.0,False,GPU -12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1341.0,+69/-50,14.232,0.247,19.55,15.7,512.11,1.17,✔️,0.0,False,CPU -13,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1316.0,+82/-52,14.463,0.233,20.98,14.65,4.51,0.124,✔️,0.0,False,CPU -14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1314.0,+154/-86,13.061,0.295,21.12,8.66,4618.5,0.426,✔️,0.0,False,GPU -15,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1304.0,+95/-85,15.018,0.223,21.67,12.32,1640.58,0.552,✔️,0.0,False,CPU -16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1301.0,+114/-106,13.757,0.273,21.83,12.55,2237.77,0.147,✔️,0.0,False,GPU -17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1296.0,+91/-80,15.531,0.229,22.17,12.6,700.96,0.743,✔️,0.0,False,CPU -18,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1291.0,+81/-60,14.324,0.211,22.43,13.73,11.09,2.46,✔️,0.0,False,GPU -19,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1283.0,+89/-81,14.659,0.218,22.95,15.45,1103.84,0.559,✔️,0.0,False,CPU -20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1277.0,+88/-86,14.953,0.227,23.29,14.99,536.23,1.738,✔️,0.0,False,GPU -21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1271.0,+78/-80,15.938,0.197,23.67,9.72,700.96,0.203,✔️,0.0,False,CPU -22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1265.0,+125/-91,15.354,0.252,24.02,13.76,4618.5,9.218,✔️,0.0,False,GPU -23,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1264.0,+69/-68,15.511,0.187,24.05,20.71,512.11,0.122,✔️,0.0,False,CPU -24,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1264.0,+82/-66,15.396,0.176,24.1,10.77,1640.58,0.059,✔️,0.0,False,CPU -25,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1258.0,+84/-78,15.373,0.189,24.45,18.99,1103.84,0.051,✔️,0.0,False,CPU -26,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1256.0,+90/-88,16.286,0.185,24.57,10.24,7.16,0.046,✔️,0.0,False,CPU -27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1251.0,+103/-86,16.289,0.231,24.86,14.35,11.3,0.149,✔️,0.0,False,GPU -28,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1245.0,+106/-96,13.383,0.257,25.19,8.48,52.04,39.82,✔️,0.0,False,GPU -29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1236.0,+100/-83,15.412,0.184,25.76,17.18,2535.0,2.546,✔️,0.0,False,CPU -30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1223.0,+86/-93,15.96,0.167,26.55,18.75,536.23,0.097,✔️,0.0,False,GPU -31,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1191.0,+106/-102,18.493,0.153,28.48,10.4,13.39,0.329,✔️,0.0,False,GPU -32,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1183.0,+100/-107,18.662,0.164,28.93,19.1,650.63,5.616,✔️,0.0,False,CPU -33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1176.0,+98/-91,17.319,0.16,29.38,22.22,2535.0,0.112,✔️,0.0,False,CPU -34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1138.0,+87/-95,20.126,0.106,31.6,24.84,206.04,0.946,✔️,0.0,False,CPU -35,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1136.0,+107/-144,19.888,0.145,31.71,17.48,650.63,0.442,✔️,0.0,False,CPU -36,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1127.0,+84/-79,19.028,0.089,32.24,26.22,1.98,0.122,✔️,0.0,False,CPU -37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1117.0,+106/-113,20.888,0.122,32.81,16.91,206.04,0.098,✔️,0.0,False,CPU -38,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1101.0,+75/-85,19.426,0.081,33.69,28.83,2.47,0.168,✔️,0.0,False,CPU -39,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1098.0,+92/-102,21.6,0.079,33.86,26.1,494.48,0.936,✔️,0.0,False,CPU -40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1064.0,+84/-103,22.348,0.029,35.74,30.46,494.48,0.093,✔️,0.0,False,CPU -41,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1053.0,+114/-152,23.728,0.105,36.29,25.95,1.29,0.149,➖,0.0,False,GPU -42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1011.0,+84/-133,23.67,0.049,38.43,32.24,7.76,0.147,✔️,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+105/-146,25.154,0.034,38.98,24.44,0.39,0.041,✔️,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),978.0,+79/-121,23.027,0.039,40.0,36.13,3.48,0.427,✔️,0.0,False,CPU -45,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),947.0,+116/-143,26.643,0.023,41.38,36.36,0.33,0.044,✔️,0.0,False,CPU -46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),947.0,+96/-134,25.487,0.05,41.38,36.81,2.66,0.702,✔️,0.0,False,GPU -47,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),946.0,+68/-106,26.094,0.0,41.43,39.93,89.61,1.627,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),931.0,+108/-155,29.287,0.048,42.07,14.35,280.42,0.382,✔️,0.0,False,CPU -49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),903.0,+115/-164,31.672,0.024,43.19,38.32,1.48,0.127,✔️,0.0,False,CPU -50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),899.0,+113/-150,29.916,0.041,43.33,22.16,280.42,0.072,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),800.0,+87/-136,31.665,0.0,46.62,45.73,89.61,0.08,✔️,0.0,False,CPU -52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),615.0,+107/-184,42.96,0.0,50.52,50.07,0.24,0.02,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1633.0,+102/-59,0.671,6.86,3.84,6.489,5.53,0.55,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1558.0,+149/-88,0.63,9.33,3.24,5.365,2125.17,9.785,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1534.0,+150/-105,0.609,10.24,3.85,5.981,2125.17,0.882,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1491.0,+143/-97,0.551,12.05,4.61,7.707,613.33,8.472,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1478.0,+92/-86,0.469,12.62,7.76,10.57,2791.97,23.942,✔️,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1468.0,+100/-79,0.466,13.1,9.33,9.79,1886.83,1.333,✔️,0.0,False,CPU +6,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1420.0,+121/-84,0.459,15.5,5.32,10.535,4.14,0.557,➖,0.0,False,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1403.0,+106/-74,0.396,16.4,6.88,12.26,2791.97,0.544,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1386.0,+107/-85,0.411,17.33,5.49,9.125,6616.87,408.881,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1371.0,+114/-85,0.36,18.14,8.26,12.409,2237.77,1.273,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1370.0,+126/-89,0.334,18.19,10.23,14.65,298.33,2.036,➖,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1349.0,+93/-90,0.368,19.43,6.21,10.701,6616.87,39.713,✔️,0.0,False,GPU +12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1340.0,+69/-51,0.249,19.98,15.9,14.232,512.11,1.17,✔️,0.0,False,CPU +13,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1315.0,+80/-52,0.233,21.4,15.72,14.463,4.51,0.124,✔️,0.0,False,CPU +14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1314.0,+153/-85,0.299,21.5,8.72,13.061,4618.5,0.426,✔️,0.0,False,GPU +15,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1304.0,+94/-84,0.222,22.1,12.49,15.018,1640.58,0.552,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1300.0,+113/-103,0.275,22.36,12.8,13.757,2237.77,0.147,✔️,0.0,False,GPU +17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1295.0,+91/-79,0.233,22.64,12.82,15.531,700.96,0.743,✔️,0.0,False,CPU +18,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1290.0,+78/-58,0.214,22.95,14.09,14.324,11.09,2.46,✔️,0.0,False,GPU +19,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1284.0,+89/-80,0.219,23.29,16.54,14.659,1103.84,0.559,✔️,0.0,False,CPU +20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1276.0,+88/-87,0.231,23.76,15.23,14.953,536.23,1.738,✔️,0.0,False,GPU +21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1270.0,+77/-78,0.202,24.14,9.89,15.938,700.96,0.203,✔️,0.0,False,CPU +22,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1264.0,+69/-69,0.19,24.52,20.93,15.511,512.11,0.122,✔️,0.0,False,CPU +23,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1264.0,+126/-92,0.253,24.55,13.89,15.354,4618.5,9.218,✔️,0.0,False,GPU +24,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1263.0,+82/-64,0.176,24.57,10.88,15.396,1640.58,0.059,✔️,0.0,False,CPU +25,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1261.0,+108/-115,0.261,24.71,10.69,16.614,8.99,1.724,➖,0.0,False,GPU +26,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1258.0,+84/-74,0.189,24.88,19.23,15.373,1103.84,0.051,✔️,0.0,False,CPU +27,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1254.0,+88/-86,0.186,25.14,10.52,16.286,7.16,0.046,✔️,0.0,False,CPU +28,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1250.0,+101/-84,0.234,25.38,14.6,16.289,11.3,0.149,✔️,0.0,False,GPU +29,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1244.0,+108/-95,0.256,25.76,8.54,13.383,52.04,39.82,✔️,0.0,False,GPU +30,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1234.0,+100/-84,0.187,26.38,17.43,15.412,2535.0,2.546,✔️,0.0,False,CPU +31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1223.0,+87/-94,0.171,27.07,19.06,15.96,536.23,0.097,✔️,0.0,False,GPU +32,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1191.0,+109/-101,0.151,29.05,10.46,18.493,13.39,0.329,✔️,0.0,False,GPU +33,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1180.0,+101/-105,0.164,29.69,20.06,18.662,650.63,5.616,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1174.0,+98/-90,0.163,30.05,22.76,17.319,2535.0,0.112,✔️,0.0,False,CPU +35,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1139.0,+86/-92,0.105,32.17,25.15,20.126,206.04,0.946,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1135.0,+106/-144,0.143,32.43,17.94,19.888,650.63,0.442,✔️,0.0,False,CPU +37,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1128.0,+82/-80,0.094,32.86,26.57,19.028,1.98,0.122,✔️,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1119.0,+109/-111,0.122,33.38,17.03,20.888,206.04,0.098,✔️,0.0,False,CPU +39,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1102.0,+77/-86,0.086,34.36,29.24,19.426,2.47,0.168,✔️,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1099.0,+91/-99,0.079,34.52,26.71,21.6,494.48,0.936,✔️,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1064.0,+84/-98,0.029,36.45,31.45,22.348,494.48,0.093,✔️,0.0,False,CPU +42,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1052.0,+112/-152,0.104,37.1,26.73,23.728,1.29,0.149,➖,0.0,False,GPU +43,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1011.0,+84/-129,0.048,39.24,32.92,23.67,7.76,0.147,✔️,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+104/-146,0.034,39.79,27.41,25.154,0.39,0.041,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),979.0,+79/-121,0.04,40.81,37.02,23.027,3.48,0.427,✔️,0.0,False,CPU +46,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),948.0,+116/-142,0.024,42.19,37.16,26.643,0.33,0.044,✔️,0.0,False,CPU +47,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),948.0,+67/-110,0.001,42.19,40.62,26.094,89.61,1.627,✔️,0.0,False,CPU +48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),947.0,+96/-129,0.049,42.24,37.7,25.487,2.66,0.702,✔️,0.0,False,GPU +49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),930.0,+108/-154,0.048,42.98,14.45,29.287,280.42,0.382,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),904.0,+116/-161,0.024,44.0,38.98,31.672,1.48,0.127,✔️,0.0,False,CPU +51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),900.0,+113/-155,0.041,44.19,22.36,29.916,280.42,0.072,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),801.0,+87/-136,0.0,47.52,46.65,31.665,89.61,0.08,✔️,0.0,False,CPU +53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),614.0,+107/-180,0.0,51.52,51.08,42.96,0.24,0.02,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_small/winrate_matrix.png.zip index 8a0590fd1fb6301747ee7da428415f13b2fd8062..76378bd19e6922cf3159b7dab909ed99fea03e48 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:933a4cd30b519b539a9c8f72be249b9ea9d2b56c98d15b6eb6c5471555e6ca75 -size 1849420 +oid sha256:e78ee1c9d844589ca8d7e79ed4f7989264e23b639f336b7158d42c1c9d9abf0e +size 2004740 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 304c71bdc61ed19882b123373741fc17957f201f..3b3324ad67441d59f73b04e5ff38c888dea84300 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bf77c37a3d2bb57e48e80595f15af3386fcace5f8426d01ccb0c6175c4911b6c -size 495281 +oid sha256:7e5906f9d9bd9fcacb029be792e9bc8371841d09c3f973bcd2774cde9dc1a614 +size 509399 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip index 71f50d1619f4e6ee30cb19d3f72300446e9d56ea..65c0ee56ec08811bcedc0cb9eec1e705df3a539f 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bc8c92b92b7c54b0e1dcfc4088484c23dd7921f61c208379a000cd9d1d95e528 -size 1020703 +oid sha256:60d22153efb6c44060d89b13c939758e6e69c2678461d497c0f73d6bfcbbbc1b +size 990322 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip index eefbe20def7f0bf5192f0ea448dc149832f4ad9d..e4cc7a255d8881799656dceee5126a66c60604cb 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7ea16d6a5bd9fd2347df00f32bfcc883a94ad747eb036a7f9aed8041fdf04cec -size 118622 +oid sha256:800699d2d2878075f669be16e9655e8ef39057f42c6f777b8731965060eb383b +size 124432 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/website_leaderboard.csv index 28185c72fd447069c3a0aa2fa1086235606fd735..ad0d5b94633c5368981d7b7cca58e58c3a012162 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1644.0,+93/-60,6.865,0.637,7.8,5.36,5.25,0.517,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1567.0,+153/-98,5.801,0.586,10.65,4.0,2092.56,8.655,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1556.0,+140/-99,6.297,0.575,11.12,4.48,2092.56,0.867,✔️,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1493.0,+130/-78,8.203,0.506,14.1,5.36,598.6,8.421,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1492.0,+91/-82,10.934,0.441,14.2,8.25,2773.9,21.391,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1476.0,+124/-90,10.58,0.456,15.02,7.12,3361.32,65.579,✔️,0.0,False,GPU -6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1471.0,+85/-74,10.37,0.413,15.3,10.76,1881.65,1.26,✔️,0.0,False,CPU -7,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1443.0,+115/-72,10.913,0.407,16.82,6.69,145.75,4.071,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1426.0,+123/-84,10.972,0.419,17.77,5.81,4.09,0.544,➖,0.0,False,GPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1420.0,+120/-99,12.618,0.371,18.18,7.33,2773.9,0.534,✔️,0.0,False,GPU -10,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1402.0,+115/-88,14.276,0.33,19.25,11.6,292.15,1.882,➖,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1401.0,+118/-76,10.185,0.371,19.27,6.01,8.05,1.864,✔️,0.0,False,GPU -12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1399.0,+132/-82,9.42,0.38,19.4,6.06,6408.95,397.026,✔️,0.0,False,GPU -13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1389.0,+117/-88,12.618,0.351,20.05,8.88,2217.3,1.258,✔️,0.0,False,GPU -14,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1384.0,+124/-88,13.232,0.331,20.32,10.16,3361.32,0.527,✔️,0.0,False,GPU -15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1368.0,+131/-93,10.945,0.347,21.3,7.46,6408.95,39.606,✔️,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1334.0,+58/-48,15.024,0.186,23.52,20.03,457.0,1.064,✔️,0.0,False,CPU -17,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1333.0,+110/-76,13.112,0.293,23.58,9.54,4442.23,0.405,✔️,0.0,False,GPU -18,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1320.0,+84/-79,15.346,0.206,24.4,13.69,1618.79,0.502,✔️,0.0,False,CPU -19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1315.0,+112/-89,13.982,0.269,24.72,14.55,2217.3,0.137,✔️,0.0,False,GPU -20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1315.0,+77/-60,15.224,0.186,24.72,16.46,4.49,0.115,✔️,0.0,False,CPU -21,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1314.0,+80/-69,14.446,0.194,24.8,14.28,10.77,2.16,✔️,0.0,False,GPU -22,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1312.0,+104/-98,15.25,0.295,24.95,7.59,3.89,0.544,✔️,0.0,False,GPU -23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1287.0,+83/-75,16.445,0.178,26.58,19.37,673.88,0.681,✔️,0.0,False,CPU -24,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1284.0,+85/-81,15.452,0.162,26.78,17.38,1103.74,0.549,✔️,0.0,False,CPU -25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1281.0,+88/-92,15.662,0.186,27.0,18.24,516.54,1.513,✔️,0.0,False,GPU -26,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1278.0,+83/-91,16.268,0.17,27.2,10.48,6.77,0.046,✔️,0.0,False,CPU -27,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1276.0,+99/-108,15.707,0.245,27.28,16.43,4442.23,8.478,✔️,0.0,False,GPU -28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1275.0,+83/-79,15.789,0.154,27.4,11.18,1618.79,0.055,✔️,0.0,False,CPU -29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1272.0,+120/-112,16.267,0.23,27.55,17.0,10.18,0.149,✔️,0.0,False,GPU -30,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1268.0,+136/-120,13.464,0.251,27.85,8.65,51.78,39.866,✔️,0.0,False,GPU -31,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1264.0,+74/-64,16.249,0.146,28.1,24.41,457.0,0.111,✔️,0.0,False,CPU -32,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1262.0,+76/-68,16.819,0.15,28.2,11.24,673.88,0.177,✔️,0.0,False,CPU -33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1255.0,+98/-87,15.396,0.183,28.7,20.47,2533.55,2.301,✔️,0.0,False,CPU -34,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1252.0,+87/-54,16.217,0.133,28.9,24.07,1103.74,0.05,✔️,0.0,False,CPU -35,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1227.0,+80/-86,16.637,0.12,30.58,21.9,516.54,0.086,✔️,0.0,False,GPU -36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1212.0,+84/-118,18.467,0.139,31.52,10.49,11.78,0.312,✔️,0.0,False,GPU -37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1204.0,+96/-119,18.721,0.154,32.02,20.34,622.79,5.389,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1196.0,+97/-94,17.128,0.153,32.6,24.44,2533.55,0.11,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1157.0,+98/-138,19.935,0.13,35.05,20.0,622.79,0.398,✔️,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1141.0,+94/-87,20.989,0.076,36.08,28.32,204.47,0.853,✔️,0.0,False,CPU -41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1121.0,+117/-112,21.645,0.109,37.3,18.33,204.47,0.091,✔️,0.0,False,CPU -42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1117.0,+81/-93,20.016,0.043,37.55,33.65,1.88,0.121,✔️,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1094.0,+98/-125,22.532,0.057,38.9,30.42,448.77,0.841,✔️,0.0,False,CPU -44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1091.0,+80/-100,20.402,0.043,39.08,35.18,2.39,0.159,✔️,0.0,False,CPU -45,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1078.0,+103/-162,23.197,0.089,39.85,28.74,1.17,0.148,➖,0.0,False,GPU -46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1065.0,+93/-110,23.044,0.03,40.55,34.95,448.77,0.092,✔️,0.0,False,CPU -47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1033.0,+109/-126,23.554,0.046,42.3,34.74,7.36,0.138,✔️,0.0,False,CPU -48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+98/-164,26.004,0.035,44.0,29.07,0.37,0.04,✔️,0.0,False,CPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),997.0,+107/-123,22.94,0.031,44.15,39.68,3.3,0.395,✔️,0.0,False,CPU -50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),963.0,+65/-111,26.281,0.0,45.75,44.14,89.46,1.579,✔️,0.0,False,CPU -51,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),953.0,+121/-156,26.17,0.044,46.2,41.0,2.66,0.553,✔️,0.0,False,GPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),949.0,+114/-129,29.0,0.05,46.38,14.32,276.0,0.368,✔️,0.0,False,CPU -53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),948.0,+121/-161,27.548,0.024,46.45,40.96,0.29,0.041,✔️,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),926.0,+106/-145,31.115,0.019,47.35,41.98,1.45,0.123,✔️,0.0,False,CPU -55,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),920.0,+122/-145,29.565,0.042,47.6,22.6,276.0,0.065,✔️,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),810.0,+84/-146,31.853,0.0,51.45,50.54,89.46,0.075,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),624.0,+110/-269,43.289,0.0,55.5,55.08,0.22,0.02,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1641.0,+95/-63,0.637,7.95,5.4,6.865,5.25,0.517,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1565.0,+152/-97,0.586,10.85,4.02,5.801,2092.56,8.655,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1554.0,+136/-99,0.574,11.32,4.49,6.297,2092.56,0.867,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1491.0,+127/-79,0.504,14.4,5.38,8.203,598.6,8.421,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1490.0,+91/-80,0.441,14.45,8.46,10.934,2773.9,21.391,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1473.0,+126/-90,0.457,15.32,7.25,10.58,3361.32,65.579,✔️,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1468.0,+84/-74,0.412,15.6,11.04,10.37,1881.65,1.26,✔️,0.0,False,CPU +7,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),1442.0,+115/-71,0.408,17.08,6.7,10.913,145.75,4.071,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1426.0,+122/-82,0.419,18.02,5.83,10.972,4.09,0.544,➖,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1418.0,+117/-96,0.369,18.52,7.84,12.618,2773.9,0.534,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1401.0,+117/-74,0.368,19.52,6.04,10.185,8.05,1.864,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1400.0,+114/-90,0.331,19.6,11.77,14.276,292.15,1.882,➖,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1399.0,+133/-83,0.382,19.7,6.07,9.42,6408.95,397.026,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1387.0,+115/-87,0.349,20.4,9.02,12.618,2217.3,1.258,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1383.0,+125/-87,0.33,20.65,10.22,13.232,3361.32,0.527,✔️,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1367.0,+128/-92,0.346,21.7,7.55,10.945,6408.95,39.606,✔️,0.0,False,GPU +16,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1333.0,+107/-76,0.294,23.92,9.58,13.112,4442.23,0.405,✔️,0.0,False,GPU +17,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1332.0,+58/-49,0.188,23.98,20.28,15.024,457.0,1.064,✔️,0.0,False,CPU +18,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1320.0,+82/-77,0.204,24.8,13.85,15.346,1618.79,0.502,✔️,0.0,False,CPU +19,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1314.0,+76/-59,0.186,25.18,17.84,15.224,4.49,0.115,✔️,0.0,False,CPU +20,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1313.0,+110/-88,0.27,25.22,14.83,13.982,2217.3,0.137,✔️,0.0,False,GPU +21,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1312.0,+79/-70,0.192,25.3,14.64,14.446,10.77,2.16,✔️,0.0,False,GPU +22,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1311.0,+103/-98,0.294,25.4,7.62,15.25,3.89,0.544,✔️,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1286.0,+83/-74,0.18,27.08,19.76,16.445,673.88,0.681,✔️,0.0,False,CPU +24,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1285.0,+84/-79,0.16,27.12,18.82,15.452,1103.74,0.549,✔️,0.0,False,CPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1280.0,+87/-92,0.185,27.5,18.49,15.662,516.54,1.513,✔️,0.0,False,GPU +26,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1276.0,+80/-90,0.169,27.75,10.75,16.268,6.77,0.046,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1276.0,+99/-108,0.246,27.78,16.56,15.707,4442.23,8.478,✔️,0.0,False,GPU +28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1274.0,+84/-80,0.152,27.85,11.27,15.789,1618.79,0.055,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1271.0,+120/-110,0.231,28.05,17.24,16.267,10.18,0.149,✔️,0.0,False,GPU +30,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1268.0,+108/-108,0.22,28.28,11.75,17.243,8.92,1.501,➖,0.0,False,GPU +31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1266.0,+137/-121,0.249,28.4,8.7,13.464,51.78,39.866,✔️,0.0,False,GPU +32,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1263.0,+76/-64,0.148,28.6,24.65,16.249,457.0,0.111,✔️,0.0,False,CPU +33,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1262.0,+74/-66,0.153,28.7,11.39,16.819,673.88,0.177,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1253.0,+97/-86,0.185,29.3,20.73,15.396,2533.55,2.301,✔️,0.0,False,CPU +35,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1252.0,+87/-53,0.131,29.35,24.38,16.217,1103.74,0.05,✔️,0.0,False,CPU +36,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1226.0,+80/-86,0.12,31.12,22.18,16.637,516.54,0.086,✔️,0.0,False,GPU +37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1212.0,+83/-118,0.139,32.08,10.53,18.467,11.78,0.312,✔️,0.0,False,GPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1201.0,+96/-120,0.154,32.78,21.33,18.721,622.79,5.389,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1194.0,+98/-95,0.155,33.25,25.0,17.128,2533.55,0.11,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1156.0,+98/-137,0.129,35.75,20.42,19.935,622.79,0.398,✔️,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1141.0,+95/-85,0.075,36.67,28.64,20.989,204.47,0.853,✔️,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1123.0,+117/-109,0.108,37.85,18.44,21.645,204.47,0.091,✔️,0.0,False,CPU +43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1117.0,+81/-93,0.043,38.2,34.13,20.016,1.88,0.121,✔️,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1094.0,+95/-124,0.057,39.6,31.09,22.532,448.77,0.841,✔️,0.0,False,CPU +45,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1091.0,+80/-102,0.045,39.78,35.69,20.402,2.39,0.159,✔️,0.0,False,CPU +46,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1076.0,+103/-158,0.088,40.65,29.49,23.197,1.17,0.148,➖,0.0,False,GPU +47,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1066.0,+92/-107,0.03,41.25,35.88,23.044,448.77,0.092,✔️,0.0,False,CPU +48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1033.0,+109/-126,0.046,43.1,35.38,23.554,7.36,0.138,✔️,0.0,False,CPU +49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+99/-165,0.036,44.8,31.69,26.004,0.37,0.04,✔️,0.0,False,CPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),997.0,+107/-123,0.031,44.95,40.57,22.94,3.3,0.395,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),965.0,+65/-115,0.001,46.5,44.83,26.281,89.46,1.579,✔️,0.0,False,CPU +52,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),953.0,+119/-157,0.042,47.05,41.88,26.17,2.66,0.553,✔️,0.0,False,GPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),948.0,+120/-158,0.025,47.25,41.74,27.548,0.29,0.041,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),948.0,+116/-129,0.05,47.28,14.4,29.0,276.0,0.368,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),928.0,+104/-142,0.02,48.15,42.62,31.115,1.45,0.123,✔️,0.0,False,CPU +56,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),921.0,+122/-141,0.042,48.45,22.77,29.565,276.0,0.065,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),811.0,+79/-142,0.0,52.35,51.45,31.853,89.46,0.075,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),622.0,+109/-264,0.0,56.5,56.09,43.289,0.22,0.02,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip index 426b162708d9fac178f86a9881d788f58ab54fad..3879411d6b5511edfdbe45ec6c39a8aff389516a 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e9e32ffc19f86c878369ffc80c8b16fc14574683c759c454886d49d73a53318c -size 2250455 +oid sha256:1c0f72abc1b028843ff6b627bdef511d98b976c2cf0d09491dcd928c60fa3ac7 +size 2209336 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index b2acb4bf89d8533d7bba60a5298730af00132696..444a288cd07f36ea61e6bc3e50bad3b8d3252904 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:eacdb7b0487ae55a23dbdb8ec0de89db0fd49b74a82a0c846a78687a308cdaed -size 440786 +oid sha256:9924310c1c5776df41cc380a97ac1da8508e907d1b9f3aa57a3b8d431b7c0f0e +size 453823 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip index 4a68495b9502c40b64a7fadf46b3a5aaf35edf65..b654f330b376981a3a64bc99daccc1fc605ccf1b 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:44e6896b71e279411fc6b24a157220f59a17677d5aab9b6b82dcf84ea596c516 -size 938069 +oid sha256:f1f340ef8c83b6542261d4fe772eccf795beb3cfd6bbe06db343fa528221dc75 +size 926692 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_all/tuning-impact-elo.png.zip index fd4d6a93cf58d8298d102a664aced2420c5371ea..958921819211abbdf70db209663784d83cfdbabc 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:204f3c81d224584d0028ca0480170449e8ebc01e9e6405e6f5e92012cfdb621e -size 124439 +oid sha256:39bcb61b99184dee4f74150240c948633d2013b35ccb2e5ce14f0f482bdf8319 +size 129713 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_classification/datasets_all/website_leaderboard.csv index 7e6ddb985f6fba4b4506a05412f5b09befe6b613..7b24292e85ae507fd7e0a850657e3cc882a25c5c 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_all/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_all/website_leaderboard.csv @@ -1,53 +1,54 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1607.0,+92/-78,6.887,0.669,8.54,3.62,521.51,7.239,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1600.0,+107/-91,4.536,0.647,8.76,2.69,2063.62,9.938,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1569.0,+100/-83,7.055,0.595,9.91,4.18,2063.62,1.087,✔️,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1555.0,+99/-65,8.539,0.579,10.45,4.35,1815.7,1.456,✔️,0.0,False,CPU -4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1550.0,+93/-73,7.411,0.571,10.62,4.56,5.62,0.771,✔️,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1508.0,+67/-58,10.213,0.49,12.42,6.88,2773.9,16.363,✔️,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1437.0,+57/-53,11.863,0.412,15.8,7.08,2773.9,0.39,✔️,0.0,False,GPU -7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1429.0,+76/-65,12.341,0.399,16.24,7.49,2461.81,1.283,✔️,0.0,False,GPU -8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1413.0,+61/-47,13.098,0.315,17.04,12.62,391.23,1.243,✔️,0.0,False,CPU -9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1386.0,+73/-54,12.614,0.335,18.47,11.49,1326.58,0.531,✔️,0.0,False,CPU -10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1386.0,+99/-64,11.346,0.355,18.5,7.28,6067.13,404.225,✔️,0.0,False,GPU -11,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1380.0,+74/-69,13.691,0.299,18.8,11.7,682.62,0.971,✔️,0.0,False,CPU -12,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1367.0,+70/-78,13.123,0.313,19.53,9.78,4853.07,0.473,✔️,0.0,False,GPU -13,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1367.0,+89/-64,13.311,0.33,19.57,10.85,2461.81,0.138,✔️,0.0,False,GPU -14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1366.0,+70/-60,13.149,0.3,19.63,13.51,1326.58,0.051,✔️,0.0,False,CPU -15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1365.0,+51/-53,13.258,0.279,19.66,10.49,5.74,0.107,✔️,0.0,False,CPU -16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1362.0,+100/-92,13.165,0.346,19.84,8.61,4853.07,9.386,✔️,0.0,False,GPU -17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1351.0,+66/-53,14.046,0.256,20.42,15.26,391.23,0.186,✔️,0.0,False,CPU -18,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1348.0,+71/-68,13.945,0.265,20.61,10.93,682.62,0.183,✔️,0.0,False,CPU -19,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1334.0,+77/-75,13.381,0.305,21.41,7.09,6067.13,39.565,✔️,0.0,False,GPU -20,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1329.0,+71/-62,15.718,0.257,21.66,12.57,250.94,2.036,➖,0.0,False,GPU -21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1324.0,+68/-62,14.425,0.259,21.95,11.98,1053.72,2.067,✔️,0.0,False,GPU -22,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1322.0,+66/-54,15.486,0.217,22.05,13.58,2399.23,0.558,✔️,0.0,False,CPU -23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1299.0,+80/-64,15.974,0.243,23.39,14.93,10.31,0.136,✔️,0.0,False,GPU -24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1277.0,+48/-53,15.239,0.174,24.68,18.36,2480.32,2.175,✔️,0.0,False,CPU -25,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1274.0,+67/-49,15.84,0.168,24.82,16.77,10.77,1.787,✔️,0.0,False,GPU -26,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1272.0,+74/-59,16.074,0.191,24.93,12.06,1053.72,0.105,✔️,0.0,False,GPU -27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1260.0,+61/-53,16.415,0.157,25.68,14.05,2399.23,0.056,✔️,0.0,False,CPU -28,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1256.0,+63/-53,17.309,0.159,25.89,13.57,6.77,0.052,✔️,0.0,False,CPU -29,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1220.0,+82/-74,16.595,0.195,27.93,10.63,47.89,39.446,✔️,0.0,False,GPU -30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1219.0,+71/-90,18.456,0.161,28.04,18.88,622.79,5.037,✔️,0.0,False,CPU -31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1205.0,+67/-61,17.532,0.137,28.82,22.17,2480.32,0.117,✔️,0.0,False,CPU -32,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1203.0,+69/-70,17.049,0.129,28.92,15.16,1.88,0.121,✔️,0.0,False,CPU -33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1196.0,+66/-58,18.674,0.119,29.29,20.81,197.16,0.721,✔️,0.0,False,CPU -34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1172.0,+63/-66,19.122,0.099,30.66,14.84,13.35,0.353,✔️,0.0,False,GPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1172.0,+67/-77,19.605,0.106,30.68,21.08,327.31,0.708,✔️,0.0,False,CPU -36,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1159.0,+66/-81,19.975,0.118,31.38,17.52,197.16,0.083,✔️,0.0,False,CPU -37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1150.0,+60/-62,18.358,0.087,31.88,27.07,2.25,0.159,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1131.0,+73/-82,20.473,0.108,32.89,19.44,622.79,0.346,✔️,0.0,False,CPU -39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1128.0,+59/-63,20.441,0.055,33.04,25.79,327.31,0.085,✔️,0.0,False,CPU -40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1028.0,+64/-78,23.172,0.035,38.03,33.9,5.73,0.148,✔️,0.0,False,CPU -41,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+68/-81,26.523,0.02,39.28,29.08,0.37,0.038,✔️,0.0,False,CPU -42,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),990.0,+78/-108,27.289,0.033,39.71,34.47,140.66,2.876,✔️,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),986.0,+71/-68,24.378,0.04,39.87,35.96,3.12,0.334,✔️,0.0,False,CPU -44,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),973.0,+91/-93,27.31,0.053,40.42,34.66,3.48,0.677,✔️,0.0,False,GPU -45,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),972.0,+86/-141,29.506,0.063,40.45,31.52,1.04,0.192,➖,0.0,False,GPU -46,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),965.0,+89/-97,30.755,0.03,40.75,19.72,256.15,0.36,✔️,0.0,False,CPU -47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),932.0,+92/-106,31.478,0.023,42.03,27.39,256.15,0.07,✔️,0.0,False,CPU -48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),926.0,+80/-103,29.05,0.016,42.24,38.26,0.24,0.037,✔️,0.0,False,CPU -49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),871.0,+93/-114,34.085,0.013,44.11,40.88,1.32,0.117,✔️,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),809.0,+92/-127,34.791,0.013,45.89,43.66,140.66,0.188,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),553.0,+116/-146,49.426,0.0,50.42,50.13,0.18,0.028,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1602.0,+92/-79,0.666,8.75,3.64,6.92,521.51,7.239,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1598.0,+110/-93,0.646,8.89,2.7,4.568,2063.62,9.938,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1566.0,+100/-81,0.594,10.07,4.22,7.088,2063.62,1.087,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1551.0,+98/-64,0.575,10.68,4.4,8.572,1815.7,1.456,✔️,0.0,False,CPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1548.0,+91/-73,0.571,10.78,4.69,7.444,5.62,0.771,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1505.0,+67/-56,0.485,12.63,7.05,10.247,2773.9,16.363,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1434.0,+57/-54,0.406,16.14,7.86,11.898,2773.9,0.39,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1426.0,+79/-62,0.392,16.55,7.59,12.375,2461.81,1.283,✔️,0.0,False,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1411.0,+62/-48,0.315,17.38,12.76,13.13,391.23,1.243,✔️,0.0,False,CPU +9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1386.0,+72/-53,0.333,18.76,11.98,12.648,1326.58,0.531,✔️,0.0,False,CPU +10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1384.0,+98/-64,0.354,18.84,7.31,11.376,6067.13,404.225,✔️,0.0,False,GPU +11,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1378.0,+74/-69,0.3,19.17,11.86,13.724,682.62,0.971,✔️,0.0,False,CPU +12,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1366.0,+72/-78,0.313,19.84,9.84,13.153,4853.07,0.473,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1364.0,+90/-64,0.324,19.96,10.99,13.345,2461.81,0.138,✔️,0.0,False,GPU +14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1364.0,+70/-58,0.298,19.97,13.66,13.182,1326.58,0.051,✔️,0.0,False,CPU +15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1364.0,+51/-54,0.276,20.0,11.07,13.292,5.74,0.107,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1359.0,+98/-90,0.343,20.26,8.66,13.197,4853.07,9.386,✔️,0.0,False,GPU +17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1350.0,+66/-54,0.256,20.79,15.43,14.078,391.23,0.186,✔️,0.0,False,CPU +18,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1346.0,+73/-66,0.266,21.0,11.11,13.977,682.62,0.183,✔️,0.0,False,CPU +19,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1331.0,+77/-75,0.303,21.86,7.16,13.412,6067.13,39.565,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1328.0,+70/-61,0.257,22.05,12.75,15.749,250.94,2.036,➖,0.0,False,GPU +21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1322.0,+65/-55,0.217,22.39,13.73,15.518,2399.23,0.558,✔️,0.0,False,CPU +22,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1322.0,+69/-61,0.256,22.39,12.17,14.456,1053.72,2.067,✔️,0.0,False,GPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1297.0,+78/-62,0.245,23.87,15.15,16.006,10.31,0.136,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1275.0,+47/-52,0.173,25.18,18.62,15.271,2480.32,2.175,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1273.0,+64/-49,0.166,25.32,17.2,15.874,10.77,1.787,✔️,0.0,False,GPU +26,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1272.0,+74/-57,0.189,25.36,12.24,16.105,1053.72,0.105,✔️,0.0,False,GPU +27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1259.0,+62/-52,0.156,26.11,14.21,16.447,2399.23,0.056,✔️,0.0,False,CPU +28,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1255.0,+69/-95,0.238,26.37,9.86,17.392,13.62,1.985,➖,0.0,False,GPU +29,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1254.0,+62/-52,0.159,26.39,13.96,17.343,6.77,0.052,✔️,0.0,False,CPU +30,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1219.0,+81/-75,0.194,28.49,10.7,16.625,47.89,39.446,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1216.0,+71/-90,0.156,28.64,19.66,18.487,622.79,5.037,✔️,0.0,False,CPU +32,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1203.0,+70/-69,0.13,29.39,15.26,17.081,1.88,0.121,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1203.0,+67/-61,0.135,29.39,22.56,17.563,2480.32,0.117,✔️,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1196.0,+66/-59,0.118,29.84,21.12,18.704,197.16,0.721,✔️,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1171.0,+65/-75,0.106,31.24,22.0,19.637,327.31,0.708,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1171.0,+64/-66,0.093,31.24,14.95,19.151,13.35,0.353,✔️,0.0,False,GPU +37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1158.0,+65/-78,0.119,31.99,17.81,20.005,197.16,0.083,✔️,0.0,False,CPU +38,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1149.0,+60/-61,0.09,32.49,27.5,18.389,2.25,0.159,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1129.0,+71/-81,0.104,33.61,19.97,20.501,622.79,0.346,✔️,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1129.0,+56/-60,0.055,33.62,26.88,20.473,327.31,0.085,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1028.0,+63/-78,0.03,38.76,34.54,23.204,5.73,0.148,✔️,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+70/-82,0.019,40.04,31.51,26.555,0.37,0.038,✔️,0.0,False,CPU +43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),990.0,+77/-106,0.033,40.5,35.08,27.319,140.66,2.876,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),985.0,+69/-67,0.035,40.71,36.86,24.408,3.12,0.334,✔️,0.0,False,CPU +45,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),972.0,+89/-94,0.049,41.24,35.54,27.341,3.48,0.677,✔️,0.0,False,GPU +46,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),970.0,+86/-137,0.058,41.32,32.45,29.537,1.04,0.192,➖,0.0,False,GPU +47,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),963.0,+88/-99,0.03,41.62,19.92,30.785,256.15,0.36,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),931.0,+92/-103,0.023,42.87,27.74,31.508,256.15,0.07,✔️,0.0,False,CPU +49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),926.0,+81/-105,0.016,43.05,39.04,29.08,0.24,0.037,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),871.0,+92/-114,0.014,44.97,41.66,34.116,1.32,0.117,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),809.0,+91/-122,0.013,46.79,44.48,34.82,140.66,0.188,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),550.0,+115/-146,0.0,51.42,51.13,49.457,0.18,0.028,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_all/winrate_matrix.png.zip index c4bded7ff986cabde8fd0fdee4fa2782d6fe9a78..d64af4084abfcee1f7466ff70d002975b24caa95 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e136b0209df5c0b9145ae0846786c898a888cf0d826f373f0507909688a51b16 -size 1915089 +oid sha256:7b89b2e1ad7be8a971b4ec5cbad81725ef98e14ecde4f6c05f9bffff58e6be82 +size 2067888 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index 4743aa9a7d6ce67f03dae026ee55e6274fbaf709..4f1a15507d853e3dbb837f33f7627c484acf0cbf 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b61bf81a7d6a5629eec57fc90ba2f1f761e92d909be067339daae69b39bfd830 -size 425862 +oid sha256:7b0c16c4eb3b1b808186638beeb32e25d418b63d43c506587435ea5fdd5d00fa +size 444532 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip index fe4ebfc08af57daad7b07fa61afcb2a19588b101..71b9a51305e744e366c0eb200a6f9e654cddac3d 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8432010802d12587598008f141a0d5d47a8e8cf02da82317b3b7ae2af405c0b8 -size 971412 +oid sha256:3a7c78030bd8eb75f203076b0a537ed8437d4dce381cc34a91b812b6fe7eb833 +size 939508 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/tuning-impact-elo.png.zip index 05c60a838736f9301732445641c562450c87a3ef..8da55f4a9c314b4228cd20847e5fa52061b67367 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c88396922b5f88c69931f0667c80fdc584027d9e39cc5b661e0e9a15d9517632 -size 118671 +oid sha256:5686c1281abc8be98487b54d6cafa970ea199ab3021c7f05b3063ceadf958fc8 +size 123544 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/website_leaderboard.csv index c065ec55b4444ae31c21a8aa6d919d58edc19f7c..12026b8b58621f69a8270fa44106b7902e52c4c2 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/website_leaderboard.csv @@ -1,54 +1,55 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",2043.0,+165/-76,1.556,0.873,3.2,2.12,307.34,2.186,✔️,0.0,False,CPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1977.0,+218/-61,2.107,0.845,4.1,2.28,266.49,2.963,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1724.0,+267/-141,5.287,0.574,10.3,3.13,780.95,11.64,✔️,0.0,False,GPU -3,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1699.0,+243/-163,5.346,0.601,11.2,4.14,2335.82,14.145,✔️,0.0,False,GPU -4,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1675.0,+165/-110,7.754,0.463,12.1,8.59,202.53,0.43,✔️,0.0,False,CPU -5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1664.0,+245/-124,7.388,0.494,12.5,6.97,780.95,1.493,✔️,0.0,False,GPU -6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1659.0,+175/-97,7.039,0.517,12.7,7.86,688.25,0.192,✔️,0.0,False,CPU -7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1647.0,+189/-83,8.663,0.444,13.2,9.54,91.02,0.568,✔️,0.0,False,CPU -8,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1638.0,+171/-87,7.113,0.491,13.55,8.8,688.25,0.036,✔️,0.0,False,CPU -9,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1632.0,+128/-91,7.465,0.463,13.8,9.46,1635.73,2.146,✔️,0.0,False,GPU -10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1604.0,+180/-105,8.84,0.429,15.0,5.57,1400.65,0.571,✔️,0.0,False,GPU -11,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1603.0,+136/-69,7.228,0.455,15.05,5.96,2.2,0.044,✔️,0.0,False,CPU -12,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1599.0,+175/-116,9.77,0.418,15.2,10.54,2335.82,0.502,✔️,0.0,False,GPU -13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1588.0,+183/-107,8.072,0.409,15.7,11.05,202.53,0.055,✔️,0.0,False,CPU -14,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1570.0,+182/-110,9.516,0.352,16.5,12.53,91.02,0.074,✔️,0.0,False,CPU -15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1542.0,+159/-111,9.875,0.351,17.8,10.04,1635.73,0.082,✔️,0.0,False,GPU -16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1522.0,+209/-168,9.432,0.353,18.7,8.98,1400.65,0.065,✔️,0.0,False,GPU -17,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1514.0,+194/-127,8.469,0.329,19.1,9.49,2.3,0.818,✔️,0.0,False,GPU -18,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1499.0,+177/-140,10.117,0.294,19.8,10.12,8.46,3.879,✔️,0.0,False,GPU -19,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1450.0,+204/-130,12.904,0.249,22.2,15.55,899.24,0.221,✔️,0.0,False,CPU -20,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1446.0,+233/-158,10.304,0.29,22.4,6.94,0.47,0.047,✔️,0.0,False,CPU -21,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1420.0,+113/-70,11.014,0.158,23.7,19.71,1128.73,1.573,✔️,0.0,False,CPU -22,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1412.0,+189/-172,12.709,0.223,24.1,19.43,5.79,0.084,✔️,0.0,False,GPU -23,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1408.0,+143/-135,11.095,0.169,24.3,18.96,4546.1,406.11,✔️,0.0,False,GPU -24,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1402.0,+158/-144,11.567,0.222,24.6,16.19,713.0,2.317,✔️,0.0,False,GPU -25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1375.0,+166/-140,14.592,0.149,26.0,18.4,713.0,0.155,✔️,0.0,False,GPU -26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1349.0,+96/-101,14.558,0.071,27.3,24.64,6.53,0.294,✔️,0.0,False,GPU -27,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1326.0,+165/-163,14.827,0.123,28.45,18.52,4546.1,39.323,✔️,0.0,False,GPU -28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1321.0,+172/-131,14.536,0.136,28.7,22.41,899.24,0.015,✔️,0.0,False,CPU -29,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1315.0,+157/-123,15.96,0.104,29.0,25.99,1.95,0.02,✔️,0.0,False,CPU -30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1315.0,+106/-156,15.425,0.099,29.0,23.43,387.73,2.446,✔️,0.0,False,CPU -31,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1311.0,+135/-106,14.282,0.073,29.2,21.79,52.7,0.202,✔️,0.0,False,CPU -32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1287.0,+118/-103,14.195,0.073,30.4,27.23,1128.73,0.054,✔️,0.0,False,CPU -33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1283.0,+145/-152,13.773,0.057,30.6,27.42,38.59,0.337,✔️,0.0,False,CPU -34,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1283.0,+183/-175,15.553,0.117,30.6,20.24,82.52,0.97,➖,0.0,False,GPU -35,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1268.0,+124/-101,14.704,0.056,31.3,26.28,52.7,0.017,✔️,0.0,False,CPU -36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1225.0,+105/-129,13.946,0.038,33.3,31.31,0.33,0.05,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1212.0,+92/-115,14.785,0.009,33.9,32.53,11.03,0.489,✔️,0.0,False,GPU -38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1187.0,+152/-168,17.259,0.031,35.0,31.95,38.59,0.028,✔️,0.0,False,CPU -39,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1186.0,+136/-139,18.812,0.022,35.05,31.57,41.63,39.259,✔️,0.0,False,GPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1131.0,+103/-160,18.995,0.016,37.4,35.51,387.73,0.089,✔️,0.0,False,CPU -41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1100.0,+94/-110,17.804,0.009,38.6,37.21,1.84,0.047,✔️,0.0,False,CPU -42,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1048.0,+178/-260,28.29,0.014,40.55,36.73,43.41,0.111,✔️,0.0,False,CPU -43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1036.0,+114/-212,24.539,0.018,41.0,38.29,332.4,33.481,✔️,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1018.0,+144/-200,22.564,0.022,41.6,38.72,1.63,0.11,✔️,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+117/-257,22.603,0.0,42.2,40.81,0.11,0.012,✔️,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),998.0,+166/-272,29.261,0.002,42.25,39.61,43.41,0.02,✔️,0.0,False,CPU -47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),838.0,+248/-564,33.498,0.024,46.7,39.85,2.96,0.183,✔️,0.0,False,GPU -48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),834.0,+103/-217,30.86,0.0,46.8,46.47,0.21,0.019,✔️,0.0,False,CPU -49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),829.0,+93/-235,27.581,0.0,46.9,46.61,0.1,0.02,✔️,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),782.0,+66/-184,33.014,0.0,47.9,47.61,332.4,1.631,✔️,0.0,False,CPU -51,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),761.0,+125/-257,35.637,0.0,48.3,47.82,0.18,0.032,➖,0.0,False,GPU -52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),426.0,+114/-399,49.883,0.0,52.2,52.17,0.12,0.094,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",2045.0,+165/-77,0.875,3.2,2.12,1.556,307.34,2.186,✔️,0.0,False,CPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1979.0,+218/-61,0.847,4.1,2.28,2.107,266.49,2.963,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1723.0,+272/-143,0.577,10.4,3.13,5.287,780.95,11.64,✔️,0.0,False,GPU +3,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1698.0,+245/-166,0.605,11.3,4.14,5.346,2335.82,14.145,✔️,0.0,False,GPU +4,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1677.0,+166/-109,0.471,12.1,8.59,7.754,202.53,0.43,✔️,0.0,False,CPU +5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1664.0,+249/-126,0.497,12.6,6.98,7.388,780.95,1.493,✔️,0.0,False,GPU +6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1662.0,+174/-96,0.525,12.7,7.86,7.039,688.25,0.192,✔️,0.0,False,CPU +7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1647.0,+193/-84,0.451,13.3,9.56,8.663,91.02,0.568,✔️,0.0,False,CPU +8,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1641.0,+170/-85,0.499,13.55,8.8,7.113,688.25,0.036,✔️,0.0,False,CPU +9,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1635.0,+132/-90,0.471,13.8,9.46,7.465,1635.73,2.146,✔️,0.0,False,GPU +10,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1605.0,+134/-69,0.464,15.05,5.96,7.228,2.2,0.044,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1604.0,+184/-105,0.432,15.1,5.57,8.84,1400.65,0.571,✔️,0.0,False,GPU +12,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1600.0,+177/-121,0.424,15.3,10.55,9.77,2335.82,0.502,✔️,0.0,False,GPU +13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1588.0,+182/-106,0.415,15.8,11.1,8.072,202.53,0.055,✔️,0.0,False,CPU +14,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1571.0,+186/-115,0.359,16.6,12.55,9.516,91.02,0.074,✔️,0.0,False,CPU +15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1540.0,+159/-113,0.359,18.0,10.06,9.875,1635.73,0.082,✔️,0.0,False,GPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1524.0,+213/-169,0.358,18.8,8.98,9.432,1400.65,0.065,✔️,0.0,False,GPU +17,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1515.0,+192/-129,0.34,19.2,9.5,8.469,2.3,0.818,✔️,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1499.0,+180/-140,0.302,20.0,10.17,10.117,8.46,3.879,✔️,0.0,False,GPU +19,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1452.0,+202/-128,0.256,22.3,15.56,12.904,899.24,0.221,✔️,0.0,False,CPU +20,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1448.0,+235/-158,0.295,22.5,6.94,10.304,0.47,0.047,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1423.0,+111/-68,0.162,23.8,19.82,11.014,1128.73,1.573,✔️,0.0,False,CPU +22,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1411.0,+194/-173,0.229,24.4,19.51,12.709,5.79,0.084,✔️,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1407.0,+147/-136,0.178,24.6,19.12,11.095,4546.1,406.11,✔️,0.0,False,GPU +24,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1402.0,+160/-148,0.224,24.9,16.25,11.567,713.0,2.317,✔️,0.0,False,GPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1378.0,+165/-136,0.15,26.1,18.42,14.592,713.0,0.155,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1349.0,+98/-103,0.081,27.6,24.79,14.558,6.53,0.294,✔️,0.0,False,GPU +27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1326.0,+172/-127,0.143,28.8,22.44,14.536,899.24,0.015,✔️,0.0,False,CPU +28,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1325.0,+166/-164,0.126,28.85,18.61,14.827,4546.1,39.323,✔️,0.0,False,GPU +29,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1318.0,+159/-124,0.114,29.2,26.05,15.96,1.95,0.02,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1316.0,+106/-156,0.1,29.3,23.52,15.425,387.73,2.446,✔️,0.0,False,CPU +31,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1312.0,+132/-101,0.074,29.5,21.96,14.282,52.7,0.202,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1288.0,+119/-104,0.08,30.7,27.37,14.195,1128.73,0.054,✔️,0.0,False,CPU +33,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1284.0,+182/-173,0.119,30.9,20.32,15.553,82.52,0.97,➖,0.0,False,GPU +34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1280.0,+148/-158,0.061,31.1,27.71,13.773,38.59,0.337,✔️,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1270.0,+129/-102,0.058,31.6,26.44,14.704,52.7,0.017,✔️,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1224.0,+109/-131,0.043,33.8,31.62,13.946,0.33,0.05,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1211.0,+95/-120,0.016,34.4,32.86,14.785,11.03,0.489,✔️,0.0,False,GPU +38,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1203.0,+112/-112,0.025,34.8,31.59,17.779,14.93,13.661,➖,0.0,False,GPU +39,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1186.0,+135/-140,0.024,35.55,31.87,18.812,41.63,39.259,✔️,0.0,False,GPU +40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1183.0,+151/-172,0.034,35.7,32.42,17.259,38.59,0.028,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1127.0,+107/-162,0.018,38.1,36.06,18.995,387.73,0.089,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1100.0,+96/-117,0.011,39.2,37.66,17.804,1.84,0.047,✔️,0.0,False,CPU +43,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1044.0,+179/-261,0.015,41.35,37.36,28.29,43.41,0.111,✔️,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1026.0,+114/-211,0.017,42.0,39.4,24.539,332.4,33.481,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1012.0,+144/-202,0.024,42.5,39.43,22.564,1.63,0.11,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+120/-256,0.0,42.9,41.4,22.603,0.11,0.012,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),996.0,+168/-271,0.002,43.05,40.27,29.261,43.41,0.02,✔️,0.0,False,CPU +48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),835.0,+243/-570,0.023,47.6,41.12,33.498,2.96,0.183,✔️,0.0,False,GPU +49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),826.0,+105/-219,0.0,47.8,47.48,30.86,0.21,0.019,✔️,0.0,False,CPU +50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),826.0,+97/-243,0.0,47.8,47.49,27.581,0.1,0.02,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),774.0,+67/-182,0.0,48.9,48.61,33.014,332.4,1.631,✔️,0.0,False,CPU +52,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),753.0,+125/-257,0.0,49.3,48.83,35.637,0.18,0.032,➖,0.0,False,GPU +53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),420.0,+113/-402,0.0,53.2,53.17,49.883,0.12,0.094,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/winrate_matrix.png.zip index 2dd7765d53d1423f0a7bf6b750869018b56caec1..6c3a8d1fc5ae4958010aa1cc761182099394ae2b 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:546c9928519cd3c31b82b8df5f1e54a5014b6f33a76a939129582cd1c3df724a -size 1687839 +oid sha256:534d320f7b6b5677e5cc581fbf4603bdb8129fb2107d315602c73395225913b4 +size 1816354 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 51f829c5651b5c3cc01d1c7e16426a87a6052502..c03ab9582b1612f12dd5e17da72128e113e6e9e2 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4a3b5ccd0756d36b97e1d5dd9f4273d0c132b28525c503d3b8e21024cfb98c83 -size 437874 +oid sha256:e5089a1c80c43d870cfe558651cd57e968dc2e4a9b391ace0e9b7fc0c6024cd6 +size 453077 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip index 9a2b252d794ecd8ea802a9c051d6a2eea8eeadd8..41bdcc51e85e3d08b90f2ccb562b66790341f155 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:54132e5a8639d149b328088aaea5e2bb0e5284e277e5352f6c0d4d5741cb1675 -size 952618 +oid sha256:5cc23e4f435af98c986012d2392dd5b0a4c6f7d69c7a2411ce98a22f62d1dfe4 +size 966678 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_small/tuning-impact-elo.png.zip index 75d4d2717b5362a87a4115f5bf12fe5d477c1aee..5b61c7e3e30a0b5fc5a13e08435e917a05c6f324 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d6c1ffd5f92c24a9003218763cfdb50a058332b5582ebb055d89ce02a81cde7f -size 128773 +oid sha256:0eafde5b3499eaeb22f0ff640cbd2d54c5b7f8342bb2a7d9dd7fb7435860b015 +size 131121 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_classification/datasets_small/website_leaderboard.csv index ffa7dda24d9167cb482dc29d6100582ec592f775..91ad95061bbe44144a2cf2d461a01e8827a09684 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_small/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_small/website_leaderboard.csv @@ -1,54 +1,55 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1610.0,+109/-82,7.306,0.641,7.88,3.92,6.03,0.703,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1592.0,+131/-96,4.54,0.66,8.46,2.69,2279.85,9.666,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1564.0,+130/-89,7.209,0.615,9.45,3.78,2279.85,0.968,✔️,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1535.0,+111/-88,8.868,0.589,10.57,4.69,635.11,8.569,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1487.0,+92/-69,11.474,0.473,12.61,6.55,3486.44,22.867,✔️,0.0,False,GPU -5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1468.0,+96/-69,11.301,0.458,13.5,7.44,2993.17,1.401,✔️,0.0,False,CPU -6,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1456.0,+99/-63,10.565,0.476,14.09,5.46,4.53,0.611,➖,0.0,False,GPU -7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1421.0,+77/-75,12.856,0.404,15.84,7.35,3486.44,1.002,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1394.0,+85/-74,11.685,0.412,17.29,6.14,7432.95,404.225,✔️,0.0,False,GPU -9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1391.0,+83/-79,13.865,0.367,17.43,8.78,3083.36,1.48,✔️,0.0,False,GPU -10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1358.0,+62/-49,14.947,0.257,19.27,15.23,774.51,1.455,✔️,0.0,False,CPU -11,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1356.0,+104/-78,16.037,0.298,19.36,11.43,303.47,2.324,➖,0.0,False,GPU -12,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1349.0,+89/-90,13.112,0.365,19.79,5.93,7432.95,39.797,✔️,0.0,False,GPU -13,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1333.0,+94/-70,14.97,0.3,20.7,12.07,3083.36,0.189,✔️,0.0,False,GPU -14,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1322.0,+76/-65,14.88,0.251,21.36,14.57,1612.87,0.724,✔️,0.0,False,CPU -15,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1313.0,+86/-74,14.576,0.264,21.89,10.05,6286.73,0.473,✔️,0.0,False,GPU -16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1312.0,+82/-73,15.706,0.258,21.93,12.15,1165.19,2.067,✔️,0.0,False,GPU -17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1310.0,+68/-72,16.083,0.223,22.05,13.9,829.32,1.642,✔️,0.0,False,CPU -18,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1308.0,+58/-51,15.686,0.2,22.2,16.21,6.97,0.144,✔️,0.0,False,CPU -19,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1300.0,+67/-46,15.572,0.221,22.66,17.19,1612.87,0.061,✔️,0.0,False,CPU -20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1299.0,+68/-66,15.928,0.21,22.71,18.04,774.51,0.282,✔️,0.0,False,CPU -21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1295.0,+71/-62,16.677,0.195,22.96,13.86,2989.36,0.609,✔️,0.0,False,CPU -22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1290.0,+73/-69,16.308,0.202,23.25,11.14,829.32,0.241,✔️,0.0,False,CPU -23,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1281.0,+92/-81,16.222,0.242,23.75,14.62,6286.73,9.386,✔️,0.0,False,GPU -24,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1275.0,+82/-83,17.407,0.24,24.11,14.13,11.86,0.149,✔️,0.0,False,GPU -25,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1260.0,+70/-66,16.571,0.183,25.0,15.68,14.59,4.035,✔️,0.0,False,GPU -26,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1250.0,+62/-68,17.351,0.156,25.61,12.67,2989.36,0.063,✔️,0.0,False,CPU -27,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1249.0,+77/-84,16.867,0.194,25.7,11.39,1165.19,0.105,✔️,0.0,False,GPU -28,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1247.0,+81/-65,18.065,0.161,25.82,12.12,8.12,0.053,✔️,0.0,False,CPU -29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1243.0,+83/-70,17.009,0.172,26.04,18.42,2894.94,2.42,✔️,0.0,False,CPU -30,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1237.0,+82/-100,16.048,0.255,26.39,8.68,51.78,39.927,✔️,0.0,False,GPU -31,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1198.0,+100/-88,19.791,0.175,28.73,18.61,1425.28,7.355,✔️,0.0,False,CPU -32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1188.0,+89/-73,18.976,0.153,29.32,21.46,2894.94,0.145,✔️,0.0,False,CPU -33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1179.0,+81/-82,20.674,0.139,29.89,19.84,402.38,1.373,✔️,0.0,False,CPU -34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1167.0,+74/-90,20.919,0.119,30.57,12.6,14.22,0.353,✔️,0.0,False,GPU -35,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1156.0,+85/-85,21.194,0.147,31.2,15.48,402.38,0.163,✔️,0.0,False,CPU -36,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1137.0,+68/-74,19.723,0.068,32.32,27.34,2.85,0.19,✔️,0.0,False,CPU -37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1136.0,+77/-96,21.769,0.114,32.36,22.36,562.0,1.155,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1136.0,+86/-93,21.239,0.135,32.39,19.06,1425.28,0.561,✔️,0.0,False,CPU -39,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1132.0,+66/-78,20.192,0.099,32.59,26.88,3.27,0.193,✔️,0.0,False,CPU -40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1092.0,+71/-81,22.751,0.052,34.84,27.16,562.0,0.138,✔️,0.0,False,CPU -41,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1012.0,+100/-117,27.565,0.079,38.93,29.19,2.45,0.519,➖,0.0,False,GPU -42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1010.0,+81/-101,25.349,0.037,39.04,33.86,9.06,0.196,✔️,0.0,False,CPU -43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1001.0,+90/-124,25.341,0.058,39.43,34.33,5.43,0.836,✔️,0.0,False,GPU -44,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+83/-113,28.169,0.026,39.48,26.99,0.46,0.07,✔️,0.0,False,CPU -45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),978.0,+88/-115,28.52,0.037,40.46,34.33,123.01,1.579,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),978.0,+86/-99,25.269,0.038,40.5,36.42,5.03,0.601,✔️,0.0,False,CPU -47,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),945.0,+93/-118,29.809,0.02,41.89,37.28,0.38,0.072,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),941.0,+88/-110,31.884,0.036,42.07,17.14,479.28,0.495,✔️,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),912.0,+89/-121,32.518,0.031,43.2,25.1,479.28,0.098,✔️,0.0,False,CPU -50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),877.0,+103/-132,35.489,0.018,44.46,40.34,2.34,0.132,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),811.0,+102/-144,35.669,0.018,46.54,43.84,123.01,0.092,✔️,0.0,False,CPU -52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),572.0,+114/-198,49.495,0.0,51.14,50.77,0.28,0.024,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1606.0,+106/-81,0.642,8.05,4.04,7.306,6.03,0.703,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1590.0,+133/-96,0.661,8.61,2.7,4.54,2279.85,9.666,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1562.0,+128/-87,0.617,9.62,3.81,7.209,2279.85,0.968,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1530.0,+111/-88,0.589,10.86,4.75,8.868,635.11,8.569,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1484.0,+90/-66,0.472,12.89,6.7,11.474,3486.44,22.867,✔️,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1465.0,+93/-69,0.46,13.82,7.61,11.301,2993.17,1.401,✔️,0.0,False,CPU +6,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1453.0,+97/-64,0.477,14.38,5.59,10.565,4.53,0.611,➖,0.0,False,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1418.0,+75/-73,0.404,16.23,7.75,12.856,3486.44,1.002,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1392.0,+85/-74,0.411,17.64,6.17,11.685,7432.95,404.225,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1388.0,+81/-78,0.365,17.82,8.94,13.865,3083.36,1.48,✔️,0.0,False,GPU +10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1356.0,+62/-48,0.259,19.7,15.45,14.947,774.51,1.455,✔️,0.0,False,CPU +11,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1354.0,+105/-78,0.301,19.79,11.62,16.037,303.47,2.324,➖,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1346.0,+88/-90,0.363,20.25,5.99,13.112,7432.95,39.797,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1330.0,+93/-70,0.3,21.2,12.27,14.97,3083.36,0.189,✔️,0.0,False,GPU +14,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1321.0,+75/-62,0.251,21.75,15.5,14.88,1612.87,0.724,✔️,0.0,False,CPU +15,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1312.0,+84/-75,0.267,22.29,10.13,14.576,6286.73,0.473,✔️,0.0,False,GPU +16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1310.0,+82/-73,0.26,22.43,12.37,15.706,1165.19,2.067,✔️,0.0,False,GPU +17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1308.0,+65/-72,0.226,22.55,14.19,16.083,829.32,1.642,✔️,0.0,False,CPU +18,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1306.0,+57/-50,0.199,22.66,17.28,15.686,6.97,0.144,✔️,0.0,False,CPU +19,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1298.0,+68/-46,0.22,23.12,17.49,15.572,1612.87,0.061,✔️,0.0,False,CPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1297.0,+68/-64,0.211,23.18,18.31,15.928,774.51,0.282,✔️,0.0,False,CPU +21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1294.0,+71/-62,0.193,23.39,14.05,16.677,2989.36,0.609,✔️,0.0,False,CPU +22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1288.0,+71/-66,0.205,23.75,11.36,16.308,829.32,0.241,✔️,0.0,False,CPU +23,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1279.0,+91/-80,0.243,24.29,14.79,16.222,6286.73,9.386,✔️,0.0,False,GPU +24,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1278.0,+92/-87,0.302,24.36,9.44,17.484,13.4,1.64,➖,0.0,False,GPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1273.0,+81/-83,0.242,24.64,14.37,17.407,11.86,0.149,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1258.0,+70/-66,0.185,25.57,16.1,16.571,14.59,4.035,✔️,0.0,False,GPU +27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1249.0,+62/-68,0.155,26.14,12.82,17.351,2989.36,0.063,✔️,0.0,False,CPU +28,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1248.0,+76/-84,0.198,26.23,11.57,16.867,1165.19,0.105,✔️,0.0,False,GPU +29,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1244.0,+80/-64,0.163,26.43,12.45,18.065,8.12,0.053,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1240.0,+83/-72,0.174,26.68,18.7,17.009,2894.94,2.42,✔️,0.0,False,CPU +31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1236.0,+84/-98,0.254,26.96,8.73,16.048,51.78,39.927,✔️,0.0,False,GPU +32,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1195.0,+100/-87,0.174,29.45,19.5,19.791,1425.28,7.355,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1186.0,+88/-73,0.156,30.0,21.91,18.976,2894.94,0.145,✔️,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1178.0,+79/-80,0.138,30.46,20.12,20.674,402.38,1.373,✔️,0.0,False,CPU +35,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1167.0,+74/-88,0.117,31.18,12.69,20.919,14.22,0.353,✔️,0.0,False,GPU +36,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1157.0,+84/-82,0.147,31.77,15.7,21.194,402.38,0.163,✔️,0.0,False,CPU +37,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1137.0,+67/-73,0.071,32.93,27.75,19.723,2.85,0.19,✔️,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1136.0,+76/-96,0.114,33.0,23.35,21.769,562.0,1.155,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1134.0,+86/-92,0.134,33.11,19.64,21.239,1425.28,0.561,✔️,0.0,False,CPU +40,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1132.0,+64/-78,0.103,33.23,27.33,20.192,3.27,0.193,✔️,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1092.0,+68/-79,0.052,35.52,28.46,22.751,562.0,0.138,✔️,0.0,False,CPU +42,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1011.0,+99/-115,0.078,39.75,30.01,27.565,2.45,0.519,➖,0.0,False,GPU +43,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1009.0,+76/-99,0.036,39.82,34.54,25.349,9.06,0.196,✔️,0.0,False,CPU +44,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1001.0,+90/-126,0.057,40.21,35.1,25.341,5.43,0.836,✔️,0.0,False,GPU +45,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+82/-113,0.026,40.27,29.74,28.169,0.46,0.07,✔️,0.0,False,CPU +46,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),980.0,+89/-114,0.038,41.18,34.79,28.52,123.01,1.579,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),977.0,+86/-98,0.038,41.32,37.36,25.269,5.03,0.601,✔️,0.0,False,CPU +48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),946.0,+94/-118,0.021,42.68,38.02,29.809,0.38,0.072,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),940.0,+89/-109,0.036,42.96,17.28,31.884,479.28,0.495,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),912.0,+90/-119,0.031,44.05,25.37,32.518,479.28,0.098,✔️,0.0,False,CPU +51,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),879.0,+104/-132,0.018,45.29,41.04,35.489,2.34,0.132,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),813.0,+101/-141,0.018,47.39,44.6,35.669,123.01,0.092,✔️,0.0,False,CPU +53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),570.0,+114/-197,0.0,52.14,51.78,49.495,0.28,0.024,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_small/winrate_matrix.png.zip index 31da0489d76be3efed0d687f3a93acc16554f3d6..4971554b6bc7503186c9cb9942e1f9df8b45672b 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9a4d933124aa7019f251cbb62e172808d912bf2b147595925824f1121dd78276 -size 2005320 +oid sha256:c72c7fc5c157ac66f36d5cc239dbc89e337cff0156b40af5fafe9af85cd5c747 +size 2151507 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 892d67df8527f538d80b1a69b8cca880dee26bdc..e2b0124cbf868a13ce349f2090b96316bc9d1b52 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fa9ac2cff2a3ff15b63f761c902ed84a6d5bb7f57f670219c09383e64ea3867c -size 481575 +oid sha256:0209a4ab9b3fb7ace5eee9d4cb6ce54a6dae4cbb252dfa8a1df253a4d3107844 +size 496077 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip index 6b071a13f2bb87a17251e466802cbef17cdfc13c..65419a102dd8b6dcb82bd32769c35ab1adc4136f 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ca05b464f55c0a1d5a5fbfaf3d98309387aaa3814f6fc035ee88270d24fb5071 -size 1023492 +oid sha256:54daaf53b3da9f85e8570f54ca5d807014dbb50b8be56194934b8215f5829062 +size 1015404 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip index f95024e4f15a828da284d45e1553c543edaf694a..f6995a58fdd450eb5273b294a7b58868ad6e4911 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1b2ea8e9fdc284d1093061fbad62f378b65118d9b815a19a5a425758003301a0 -size 121062 +oid sha256:fe2f2d7ecb3c17cc4159ef1ed078d24813f636d1a4473ab2fc571608d4bcdceb +size 126122 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/website_leaderboard.csv index 1fe395cd30ca8ffe789f7e6c60ff47aaab3d9097..7864d15319412f0ba8ab42040dc2f659f7abf2d7 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1648.0,+88/-68,7.839,0.635,8.1,5.12,5.78,0.586,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1624.0,+124/-100,4.968,0.653,8.92,2.96,2162.76,8.93,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1601.0,+124/-104,7.727,0.61,9.81,4.39,2162.76,0.894,✔️,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1533.0,+113/-82,9.618,0.548,12.73,5.43,620.85,8.569,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1510.0,+93/-63,12.159,0.463,13.81,6.88,3159.9,21.159,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1509.0,+107/-89,11.53,0.488,13.87,6.57,3361.32,51.669,✔️,0.0,False,GPU -6,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1472.0,+108/-68,12.029,0.432,15.79,7.34,150.4,4.403,✔️,0.0,False,GPU -7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1467.0,+82/-70,12.218,0.41,16.08,8.22,2628.27,1.401,✔️,0.0,False,CPU -8,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1460.0,+100/-66,11.286,0.439,16.44,6.0,4.3,0.584,➖,0.0,False,GPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1446.0,+89/-57,13.535,0.394,17.25,8.14,3159.9,0.646,✔️,0.0,False,GPU -10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1403.0,+84/-80,14.036,0.341,19.75,10.41,3361.32,0.527,✔️,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1403.0,+103/-85,12.433,0.381,19.77,6.66,6883.9,405.331,✔️,0.0,False,GPU -12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1392.0,+106/-73,14.604,0.335,20.42,9.82,2899.02,1.461,✔️,0.0,False,GPU -13,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1390.0,+84/-96,13.131,0.354,20.52,6.2,9.07,2.008,✔️,0.0,False,GPU -14,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1377.0,+96/-83,16.382,0.298,21.31,12.92,303.47,2.098,➖,0.0,False,GPU -15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1360.0,+81/-99,13.867,0.341,22.42,6.69,6883.9,39.606,✔️,0.0,False,GPU -16,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1345.0,+83/-90,15.148,0.316,23.35,7.94,4.28,0.511,✔️,0.0,False,GPU -17,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1345.0,+53/-41,16.138,0.192,23.37,19.01,691.46,1.455,✔️,0.0,False,CPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1336.0,+73/-88,15.115,0.265,23.88,10.86,6020.89,0.454,✔️,0.0,False,GPU -19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1330.0,+102/-70,15.756,0.27,24.33,14.64,2899.02,0.171,✔️,0.0,False,GPU -20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1326.0,+85/-72,16.007,0.213,24.52,16.47,1394.45,0.618,✔️,0.0,False,CPU -21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1308.0,+69/-57,17.318,0.182,25.71,19.95,785.9,1.281,✔️,0.0,False,CPU -22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1306.0,+76/-53,16.868,0.169,25.83,18.07,5.74,0.139,✔️,0.0,False,CPU -23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1305.0,+73/-64,17.587,0.185,25.92,15.16,2399.23,0.565,✔️,0.0,False,CPU -24,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1300.0,+79/-73,16.877,0.2,26.27,18.27,1053.72,1.872,✔️,0.0,False,GPU -25,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1297.0,+91/-84,17.031,0.239,26.42,17.12,6020.89,8.478,✔️,0.0,False,GPU -26,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1293.0,+74/-51,16.785,0.173,26.69,21.51,1394.45,0.053,✔️,0.0,False,CPU -27,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1290.0,+72/-60,17.51,0.166,26.88,12.61,785.9,0.213,✔️,0.0,False,CPU -28,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1288.0,+71/-52,17.102,0.163,27.0,21.23,691.46,0.264,✔️,0.0,False,CPU -29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1285.0,+102/-73,18.07,0.23,27.19,16.18,11.6,0.148,✔️,0.0,False,GPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1284.0,+83/-65,17.108,0.17,27.27,16.3,12.75,3.502,✔️,0.0,False,GPU -31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1257.0,+98/-127,16.756,0.259,29.04,8.67,51.52,40.126,✔️,0.0,False,GPU -32,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1254.0,+90/-93,18.783,0.139,29.23,12.47,7.41,0.052,✔️,0.0,False,CPU -33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1254.0,+60/-73,18.357,0.136,29.27,13.11,2399.23,0.061,✔️,0.0,False,CPU -34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1253.0,+73/-74,17.668,0.169,29.31,21.75,2792.14,2.175,✔️,0.0,False,CPU -35,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1234.0,+85/-76,18.071,0.126,30.56,21.73,1053.72,0.103,✔️,0.0,False,GPU -36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1221.0,+93/-108,20.456,0.17,31.37,19.9,1350.04,6.57,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1191.0,+72/-113,21.041,0.109,33.29,12.56,13.67,0.339,✔️,0.0,False,GPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1183.0,+80/-82,19.614,0.119,33.81,26.42,2792.14,0.133,✔️,0.0,False,CPU -39,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1160.0,+84/-81,22.147,0.086,35.21,27.55,378.34,1.373,✔️,0.0,False,CPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1154.0,+96/-109,21.966,0.126,35.58,21.37,1350.04,0.613,✔️,0.0,False,CPU -41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1137.0,+91/-92,22.595,0.107,36.62,19.58,378.34,0.154,✔️,0.0,False,CPU -42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1128.0,+60/-59,21.116,0.033,37.19,33.98,2.66,0.17,✔️,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1118.0,+107/-106,23.308,0.078,37.77,26.65,527.27,1.155,✔️,0.0,False,CPU -44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1103.0,+69/-69,21.73,0.047,38.63,34.62,3.01,0.177,✔️,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1079.0,+91/-89,24.138,0.045,40.04,32.03,527.27,0.128,✔️,0.0,False,CPU -46,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1029.0,+86/-152,28.036,0.069,42.69,32.05,2.28,0.454,➖,0.0,False,GPU -47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1013.0,+91/-94,26.238,0.036,43.5,36.97,8.93,0.172,✔️,0.0,False,CPU -48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+86/-108,28.762,0.027,44.15,31.42,0.44,0.065,✔️,0.0,False,CPU -49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),999.0,+93/-122,26.76,0.051,44.19,38.38,5.0,0.753,✔️,0.0,False,GPU -50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),990.0,+94/-109,25.606,0.031,44.62,40.08,4.68,0.547,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),959.0,+80/-107,29.843,0.009,46.04,43.24,118.4,1.479,✔️,0.0,False,CPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),956.0,+94/-99,31.197,0.038,46.17,16.98,429.17,0.495,✔️,0.0,False,CPU -53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),946.0,+87/-146,30.28,0.02,46.62,41.57,0.36,0.068,✔️,0.0,False,CPU -54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),930.0,+99/-111,31.807,0.032,47.27,25.54,429.17,0.081,✔️,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),897.0,+108/-120,34.214,0.015,48.58,43.99,2.16,0.129,✔️,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),766.0,+80/-151,37.314,0.0,52.58,51.76,118.4,0.082,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),576.0,+118/-291,49.46,0.0,56.08,55.73,0.27,0.024,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1643.0,+86/-65,0.635,8.29,5.33,7.839,5.78,0.586,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1621.0,+125/-100,0.652,9.08,2.97,4.968,2162.76,8.93,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1597.0,+126/-103,0.609,10.0,4.42,7.727,2162.76,0.894,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1528.0,+113/-80,0.546,13.04,5.48,9.618,620.85,8.569,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1506.0,+86/-60,0.461,14.12,7.02,12.159,3159.9,21.159,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1504.0,+104/-88,0.489,14.21,6.69,11.53,3361.32,51.669,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),1468.0,+101/-67,0.431,16.13,7.41,12.029,150.4,4.403,✔️,0.0,False,GPU +7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1463.0,+80/-69,0.41,16.42,8.4,12.218,2628.27,1.401,✔️,0.0,False,CPU +8,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1457.0,+98/-64,0.439,16.75,6.1,11.286,4.3,0.584,➖,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1441.0,+88/-57,0.391,17.63,8.64,13.535,3159.9,0.646,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1400.0,+84/-79,0.339,20.15,10.7,14.036,3361.32,0.527,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1400.0,+103/-86,0.38,20.15,6.68,12.433,6883.9,405.331,✔️,0.0,False,GPU +12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1389.0,+100/-72,0.332,20.81,9.98,14.604,2899.02,1.461,✔️,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1388.0,+84/-94,0.351,20.87,6.23,13.131,9.07,2.008,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1374.0,+94/-84,0.298,21.73,13.13,16.382,303.47,2.098,➖,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1356.0,+80/-98,0.34,22.88,6.75,13.867,6883.9,39.606,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1342.0,+51/-42,0.194,23.83,19.27,16.138,691.46,1.455,✔️,0.0,False,CPU +17,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1341.0,+82/-91,0.315,23.85,8.22,15.148,4.28,0.511,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1335.0,+77/-86,0.265,24.27,10.91,15.115,6020.89,0.454,✔️,0.0,False,GPU +19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1326.0,+98/-71,0.268,24.83,14.9,15.756,2899.02,0.171,✔️,0.0,False,GPU +20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1325.0,+83/-68,0.21,24.94,17.66,16.007,1394.45,0.618,✔️,0.0,False,CPU +21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1305.0,+68/-56,0.183,26.25,20.41,17.318,785.9,1.281,✔️,0.0,False,CPU +22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1304.0,+74/-52,0.167,26.33,19.44,16.868,5.74,0.139,✔️,0.0,False,CPU +23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1303.0,+73/-63,0.183,26.35,15.34,17.587,2399.23,0.565,✔️,0.0,False,CPU +24,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1298.0,+124/-117,0.282,26.71,9.9,18.367,12.42,1.417,➖,0.0,False,GPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1296.0,+77/-71,0.198,26.81,18.62,16.877,1053.72,1.872,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1294.0,+92/-83,0.238,26.96,17.29,17.031,6020.89,8.478,✔️,0.0,False,GPU +27,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1291.0,+70/-49,0.17,27.19,21.87,16.785,1394.45,0.053,✔️,0.0,False,CPU +28,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1287.0,+71/-58,0.166,27.42,12.81,17.51,785.9,0.213,✔️,0.0,False,CPU +29,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1286.0,+69/-54,0.164,27.5,21.5,17.102,691.46,0.264,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1283.0,+100/-71,0.23,27.73,16.4,18.07,11.6,0.148,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1281.0,+81/-65,0.167,27.85,16.72,17.108,12.75,3.502,✔️,0.0,False,GPU +32,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1254.0,+97/-127,0.258,29.62,8.71,16.756,51.52,40.126,✔️,0.0,False,GPU +33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1252.0,+60/-71,0.134,29.81,13.25,18.357,2399.23,0.061,✔️,0.0,False,CPU +34,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1251.0,+88/-92,0.139,29.85,12.79,18.783,7.41,0.052,✔️,0.0,False,CPU +35,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1249.0,+72/-74,0.169,29.96,22.06,17.668,2792.14,2.175,✔️,0.0,False,CPU +36,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1232.0,+84/-74,0.126,31.13,22.19,18.071,1053.72,0.103,✔️,0.0,False,GPU +37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1217.0,+92/-107,0.17,32.1,20.76,20.456,1350.04,6.57,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1190.0,+71/-115,0.109,33.87,12.63,21.041,13.67,0.339,✔️,0.0,False,GPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1180.0,+80/-84,0.121,34.5,27.01,19.614,2792.14,0.133,✔️,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1159.0,+82/-82,0.085,35.83,27.98,22.147,378.34,1.373,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1152.0,+95/-107,0.124,36.31,21.92,21.966,1350.04,0.613,✔️,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1138.0,+92/-91,0.107,37.19,19.83,22.595,378.34,0.154,✔️,0.0,False,CPU +43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1127.0,+60/-60,0.033,37.85,34.5,21.116,2.66,0.17,✔️,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1117.0,+107/-103,0.077,38.46,27.77,23.308,527.27,1.155,✔️,0.0,False,CPU +45,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1102.0,+70/-70,0.047,39.33,35.23,21.73,3.01,0.177,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1078.0,+88/-89,0.045,40.73,33.27,24.138,527.27,0.128,✔️,0.0,False,CPU +47,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1028.0,+85/-152,0.068,43.5,32.84,28.036,2.28,0.454,➖,0.0,False,GPU +48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1012.0,+90/-94,0.035,44.31,37.65,26.238,8.93,0.172,✔️,0.0,False,CPU +49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+82/-103,0.028,44.92,33.83,28.762,0.44,0.065,✔️,0.0,False,CPU +50,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),998.0,+92/-120,0.049,45.0,39.17,26.76,5.0,0.753,✔️,0.0,False,GPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),990.0,+92/-104,0.031,45.42,41.01,25.606,4.68,0.547,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),961.0,+81/-103,0.009,46.77,43.84,29.843,118.4,1.479,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),954.0,+94/-96,0.038,47.06,17.1,31.197,429.17,0.495,✔️,0.0,False,CPU +54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),947.0,+86/-142,0.02,47.38,42.29,30.28,0.36,0.068,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),930.0,+98/-110,0.033,48.12,25.77,31.807,429.17,0.081,✔️,0.0,False,CPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),898.0,+103/-116,0.015,49.38,44.67,34.214,2.16,0.129,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),768.0,+83/-149,0.0,53.46,52.66,37.314,118.4,0.082,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),574.0,+118/-291,0.0,57.08,56.73,49.46,0.27,0.024,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip index 7e388b6610bb266ed5ca0621a47199775463a075..2a75bf7443cf2a4840f6344329275c50105938f9 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4afa3894e7ed5e2dbc8a8d4b95fe3efbbaf117d85269c826b457149f91312eee -size 2336052 +oid sha256:9260ce735f8dc0a49272e16a407628032e7a560695c025faa55d09c8408db243 +size 2299751 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index 26a1c306e38d539694e8881ffd80d95458aa7c89..ebd3374b45a495d0a2c8a95383608c6a5d965ab9 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9a3b951b5f0d70bbe82960d91ace39266362a513575e5b5735680accad76be86 -size 422738 +oid sha256:fa4337716c8c8a87ebbc4a881a9dacf303679fcbdb172effe5ed66dd8001943c +size 436355 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip index e00989f7ea7d7fbffceb6a81c8ad0c22b5d30983..a58a0a698b895524eb816b04ec985f1497b2742b 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:58eccea595fee2dee016f0b8fde884cf0a11d94000ce6911a50595aa8ab45dce -size 943077 +oid sha256:1ba2691df58666972924d452dd71739635d70fd8f0df252430e7d85da55c8f3e +size 951432 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip index 53e4edaf0dbb078ecc7fa68a89d247761e76a3c2..566ef5bc5fecb18ab2e39a80c2308cbfe229cc68 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c20a4e628f2f36dab93a9b3ea8627450655bdbf58d75cf117c582ccc91e6eb29 -size 116890 +oid sha256:305c91a06274a97b824068e5174feddc2d36d63a4fc4bc93fb140e8122e43276 +size 120684 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/website_leaderboard.csv index a1be6ad3fc77b2f8595469cd1d71ae1e4b9be11a..44a7e24229753b87433c75cb3f08081b5b80c71d 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/website_leaderboard.csv @@ -1,53 +1,54 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1720.0,+746/-199,1.808,0.786,5.62,1.64,2793.65,8.93,✔️,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1694.0,+276/-91,11.082,0.731,6.25,4.56,1008.06,9.121,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1678.0,+370/-169,9.911,0.673,6.69,3.46,2793.65,1.513,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1521.0,+345/-142,10.367,0.537,12.06,4.52,9.88,0.825,✔️,0.0,False,GPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1488.0,+229/-105,14.071,0.507,13.5,4.06,5546.65,3.268,✔️,0.0,False,CPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1454.0,+218/-127,15.484,0.426,15.12,5.15,4334.8,21.159,✔️,0.0,False,GPU -6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1448.0,+204/-96,15.024,0.378,15.38,11.91,3307.58,1.182,✔️,0.0,False,CPU -7,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1441.0,+200/-70,15.552,0.351,15.75,12.54,3307.58,0.14,✔️,0.0,False,CPU -8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1428.0,+142/-91,16.245,0.334,16.38,11.6,1280.01,4.078,✔️,0.0,False,CPU -9,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1421.0,+136/-103,16.321,0.324,16.75,10.98,1280.01,1.051,✔️,0.0,False,CPU -10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1418.0,+206/-131,15.962,0.381,16.88,12.61,4334.8,3.529,✔️,0.0,False,GPU -11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1413.0,+174/-112,18.699,0.339,17.12,12.01,5568.31,1.781,✔️,0.0,False,GPU -12,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1396.0,+154/-116,19.038,0.337,18.0,11.47,5568.31,0.372,✔️,0.0,False,GPU -13,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1394.0,+223/-132,19.449,0.377,18.12,10.3,9411.86,344.009,✔️,0.0,False,GPU -14,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1370.0,+216/-163,19.369,0.313,19.38,8.26,2079.21,3.347,✔️,0.0,False,GPU -15,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1361.0,+134/-89,16.949,0.255,19.88,18.12,2029.77,4.113,✔️,0.0,False,CPU -16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1354.0,+130/-108,16.708,0.262,20.25,18.02,2029.77,1.04,✔️,0.0,False,CPU -17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1333.0,+163/-121,18.145,0.249,21.44,16.78,6684.65,9.59,✔️,0.0,False,GPU -18,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1314.0,+218/-144,20.809,0.304,22.5,5.96,9411.86,39.507,✔️,0.0,False,GPU -19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1308.0,+161/-121,18.488,0.205,22.81,18.67,6684.65,0.752,✔️,0.0,False,GPU -20,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1305.0,+144/-127,22.14,0.236,23.0,14.3,17.09,0.145,✔️,0.0,False,GPU -21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1298.0,+155/-114,20.217,0.172,23.38,15.32,27108.48,0.742,✔️,0.0,False,CPU -22,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1295.0,+210/-158,21.612,0.243,23.56,12.64,728.32,2.441,✔️,0.0,False,CPU -23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1290.0,+104/-78,18.577,0.146,23.88,21.87,43.1,0.254,✔️,0.0,False,CPU -24,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1287.0,+249/-173,20.994,0.245,24.0,5.71,2079.21,0.319,✔️,0.0,False,GPU -25,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1280.0,+230/-206,20.771,0.265,24.38,11.89,729.17,1.833,✔️,0.0,False,CPU -26,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1268.0,+206/-145,21.742,0.213,25.06,12.59,728.32,0.356,✔️,0.0,False,CPU -27,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1267.0,+129/-128,21.941,0.181,25.12,19.54,283.57,4.104,➖,0.0,False,GPU -28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1251.0,+131/-104,21.705,0.125,26.0,23.18,3646.83,2.162,✔️,0.0,False,CPU -29,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1233.0,+119/-130,21.727,0.153,27.0,22.32,27108.48,0.066,✔️,0.0,False,CPU -30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1228.0,+122/-124,22.231,0.131,27.25,24.61,17.86,0.099,✔️,0.0,False,CPU -31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1220.0,+145/-166,23.531,0.124,27.75,19.6,3646.83,0.194,✔️,0.0,False,CPU -32,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1208.0,+209/-184,22.297,0.179,28.38,16.27,729.17,0.325,✔️,0.0,False,CPU -33,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1195.0,+219/-198,25.651,0.186,29.12,17.06,2721.87,12.593,✔️,0.0,False,CPU -34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1195.0,+155/-153,22.982,0.136,29.12,22.73,5.12,0.444,✔️,0.0,False,CPU -35,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1188.0,+255/-173,24.06,0.216,29.5,10.16,46.36,39.812,✔️,0.0,False,GPU -36,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1188.0,+124/-124,20.547,0.069,29.5,26.9,4.93,0.592,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1126.0,+151/-163,25.251,0.09,32.75,27.3,19.12,6.922,✔️,0.0,False,GPU -38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1094.0,+168/-176,27.296,0.095,34.38,25.91,2721.87,1.083,✔️,0.0,False,CPU -39,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1088.0,+162/-219,29.013,0.074,34.62,28.92,8.99,1.372,✔️,0.0,False,GPU -40,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1088.0,+126/-157,27.29,0.022,34.62,32.43,17.24,0.565,✔️,0.0,False,GPU -41,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1012.0,+267/-384,38.476,0.13,38.12,25.29,155.52,2.046,✔️,0.0,False,CPU -42,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+105/-213,35.562,0.0,38.62,36.72,0.74,0.148,✔️,0.0,False,CPU -43,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),975.0,+97/-144,30.565,0.0,39.62,38.73,14.37,0.363,✔️,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),928.0,+173/-268,34.532,0.031,41.38,36.93,8.37,0.659,✔️,0.0,False,CPU -45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),923.0,+84/-249,41.326,0.0,41.56,40.55,909.67,0.609,✔️,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),901.0,+89/-241,42.282,0.0,42.31,41.21,909.67,0.107,✔️,0.0,False,CPU -47,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),895.0,+167/-230,40.712,0.011,42.5,39.66,0.76,0.153,✔️,0.0,False,CPU -48,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),769.0,+281/-1308,50.559,0.063,46.0,38.91,155.52,0.191,✔️,0.0,False,CPU -49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),752.0,+76/-209,41.42,0.0,46.38,46.17,4.67,0.855,➖,0.0,False,GPU -50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),710.0,+102/-321,48.104,0.0,47.25,46.92,4.62,0.135,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-292.0,+145/-1079,70.927,0.0,52.0,52.0,0.52,0.075,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1715.0,+758/-197,0.786,5.62,1.64,1.808,2793.65,8.93,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1680.0,+243/-93,0.731,6.5,4.71,11.082,1008.06,9.121,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1668.0,+346/-169,0.674,6.81,3.59,9.911,2793.65,1.513,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1512.0,+320/-135,0.536,12.31,5.0,10.367,9.88,0.825,✔️,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1478.0,+209/-104,0.504,13.88,4.14,14.071,5546.65,3.268,✔️,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1444.0,+189/-121,0.426,15.5,5.27,15.484,4334.8,21.159,✔️,0.0,False,GPU +6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1436.0,+177/-94,0.377,15.88,12.46,15.024,3307.58,1.182,✔️,0.0,False,CPU +7,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1429.0,+156/-66,0.348,16.25,12.85,15.552,3307.58,0.14,✔️,0.0,False,CPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1419.0,+133/-91,0.328,16.75,11.78,16.245,1280.01,4.078,✔️,0.0,False,CPU +9,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1412.0,+131/-105,0.317,17.12,11.21,16.321,1280.01,1.051,✔️,0.0,False,CPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1407.0,+181/-130,0.379,17.38,12.97,15.962,4334.8,3.529,✔️,0.0,False,GPU +11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1405.0,+158/-112,0.337,17.5,12.35,18.699,5568.31,1.781,✔️,0.0,False,GPU +12,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1388.0,+154/-114,0.335,18.38,11.61,19.038,5568.31,0.372,✔️,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1384.0,+217/-130,0.374,18.62,10.43,19.449,9411.86,344.009,✔️,0.0,False,GPU +14,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1359.0,+209/-169,0.306,20.0,8.48,19.369,2079.21,3.347,✔️,0.0,False,GPU +15,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1352.0,+108/-84,0.252,20.38,18.72,16.949,2029.77,4.113,✔️,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1345.0,+124/-103,0.259,20.75,18.66,16.708,2029.77,1.04,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1325.0,+152/-118,0.243,21.94,17.04,18.145,6684.65,9.59,✔️,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1306.0,+211/-142,0.304,23.0,5.98,20.809,9411.86,39.507,✔️,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1303.0,+153/-115,0.205,23.19,18.85,18.488,6684.65,0.752,✔️,0.0,False,GPU +20,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1296.0,+142/-131,0.237,23.62,14.57,22.14,17.09,0.145,✔️,0.0,False,GPU +21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1294.0,+143/-113,0.171,23.75,15.43,20.217,27108.48,0.742,✔️,0.0,False,CPU +22,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1289.0,+226/-310,0.371,24.0,7.66,20.668,21.84,1.702,➖,0.0,False,GPU +23,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1288.0,+200/-158,0.246,24.06,12.87,21.612,728.32,2.441,✔️,0.0,False,CPU +24,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1283.0,+98/-76,0.142,24.38,22.48,18.577,43.1,0.254,✔️,0.0,False,CPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1278.0,+251/-178,0.245,24.62,5.82,20.994,2079.21,0.319,✔️,0.0,False,GPU +26,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1274.0,+226/-205,0.268,24.88,12.88,20.771,729.17,1.833,✔️,0.0,False,CPU +27,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1262.0,+204/-147,0.217,25.56,13.0,21.742,728.32,0.356,✔️,0.0,False,CPU +28,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1258.0,+121/-127,0.178,25.75,20.19,21.941,283.57,4.104,➖,0.0,False,GPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1243.0,+111/-106,0.129,26.62,23.64,21.705,3646.83,2.162,✔️,0.0,False,CPU +30,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1226.0,+115/-130,0.15,27.62,22.85,21.727,27108.48,0.066,✔️,0.0,False,CPU +31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1221.0,+115/-129,0.134,27.88,24.98,22.231,17.86,0.099,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1212.0,+145/-164,0.128,28.38,19.83,23.531,3646.83,0.194,✔️,0.0,False,CPU +33,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1204.0,+206/-176,0.18,28.88,17.52,22.297,729.17,0.325,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1189.0,+210/-193,0.177,29.75,17.85,25.651,2721.87,12.593,✔️,0.0,False,CPU +35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1189.0,+153/-152,0.135,29.75,23.29,22.982,5.12,0.444,✔️,0.0,False,CPU +36,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1184.0,+114/-119,0.069,30.0,27.34,20.547,4.93,0.592,✔️,0.0,False,CPU +37,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1184.0,+244/-170,0.217,30.0,10.2,24.06,46.36,39.812,✔️,0.0,False,GPU +38,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1120.0,+146/-158,0.09,33.5,28.17,25.251,19.12,6.922,✔️,0.0,False,GPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1088.0,+158/-172,0.094,35.12,27.3,27.296,2721.87,1.083,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1086.0,+118/-150,0.019,35.25,32.98,27.29,17.24,0.565,✔️,0.0,False,GPU +41,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1086.0,+161/-220,0.074,35.25,29.49,29.013,8.99,1.372,✔️,0.0,False,GPU +42,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1012.0,+263/-373,0.132,38.75,25.46,38.476,155.52,2.046,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+102/-189,0.0,39.25,37.31,35.562,0.74,0.148,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),973.0,+93/-146,0.0,40.38,39.42,30.565,14.37,0.363,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),924.0,+158/-247,0.03,42.25,38.04,34.532,8.37,0.659,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),919.0,+76/-227,0.0,42.44,41.48,41.326,909.67,0.609,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),897.0,+81/-222,0.0,43.19,42.14,42.282,909.67,0.107,✔️,0.0,False,CPU +48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),896.0,+166/-233,0.013,43.25,40.23,40.712,0.76,0.153,✔️,0.0,False,CPU +49,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),774.0,+274/-873,0.064,46.75,39.36,50.559,155.52,0.191,✔️,0.0,False,CPU +50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),752.0,+78/-199,0.0,47.25,47.02,41.42,4.67,0.855,➖,0.0,False,GPU +51,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),712.0,+92/-262,0.0,48.12,47.82,48.104,4.62,0.135,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-304.0,+166/-722,0.0,53.0,53.0,70.927,0.52,0.075,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/winrate_matrix.png.zip index 1fb03d9a2a03a62bf82d34a4e8dc2a6800c5f9a7..a356734c90a156f2c5d7aef361f38092edf496df 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9f39ef4c7bf315a01d012103b114904510134a874123ca6904b3a65f8c6320af -size 1446692 +oid sha256:9590c7fce16ec3a5df2dcbd3e41b3174e4939edc569363ed62169e547cd49004 +size 1552025 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index ab51e9d26c14388a434bbd59e9da4f1ef669f0ec..09c1d31e6e9629ff76244757a9862efc59bcce8a 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6eccc6c810acafd9438808198bdad00e6a59783fad05270f9dc318f9e33db035 -size 451105 +oid sha256:daf703111fcc26e08bfba57b11bcfdaf2c91e51188bcb9eaf596707da843df74 +size 465220 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip index daa8197457d202b447e7243878e38088f348574b..4f4802133a5568f37ec4296bc01c7ea9a474928f 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9d910235fd0dcd742fe79457585aa7841148531ce81b233c76654399523a6e13 -size 948654 +oid sha256:ed3ee25988c7e1a338adfc3501f0de30806c73684449e7d91426d85b37cdf3e1 +size 986657 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip index 71ea97517666cfee2471289cd5b0125c2f0f4ba2..4135346756b3086cce75833b61e6d73c353a6667 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4019e3af637e694d1c311d16688b18e33230eeb9a7b9191ebf607827db4d33e0 -size 107975 +oid sha256:d359de06b74f698c116558d93aa2eb2e78ee429962274eaa0ec7defaadfe9be7 +size 113816 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/website_leaderboard.csv index f680a6b8392e345fe931fd78494d7ac947fec6fd..d0fe50aa14cb9ce5a4b243ea5ac30d0a27b22e02 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/website_leaderboard.csv @@ -1,54 +1,55 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),25228.0,+0/-0,0.0,1.0,1.0,1.0,134.59,8.308,✔️,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",23917.0,+0/-0,1.729,0.934,2.0,2.0,276.98,4.897,✔️,0.0,False,CPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",22310.0,+0/-0,2.208,0.915,3.0,3.0,276.84,3.303,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),20483.0,+0/-0,3.024,0.883,4.0,4.0,134.59,2.819,✔️,0.0,False,GPU -4,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),19432.0,+0/-0,10.141,0.576,5.0,5.0,7855.1,0.565,✔️,0.0,False,CPU -5,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),18258.0,+0/-0,10.238,0.572,6.0,6.0,61.47,0.154,✔️,0.0,False,CPU -6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),17510.0,+0/-0,10.303,0.569,7.0,7.0,157.4,4.579,✔️,0.0,False,CPU -7,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),16643.0,+0/-0,10.333,0.567,8.0,8.0,157.4,0.401,✔️,0.0,False,CPU -8,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),15885.0,+0/-0,10.67,0.551,9.0,9.0,61.47,0.015,✔️,0.0,False,CPU -9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),15137.0,+0/-0,11.231,0.525,10.0,10.0,2687.46,0.361,✔️,0.0,False,CPU -10,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),14345.0,+0/-0,11.295,0.522,11.0,11.0,7855.1,0.048,✔️,0.0,False,CPU -11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),13528.0,+0/-0,11.389,0.517,12.0,12.0,2687.46,0.793,✔️,0.0,False,CPU -12,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),12768.0,+0/-0,11.441,0.515,13.0,13.0,456.33,1.171,✔️,0.0,False,CPU -13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),12036.0,+0/-0,11.722,0.501,14.0,14.0,456.33,0.047,✔️,0.0,False,CPU -14,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),11311.0,+0/-0,11.722,0.501,15.0,15.0,0.32,0.047,✔️,0.0,False,CPU -15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),10518.0,+0/-0,13.143,0.432,16.0,16.0,8.22,0.049,✔️,0.0,False,CPU -16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),9807.0,+0/-0,13.383,0.42,17.0,17.0,4378.82,25.595,✔️,0.0,False,GPU -17,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),9111.0,+0/-0,14.026,0.387,18.0,18.0,20.73,0.079,✔️,0.0,False,CPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),8390.0,+0/-0,14.047,0.386,19.0,19.0,4378.82,2.216,✔️,0.0,False,GPU -19,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),7635.0,+0/-0,14.639,0.356,20.0,20.0,1.48,0.535,✔️,0.0,False,GPU -20,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),6953.0,+0/-0,15.857,0.292,21.0,21.0,2.79,1.057,✔️,0.0,False,GPU -21,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),6264.0,+0/-0,16.662,0.249,22.0,22.0,20.88,0.359,✔️,0.0,False,CPU -22,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),5562.0,+0/-0,19.143,0.111,23.0,23.0,20.88,0.026,✔️,0.0,False,CPU -23,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),5007.0,+0/-0,20.028,0.059,24.0,24.0,4390.96,275.957,✔️,0.0,False,GPU -24,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),4306.0,+0/-0,20.621,0.024,25.0,25.0,2341.91,0.027,✔️,0.0,False,CPU -25,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),3678.0,+0/-0,20.955,0.004,26.0,26.0,14.53,1.747,✔️,0.0,False,GPU -26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),3004.0,+0/-0,21.025,0.0,27.0,27.0,2341.91,0.172,✔️,0.0,False,CPU -27,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),2329.0,+0/-0,21.961,0.0,28.0,28.0,1993.57,0.469,✔️,0.0,False,GPU -28,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1701.0,+0/-0,22.049,0.0,29.0,29.0,1993.57,0.055,✔️,0.0,False,GPU -29,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+0/-0,24.001,0.0,30.0,30.0,0.18,0.007,✔️,0.0,False,CPU -30,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),445.0,+0/-0,24.06,0.0,31.0,31.0,4390.96,38.525,✔️,0.0,False,GPU -31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),-256.0,+0/-0,24.187,0.0,32.0,32.0,41.21,39.406,✔️,0.0,False,GPU -32,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),-946.0,+0/-0,24.591,0.0,33.0,33.0,1322.48,0.546,✔️,0.0,False,GPU -33,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),-1628.0,+0/-0,25.183,0.0,34.0,34.0,1322.48,0.055,✔️,0.0,False,GPU -34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),-2382.0,+0/-0,26.439,0.0,35.0,35.0,0.21,0.047,✔️,0.0,False,CPU -35,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),-3104.0,+0/-0,29.215,0.0,36.0,36.0,2009.11,3.767,✔️,0.0,False,GPU -36,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),-3800.0,+0/-0,30.84,0.0,37.0,37.0,2009.11,0.974,✔️,0.0,False,GPU -37,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),-4511.0,+0/-0,31.796,0.0,38.0,38.0,6.84,0.09,✔️,0.0,False,GPU -38,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),-5304.0,+0/-0,31.813,0.0,39.0,39.0,3.34,0.028,✔️,0.0,False,CPU -39,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),-6029.0,+0/-0,34.136,0.0,40.0,40.0,66.06,1.783,➖,0.0,False,GPU -40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),-6760.0,+0/-0,38.819,0.0,41.0,41.0,6.51,0.162,✔️,0.0,False,GPU -41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),-7521.0,+0/-0,41.321,0.0,42.0,42.0,553.48,0.174,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),-8338.0,+0/-0,42.976,0.0,43.0,43.0,553.48,1.636,✔️,0.0,False,CPU -43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),-9130.0,+0/-0,50.562,0.0,44.0,44.0,0.08,0.017,✔️,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),-9878.0,+0/-0,52.296,0.0,45.0,45.0,2.73,0.083,✔️,0.0,False,CPU -45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),-10636.0,+0/-0,52.871,0.0,46.0,46.0,30.19,0.134,✔️,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),-11503.0,+0/-0,55.965,0.0,47.0,47.0,30.19,0.017,✔️,0.0,False,CPU -47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),-12250.0,+0/-0,56.26,0.0,48.0,48.0,0.13,0.015,✔️,0.0,False,CPU -48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),-13425.0,+0/-0,57.211,0.0,49.0,49.0,76.41,2.853,✔️,0.0,False,CPU -49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),-14476.0,+0/-0,57.769,0.0,50.0,50.0,3.14,0.921,✔️,0.0,False,GPU -50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),-16303.0,+0/-0,57.822,0.0,51.0,51.0,0.11,0.023,➖,0.0,False,GPU -51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),-17910.0,+0/-0,70.703,0.0,52.0,52.0,76.41,0.258,✔️,0.0,False,CPU -52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-19221.0,+0/-0,83.729,0.0,53.0,53.0,0.14,0.123,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),25529.0,+0/-0,1.0,1.0,1.0,0.0,134.59,8.308,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",24211.0,+0/-0,0.936,2.0,2.0,1.729,276.98,4.897,✔️,0.0,False,CPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",22594.0,+0/-0,0.918,3.0,3.0,2.208,276.84,3.303,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),20725.0,+0/-0,0.886,4.0,4.0,3.024,134.59,2.819,✔️,0.0,False,GPU +4,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),19652.0,+0/-0,0.588,5.0,5.0,10.141,7855.1,0.565,✔️,0.0,False,CPU +5,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),18457.0,+0/-0,0.583,6.0,6.0,10.238,61.47,0.154,✔️,0.0,False,CPU +6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),17698.0,+0/-0,0.58,7.0,7.0,10.303,157.4,4.579,✔️,0.0,False,CPU +7,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),16836.0,+0/-0,0.579,8.0,8.0,10.333,157.4,0.401,✔️,0.0,False,CPU +8,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),16053.0,+0/-0,0.564,9.0,9.0,10.67,61.47,0.015,✔️,0.0,False,CPU +9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),15299.0,+0/-0,0.538,10.0,10.0,11.231,2687.46,0.361,✔️,0.0,False,CPU +10,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),14521.0,+0/-0,0.535,11.0,11.0,11.295,7855.1,0.048,✔️,0.0,False,CPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),13703.0,+0/-0,0.531,12.0,12.0,11.389,2687.46,0.793,✔️,0.0,False,CPU +12,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),12920.0,+0/-0,0.528,13.0,13.0,11.441,456.33,1.171,✔️,0.0,False,CPU +13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),12173.0,+0/-0,0.515,14.0,14.0,11.722,456.33,0.047,✔️,0.0,False,CPU +14,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),11454.0,+0/-0,0.515,15.0,15.0,11.722,0.32,0.047,✔️,0.0,False,CPU +15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),10661.0,+0/-0,0.447,16.0,16.0,13.143,8.22,0.049,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),9930.0,+0/-0,0.436,17.0,17.0,13.383,4378.82,25.595,✔️,0.0,False,GPU +17,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),9235.0,+0/-0,0.404,18.0,18.0,14.026,20.73,0.079,✔️,0.0,False,CPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),8520.0,+0/-0,0.403,19.0,19.0,14.047,4378.82,2.216,✔️,0.0,False,GPU +19,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),7765.0,+0/-0,0.374,20.0,20.0,14.639,1.48,0.535,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),7045.0,+0/-0,0.312,21.0,21.0,15.857,2.79,1.057,✔️,0.0,False,GPU +21,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),6363.0,+0/-0,0.27,22.0,22.0,16.662,20.88,0.359,✔️,0.0,False,CPU +22,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),5667.0,+0/-0,0.135,23.0,23.0,19.143,20.88,0.026,✔️,0.0,False,CPU +23,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),4962.0,+0/-0,0.085,24.0,24.0,20.028,4390.96,275.957,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),4493.0,+0/-0,0.051,25.0,25.0,20.621,2341.91,0.027,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),3732.0,+0/-0,0.032,26.0,26.0,20.955,14.53,1.747,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),3107.0,+0/-0,0.028,27.0,27.0,21.025,2341.91,0.172,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),2386.0,+0/-0,0.0,28.0,28.0,21.961,1993.57,0.469,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1761.0,+0/-0,0.0,29.0,29.0,22.049,1993.57,0.055,✔️,0.0,False,GPU +29,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+0/-0,0.0,30.0,30.0,24.001,0.18,0.007,✔️,0.0,False,CPU +30,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),531.0,+0/-0,0.0,31.0,31.0,24.06,4390.96,38.525,✔️,0.0,False,GPU +31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),-174.0,+0/-0,0.0,32.0,32.0,24.187,41.21,39.406,✔️,0.0,False,GPU +32,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),-870.0,+0/-0,0.0,33.0,33.0,24.591,1322.48,0.546,✔️,0.0,False,GPU +33,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),-1552.0,+0/-0,0.0,34.0,34.0,24.684,9.89,9.237,➖,0.0,False,GPU +34,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),-2272.0,+0/-0,0.0,35.0,35.0,25.183,1322.48,0.055,✔️,0.0,False,GPU +35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),-3027.0,+0/-0,0.0,36.0,36.0,26.439,0.21,0.047,✔️,0.0,False,CPU +36,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),-3742.0,+0/-0,0.0,37.0,37.0,29.215,2009.11,3.767,✔️,0.0,False,GPU +37,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),-4436.0,+0/-0,0.0,38.0,38.0,30.84,2009.11,0.974,✔️,0.0,False,GPU +38,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),-5168.0,+0/-0,0.0,39.0,39.0,31.796,6.84,0.09,✔️,0.0,False,GPU +39,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),-5961.0,+0/-0,0.0,40.0,40.0,31.813,3.34,0.028,✔️,0.0,False,CPU +40,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),-6680.0,+0/-0,0.0,41.0,41.0,34.136,66.06,1.783,➖,0.0,False,GPU +41,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),-7427.0,+0/-0,0.0,42.0,42.0,38.819,6.51,0.162,✔️,0.0,False,GPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),-8210.0,+0/-0,0.0,43.0,43.0,41.321,553.48,0.174,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),-9028.0,+0/-0,0.0,44.0,44.0,42.976,553.48,1.636,✔️,0.0,False,CPU +44,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),-9806.0,+0/-0,0.0,45.0,45.0,50.562,0.08,0.017,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),-10560.0,+0/-0,0.0,46.0,46.0,52.296,2.73,0.083,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),-11343.0,+0/-0,0.0,47.0,47.0,52.871,30.19,0.134,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),-12204.0,+0/-0,0.0,48.0,48.0,55.965,30.19,0.017,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),-12964.0,+0/-0,0.0,49.0,49.0,56.26,0.13,0.015,✔️,0.0,False,CPU +49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),-14158.0,+0/-0,0.0,50.0,50.0,57.211,76.41,2.853,✔️,0.0,False,CPU +50,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),-15232.0,+0/-0,0.0,51.0,51.0,57.769,3.14,0.921,✔️,0.0,False,GPU +51,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),-17101.0,+0/-0,0.0,52.0,52.0,57.822,0.11,0.023,➖,0.0,False,GPU +52,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),-18718.0,+0/-0,0.0,53.0,53.0,70.703,76.41,0.258,✔️,0.0,False,CPU +53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-20036.0,+0/-0,0.0,54.0,54.0,83.729,0.14,0.123,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/winrate_matrix.png.zip index 65aa98995169e5887a86a31a9f6b72d31293a378..cbe255c82e0439462a388ccfb1b18e949e10d679 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c56ce20c279dd3d62a929a1594d21d3fd3a6fcffe8b78f639a8128374228941b -size 1057411 +oid sha256:2d8175910c71a3c066658b9e348642f3c74097f0aa8a5953b3707259b2e969d7 +size 1130689 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 91d5f965a91363e5d3c1830c26b8fb8a3b9d4074..305d8b3246d8869d46875d0e08f2ef0963845cef 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:705dc4364e8d077276487955c04c1b751470216d73571cc7152fa63925a281aa -size 420132 +oid sha256:d2ead3973e25443fe17e12eb780ace6d3957af18a2069f8bcbe82b41fe1a0ef0 +size 434286 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip index 1b0d221d93922aa4ab256bad065b159179bae349..c13961cdbdbc208117a45ee2d6e70f26b54b92dd 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d8faed2226edad7d3538f50a0c61f66f275a792ddb190fef162f3c3130356814 -size 917622 +oid sha256:3dfe454f562cb33e84d1d7d539034939f4280e66594767225ae2dc29b44f8ff3 +size 938367 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip index 98f43b263e965d57ec969c632fc000a9339f12d4..42b6337048ff9f93895a1c9ae4eb1b553485bfe4 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9cbf1c64d1d616e2e4ded898808c3673a6ed72488dba8cea473ad09325b338bf -size 120826 +oid sha256:ef0a5886b7d7b35ab2c48d541fe3a71a97487dfcd557e9cd17cdc5797340452a +size 125195 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/website_leaderboard.csv index ebfc92896694a5444c2bfe8cccf3696a7fe65bfc..96d7791d1dae581f11a174310ae4458ab174932b 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/website_leaderboard.csv @@ -1,54 +1,55 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1735.0,+490/-208,2.066,0.754,6.43,1.8,2837.96,9.546,✔️,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1714.0,+270/-90,12.349,0.701,7.0,5.03,1060.41,10.928,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1692.0,+452/-185,10.895,0.639,7.64,3.61,2837.96,1.134,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1606.0,+237/-104,10.653,0.529,10.57,6.05,6.19,1.301,➖,0.0,False,GPU -4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1586.0,+384/-174,9.756,0.555,11.36,4.28,10.97,0.856,✔️,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1542.0,+286/-141,14.183,0.482,13.29,4.65,4655.15,21.791,✔️,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1499.0,+207/-149,14.644,0.431,15.29,12.16,4655.15,4.031,✔️,0.0,False,GPU -7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1493.0,+283/-109,15.834,0.439,15.57,4.78,6318.92,3.195,✔️,0.0,False,CPU -8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1476.0,+162/-116,18.233,0.383,16.43,11.56,5729.39,1.817,✔️,0.0,False,GPU -9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1473.0,+246/-122,15.543,0.35,16.57,12.44,3375.11,1.208,✔️,0.0,False,CPU -10,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1459.0,+174/-119,18.608,0.381,17.29,10.81,5729.39,0.373,✔️,0.0,False,GPU -11,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1459.0,+194/-76,16.169,0.315,17.29,13.39,3375.11,0.132,✔️,0.0,False,CPU -12,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1448.0,+180/-192,17.962,0.351,17.86,7.75,2149.3,2.928,✔️,0.0,False,GPU -13,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1445.0,+228/-195,19.367,0.412,18.0,9.68,10182.26,399.569,✔️,0.0,False,GPU -14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1437.0,+194/-90,17.093,0.287,18.43,13.96,1394.84,3.578,✔️,0.0,False,CPU -15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1432.0,+228/-90,17.176,0.279,18.71,13.01,1394.84,1.087,✔️,0.0,False,CPU -16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1376.0,+118/-82,17.736,0.206,21.71,20.11,2342.58,5.131,✔️,0.0,False,CPU -17,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1374.0,+204/-136,20.76,0.267,21.86,13.5,19.13,0.146,✔️,0.0,False,GPU -18,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1371.0,+119/-98,17.42,0.217,22.0,19.75,2342.58,1.065,✔️,0.0,False,CPU -19,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1369.0,+252/-210,20.345,0.345,22.14,5.38,10182.26,40.124,✔️,0.0,False,GPU -20,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1355.0,+170/-136,18.825,0.214,22.93,18.01,6994.17,9.554,✔️,0.0,False,GPU -21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1351.0,+203/-188,19.587,0.277,23.14,5.23,2149.3,0.147,✔️,0.0,False,GPU -22,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1336.0,+148/-119,20.198,0.204,24.0,19.03,326.23,4.282,➖,0.0,False,GPU -23,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1332.0,+178/-142,19.122,0.17,24.21,19.46,6994.17,0.583,✔️,0.0,False,GPU -24,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1322.0,+208/-231,22.319,0.236,24.79,12.36,772.2,2.495,✔️,0.0,False,CPU -25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1303.0,+128/-72,19.353,0.1,25.86,23.83,57.99,0.317,✔️,0.0,False,CPU -26,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1294.0,+198/-204,22.113,0.22,26.36,12.36,772.2,0.373,✔️,0.0,False,CPU -27,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1285.0,+151/-114,21.802,0.137,26.86,23.46,3833.57,2.294,✔️,0.0,False,CPU -28,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1285.0,+127/-98,21.656,0.11,26.86,22.23,44054.5,0.843,✔️,0.0,False,CPU -29,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1268.0,+204/-281,22.276,0.216,27.86,15.62,731.01,2.021,✔️,0.0,False,CPU -30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1263.0,+270/-230,23.176,0.206,28.14,17.27,2750.22,13.569,✔️,0.0,False,CPU -31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1245.0,+189/-158,23.947,0.132,29.14,19.47,3833.57,0.22,✔️,0.0,False,CPU -32,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1242.0,+194/-213,22.488,0.154,29.29,22.35,5.15,0.526,✔️,0.0,False,CPU -33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1237.0,+117/-124,23.403,0.09,29.57,27.07,14.99,0.113,✔️,0.0,False,CPU -34,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1230.0,+256/-231,24.042,0.247,30.0,9.32,51.52,40.219,✔️,0.0,False,GPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1227.0,+129/-113,23.217,0.095,30.14,26.81,44054.5,0.068,✔️,0.0,False,CPU -36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1190.0,+137/-224,23.958,0.122,32.14,20.49,731.01,0.334,✔️,0.0,False,CPU -37,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1182.0,+76/-92,21.807,0.004,32.57,31.38,5.38,0.624,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1179.0,+159/-187,23.313,0.1,32.71,27.33,20.63,7.532,✔️,0.0,False,GPU -39,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1163.0,+154/-219,24.905,0.081,33.57,28.55,9.4,1.444,✔️,0.0,False,GPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1146.0,+234/-242,25.292,0.108,34.43,26.13,2750.22,1.239,✔️,0.0,False,CPU -41,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1096.0,+121/-168,28.196,0.015,36.86,34.67,19.64,0.478,✔️,0.0,False,GPU -42,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1080.0,+257/-419,35.799,0.147,37.57,24.17,180.38,1.24,✔️,0.0,False,CPU -43,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1004.0,+115/-178,30.387,0.0,40.86,39.86,14.87,0.391,✔️,0.0,False,CPU -44,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+112/-308,37.213,0.0,41.0,39.22,0.78,0.155,✔️,0.0,False,CPU -45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),974.0,+204/-308,31.994,0.035,42.0,37.31,8.78,0.688,✔️,0.0,False,CPU -46,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),972.0,+82/-247,39.676,0.0,42.07,41.01,913.1,0.738,✔️,0.0,False,CPU -47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),952.0,+100/-294,40.327,0.0,42.79,41.64,913.1,0.13,✔️,0.0,False,CPU -48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),934.0,+190/-297,39.305,0.01,43.43,40.35,0.77,0.156,✔️,0.0,False,CPU -49,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),836.0,+283/-1332,47.681,0.072,46.29,39.0,180.38,0.136,✔️,0.0,False,CPU -50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),813.0,+47/-188,39.077,0.0,46.86,46.67,5.48,0.91,➖,0.0,False,GPU -51,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),743.0,+101/-303,46.939,0.0,48.29,47.92,4.65,0.14,✔️,0.0,False,CPU -52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-420.0,+334/-866,69.099,0.0,53.0,53.0,0.58,0.071,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1729.0,+476/-207,0.754,6.43,1.8,2.066,2837.96,9.546,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1698.0,+238/-91,0.7,7.29,5.22,12.349,1060.41,10.928,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1682.0,+406/-180,0.639,7.79,3.7,10.895,2837.96,1.134,✔️,0.0,False,GPU +3,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1592.0,+208/-101,0.531,11.0,6.64,10.653,6.19,1.301,➖,0.0,False,GPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1576.0,+366/-170,0.555,11.64,4.79,9.756,10.97,0.856,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1529.0,+255/-134,0.482,13.71,4.76,14.183,4655.15,21.791,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1488.0,+192/-142,0.429,15.71,12.47,14.644,4655.15,4.031,✔️,0.0,False,GPU +7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1482.0,+272/-110,0.441,16.0,4.91,15.834,6318.92,3.195,✔️,0.0,False,CPU +8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1466.0,+160/-115,0.379,16.86,11.86,18.233,5729.39,1.817,✔️,0.0,False,GPU +9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1460.0,+217/-115,0.348,17.14,13.03,15.543,3375.11,1.208,✔️,0.0,False,CPU +10,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1449.0,+161/-117,0.378,17.71,10.94,18.608,5729.39,0.373,✔️,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1446.0,+171/-67,0.313,17.86,13.75,16.169,3375.11,0.132,✔️,0.0,False,CPU +12,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1436.0,+176/-195,0.349,18.43,7.92,17.962,2149.3,2.928,✔️,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1433.0,+219/-195,0.41,18.57,9.81,19.367,10182.26,399.569,✔️,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1428.0,+182/-95,0.288,18.86,14.23,17.093,1394.84,3.578,✔️,0.0,False,CPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1423.0,+219/-90,0.276,19.14,13.32,17.176,1394.84,1.087,✔️,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1367.0,+101/-76,0.205,22.29,20.87,17.736,2342.58,5.131,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1364.0,+193/-136,0.266,22.43,13.73,20.76,19.13,0.146,✔️,0.0,False,GPU +18,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1362.0,+102/-95,0.216,22.57,20.54,17.42,2342.58,1.065,✔️,0.0,False,CPU +19,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1359.0,+247/-207,0.345,22.71,5.41,20.345,10182.26,40.124,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1350.0,+280/-250,0.424,23.29,6.99,20.095,22.26,1.556,➖,0.0,False,GPU +21,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1346.0,+159/-134,0.212,23.5,18.33,18.825,6994.17,9.554,✔️,0.0,False,GPU +22,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1342.0,+203/-190,0.278,23.71,5.31,19.587,2149.3,0.147,✔️,0.0,False,GPU +23,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1328.0,+144/-120,0.202,24.57,19.6,20.198,326.23,4.282,➖,0.0,False,GPU +24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1326.0,+165/-142,0.17,24.64,19.65,19.122,6994.17,0.583,✔️,0.0,False,GPU +25,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1314.0,+201/-229,0.236,25.36,12.58,22.319,772.2,2.495,✔️,0.0,False,CPU +26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1296.0,+119/-68,0.096,26.43,24.61,19.353,57.99,0.317,✔️,0.0,False,CPU +27,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1287.0,+192/-206,0.222,26.93,12.71,22.113,772.2,0.373,✔️,0.0,False,CPU +28,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1281.0,+127/-94,0.107,27.29,22.48,21.656,44054.5,0.843,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1276.0,+144/-112,0.135,27.57,23.96,21.802,3833.57,2.294,✔️,0.0,False,CPU +30,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1261.0,+198/-276,0.218,28.43,16.95,22.276,731.01,2.021,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1256.0,+262/-226,0.204,28.71,18.0,23.176,2750.22,13.569,✔️,0.0,False,CPU +32,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1237.0,+186/-204,0.152,29.86,22.85,22.488,5.15,0.526,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1237.0,+185/-161,0.133,29.86,19.71,23.947,3833.57,0.22,✔️,0.0,False,CPU +34,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1229.0,+115/-126,0.092,30.29,27.55,23.403,14.99,0.113,✔️,0.0,False,CPU +35,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1224.0,+235/-225,0.247,30.57,9.34,24.042,51.52,40.219,✔️,0.0,False,GPU +36,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1220.0,+115/-111,0.094,30.86,27.68,23.217,44054.5,0.068,✔️,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1186.0,+132/-217,0.122,32.71,22.15,23.958,731.01,0.334,✔️,0.0,False,CPU +38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1178.0,+67/-103,0.004,33.14,32.01,21.807,5.38,0.624,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1173.0,+144/-183,0.097,33.43,28.17,23.313,20.63,7.532,✔️,0.0,False,GPU +40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1160.0,+151/-212,0.081,34.14,29.08,24.905,9.4,1.444,✔️,0.0,False,GPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1141.0,+217/-232,0.106,35.14,27.44,25.292,2750.22,1.239,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1093.0,+112/-157,0.014,37.57,35.36,28.196,19.64,0.478,✔️,0.0,False,GPU +43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1081.0,+254/-420,0.149,38.14,24.31,35.799,180.38,1.24,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1004.0,+112/-185,0.0,41.57,40.51,30.387,14.87,0.391,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+106/-279,0.0,41.71,39.95,37.213,0.78,0.155,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),971.0,+195/-285,0.032,42.86,38.41,31.994,8.78,0.688,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),969.0,+72/-243,0.0,42.93,41.93,39.676,913.1,0.738,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),950.0,+90/-287,0.0,43.64,42.56,40.327,913.1,0.13,✔️,0.0,False,CPU +49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),936.0,+172/-290,0.013,44.14,40.87,39.305,0.77,0.156,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),842.0,+274/-769,0.072,47.0,39.41,47.681,180.38,0.136,✔️,0.0,False,CPU +51,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),814.0,+44/-196,0.0,47.71,47.51,39.077,5.48,0.91,➖,0.0,False,GPU +52,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),747.0,+91/-276,0.0,49.14,48.8,46.939,4.65,0.14,✔️,0.0,False,CPU +53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-489.0,+398/-405,0.0,54.0,54.0,69.099,0.58,0.071,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/winrate_matrix.png.zip index 9110ac5ac9f85cc25d49d31d88b8f8cc9ce0513c..de847e35d7d630611914c3a64608538b1c2fdcfa 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:784a04ce1ab4e79da2595b241e40589d9f11239b53278bcf609d587414fb2793 -size 1435771 +oid sha256:df94839e6023bde00ac130c10c609788bfac120ea59854cd886cf8f840311d23 +size 1541577 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 3aef60bac3c162343db2666bd53abe84855be7f9..585645f437f3ae2f6878fa4ae1a68f094201f9cf 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:31238b7a81d2723b92a5f60639c05ac16a8333347a1fd57a48f685e3c733e955 -size 456644 +oid sha256:c412b4ae8ae566647419c6c3a2d1930eb9fcc15f5fc88b252f93a5e329dc0d95 +size 471777 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip index 3a3da247bcca7b19a9762a040b54317624ed24f5..b2a6cd1d395439def1e9e33a74688315ca136177 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f5601473ef5b4871a921fc6ee01869603bee1a7bc35b68edd6401e0b30418689 -size 975573 +oid sha256:5ff8805617d56b31ac7e84912422f72df2c40cf43dcd3cd09bfac2942fafddff +size 997903 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip index 5e04931cf7095d7842e8bf36d548403db41676fb..92d96c31dea73ac9b177729ae0b967d7a9beae79 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c6e00cef3f52b349a4f96f6f2bad5d782c604e4c52e8a8036ea79fcda149c108 -size 125861 +oid sha256:9bb390e3fc8495c2c548f19ffe930f8ab7df63d136a117520e4698b0a015bc4c +size 131002 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv index a4c2ec6e6121067ab7b4a9e628750c998dfa2ed1..de4beda66c35f1f0daaabadfa2bfe9e326870a75 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1997.0,+809/-100,2.191,0.875,3.17,1.58,2793.65,8.93,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1857.0,+337/-82,12.495,0.729,5.42,4.12,2793.65,1.02,✔️,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1746.0,+356/-135,14.332,0.689,8.17,5.69,1295.16,14.354,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1716.0,+553/-144,11.083,0.632,9.08,4.46,9.88,0.825,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1688.0,+378/-116,14.698,0.596,10.0,5.22,3561.2,34.413,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1634.0,+358/-88,12.332,0.506,12.0,6.72,5.93,0.989,➖,0.0,False,GPU -6,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1626.0,+204/-38,15.748,0.513,12.33,10.85,206.59,4.778,✔️,0.0,False,GPU -7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1622.0,+272/-96,16.24,0.534,12.5,4.43,4334.8,21.159,✔️,0.0,False,GPU -8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1583.0,+233/-63,16.59,0.472,14.17,12.85,4334.8,3.529,✔️,0.0,False,GPU -9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1522.0,+224/-109,17.857,0.384,17.0,14.03,3307.58,1.182,✔️,0.0,False,CPU -10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1506.0,+327/-108,16.716,0.373,17.83,11.35,3561.2,0.612,✔️,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1502.0,+371/-116,14.808,0.388,18.0,9.36,5.48,0.434,✔️,0.0,False,GPU -12,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1489.0,+378/-152,18.377,0.401,18.67,4.6,6700.69,3.268,✔️,0.0,False,CPU -13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1476.0,+201/-74,18.68,0.307,19.33,15.87,3307.58,0.108,✔️,0.0,False,CPU -14,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1445.0,+344/-241,22.477,0.382,21.0,9.92,9411.86,433.416,✔️,0.0,False,GPU -15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1433.0,+268/-147,21.223,0.283,21.67,15.18,5568.31,1.781,✔️,0.0,False,GPU -16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1418.0,+175/-44,19.816,0.219,22.5,21.18,2029.77,1.073,✔️,0.0,False,CPU -17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1412.0,+142/-39,20.228,0.197,22.83,22.15,2029.77,6.631,✔️,0.0,False,CPU -18,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1412.0,+262/-84,19.85,0.213,22.83,16.26,1280.01,5.479,✔️,0.0,False,CPU -19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1409.0,+244/-141,21.671,0.272,23.0,14.96,5568.31,0.372,✔️,0.0,False,GPU -20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1403.0,+285/-92,19.946,0.219,23.33,14.81,1280.01,1.143,✔️,0.0,False,CPU -21,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1398.0,+205/-127,21.447,0.219,23.58,19.93,6829.34,7.818,✔️,0.0,False,GPU -22,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1394.0,+268/-198,20.929,0.246,23.83,18.35,2059.45,3.769,✔️,0.0,False,GPU -23,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1379.0,+437/-228,22.95,0.297,24.67,6.92,13.87,2.152,✔️,0.0,False,GPU -24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1375.0,+264/-171,21.793,0.169,24.92,20.15,6829.34,0.53,✔️,0.0,False,GPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1356.0,+255/-176,24.08,0.229,26.0,13.94,17.09,0.145,✔️,0.0,False,GPU -26,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1353.0,+340/-277,23.608,0.32,26.17,4.97,9411.86,42.924,✔️,0.0,False,GPU -27,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1319.0,+250/-164,23.403,0.191,28.17,20.83,410.08,4.104,➖,0.0,False,GPU -28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1302.0,+373/-354,26.238,0.223,29.17,18.56,2721.87,14.595,✔️,0.0,False,CPU -29,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1296.0,+161/-125,22.348,0.109,29.5,26.85,155.57,0.254,✔️,0.0,False,CPU -30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1278.0,+292/-226,22.852,0.145,30.5,21.21,2059.45,0.127,✔️,0.0,False,GPU -31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1270.0,+202/-155,25.058,0.115,31.0,23.64,27108.48,0.742,✔️,0.0,False,CPU -32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1264.0,+222/-200,25.243,0.119,31.33,27.49,4370.77,2.162,✔️,0.0,False,CPU -33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1246.0,+253/-279,26.005,0.12,32.33,25.27,728.32,2.548,✔️,0.0,False,CPU -34,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1235.0,+397/-370,27.727,0.284,33.0,8.74,42.75,45.46,✔️,0.0,False,GPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1217.0,+342/-353,25.893,0.148,34.0,18.85,729.17,2.154,✔️,0.0,False,CPU -36,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1211.0,+276/-254,25.765,0.102,34.33,25.35,728.32,0.356,✔️,0.0,False,CPU -37,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1190.0,+175/-206,26.917,0.075,35.5,30.79,27108.48,0.066,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1190.0,+176/-212,25.981,0.088,35.5,30.85,19.2,6.922,✔️,0.0,False,GPU -39,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1181.0,+89/-95,24.783,0.0,36.0,35.1,4.93,0.671,✔️,0.0,False,CPU -40,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1181.0,+170/-206,27.166,0.035,36.0,34.18,11.78,0.099,✔️,0.0,False,CPU -41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1159.0,+197/-173,26.155,0.06,37.17,32.88,5.12,0.444,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1156.0,+276/-281,28.736,0.113,37.33,27.67,2721.87,1.334,✔️,0.0,False,CPU -43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1153.0,+228/-281,28.727,0.078,37.5,31.64,8.99,1.498,✔️,0.0,False,GPU -44,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1146.0,+149/-137,27.899,0.006,37.83,36.17,4370.77,0.22,✔️,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1137.0,+257/-308,27.785,0.097,38.33,25.06,729.17,0.337,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1120.0,+154/-209,29.623,0.01,39.17,36.91,17.24,0.42,✔️,0.0,False,GPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+143/-432,37.954,0.0,44.67,42.99,0.74,0.148,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),979.0,+86/-362,38.521,0.0,45.5,44.67,909.67,0.893,✔️,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),962.0,+100/-303,39.28,0.0,46.17,45.13,909.67,0.107,✔️,0.0,False,CPU -50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),962.0,+232/-508,34.491,0.032,46.17,41.47,8.37,0.685,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),939.0,+300/-564,41.716,0.037,47.0,40.48,155.52,1.033,✔️,0.0,False,CPU -52,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),934.0,+277/-430,39.387,0.004,47.17,43.75,0.76,0.153,✔️,0.0,False,CPU -53,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),924.0,+96/-253,35.184,0.0,47.5,47.01,14.57,0.363,✔️,0.0,False,CPU -54,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),748.0,+71/-303,44.164,0.0,52.17,51.98,4.67,0.855,➖,0.0,False,GPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),723.0,+130/-462,44.545,0.0,52.67,52.34,4.62,0.135,✔️,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),372.0,+132/-1205,55.52,0.0,56.33,56.29,155.52,0.12,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-548.0,+343/-1220,70.032,0.0,58.0,58.0,0.52,0.058,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1985.0,+819/-92,0.874,3.17,1.58,2.191,2793.65,8.93,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1838.0,+303/-74,0.726,5.58,4.21,12.495,2793.65,1.02,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1727.0,+326/-137,0.684,8.5,5.83,14.332,1295.16,14.354,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1699.0,+521/-146,0.63,9.42,5.11,11.083,9.88,0.825,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1668.0,+331/-107,0.592,10.5,5.33,14.698,3561.2,34.413,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1616.0,+349/-106,0.503,12.5,7.2,12.332,5.93,0.989,➖,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1604.0,+232/-93,0.529,13.0,4.49,16.24,4334.8,21.159,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),1604.0,+181/-40,0.508,13.0,11.38,15.748,206.59,4.778,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1567.0,+189/-61,0.465,14.67,13.1,16.59,4334.8,3.529,✔️,0.0,False,GPU +9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1506.0,+200/-100,0.376,17.67,14.64,17.857,3307.58,1.182,✔️,0.0,False,CPU +10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1489.0,+288/-107,0.369,18.5,12.65,16.716,3561.2,0.612,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1486.0,+335/-131,0.385,18.67,11.14,14.808,5.48,0.434,✔️,0.0,False,GPU +12,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1477.0,+342/-144,0.402,19.17,4.67,18.377,6700.69,3.268,✔️,0.0,False,CPU +13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1461.0,+178/-67,0.299,20.0,16.28,18.68,3307.58,0.108,✔️,0.0,False,CPU +14,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1433.0,+592/-306,0.488,21.5,6.49,22.111,21.84,1.394,➖,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1430.0,+312/-243,0.375,21.67,10.03,22.477,9411.86,433.416,✔️,0.0,False,GPU +16,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1422.0,+263/-141,0.275,22.17,15.51,21.223,5568.31,1.781,✔️,0.0,False,GPU +17,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1405.0,+164/-47,0.209,23.17,21.97,19.816,2029.77,1.073,✔️,0.0,False,CPU +18,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1402.0,+229/-82,0.213,23.33,16.53,19.85,1280.01,5.479,✔️,0.0,False,CPU +19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1399.0,+221/-145,0.263,23.5,15.15,21.671,5568.31,0.372,✔️,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1399.0,+112/-36,0.191,23.5,22.92,20.228,2029.77,6.631,✔️,0.0,False,CPU +21,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1394.0,+265/-96,0.216,23.83,15.08,19.946,1280.01,1.143,✔️,0.0,False,CPU +22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1387.0,+180/-132,0.213,24.25,20.29,21.447,6829.34,7.818,✔️,0.0,False,GPU +23,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1382.0,+259/-209,0.241,24.5,19.06,20.929,2059.45,3.769,✔️,0.0,False,GPU +24,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1369.0,+370/-235,0.293,25.33,6.95,22.95,13.87,2.152,✔️,0.0,False,GPU +25,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1367.0,+232/-176,0.168,25.42,20.34,21.793,6829.34,0.53,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1346.0,+255/-174,0.224,26.67,14.13,24.08,17.09,0.145,✔️,0.0,False,GPU +27,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1343.0,+323/-276,0.318,26.83,4.99,23.608,9411.86,42.924,✔️,0.0,False,GPU +28,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1309.0,+234/-162,0.189,28.83,21.41,23.403,410.08,4.104,➖,0.0,False,GPU +29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1292.0,+336/-351,0.223,29.83,19.05,26.238,2721.87,14.595,✔️,0.0,False,CPU +30,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1287.0,+129/-116,0.107,30.17,27.74,22.348,155.57,0.254,✔️,0.0,False,CPU +31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1270.0,+286/-229,0.143,31.17,22.23,22.852,2059.45,0.127,✔️,0.0,False,GPU +32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1264.0,+194/-149,0.115,31.5,23.88,25.058,27108.48,0.742,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1253.0,+196/-195,0.114,32.17,28.09,25.243,4370.77,2.162,✔️,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1238.0,+248/-283,0.117,33.0,25.98,26.005,728.32,2.548,✔️,0.0,False,CPU +35,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1227.0,+379/-376,0.285,33.67,8.76,27.727,42.75,45.46,✔️,0.0,False,GPU +36,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1210.0,+334/-355,0.146,34.67,20.47,25.893,729.17,2.154,✔️,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1205.0,+263/-259,0.1,35.0,26.53,25.765,728.32,0.356,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1182.0,+162/-209,0.085,36.33,31.71,25.981,19.2,6.922,✔️,0.0,False,GPU +39,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1182.0,+160/-204,0.075,36.33,31.9,26.917,27108.48,0.066,✔️,0.0,False,CPU +40,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1176.0,+71/-116,0.0,36.67,35.78,24.783,4.93,0.671,✔️,0.0,False,CPU +41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1173.0,+142/-204,0.038,36.83,34.87,27.166,11.78,0.099,✔️,0.0,False,CPU +42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1155.0,+186/-181,0.053,37.83,33.78,26.155,5.12,0.444,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1149.0,+268/-333,0.108,38.17,29.05,28.736,2721.87,1.334,✔️,0.0,False,CPU +44,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1149.0,+225/-269,0.074,38.17,32.2,28.727,8.99,1.498,✔️,0.0,False,GPU +45,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1139.0,+123/-136,0.009,38.67,36.91,27.899,4370.77,0.22,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1133.0,+254/-292,0.095,39.0,26.77,27.785,729.17,0.337,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1117.0,+145/-209,0.01,39.83,37.58,29.623,17.24,0.42,✔️,0.0,False,GPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+140/-390,0.0,45.33,43.69,37.954,0.74,0.148,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),976.0,+75/-309,0.0,46.33,45.57,38.521,909.67,0.893,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),958.0,+93/-290,0.0,47.0,46.03,39.28,909.67,0.107,✔️,0.0,False,CPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),958.0,+222/-468,0.029,47.0,42.52,34.491,8.37,0.685,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),940.0,+269/-584,0.037,47.67,40.86,41.716,155.52,1.033,✔️,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),936.0,+247/-400,0.004,47.83,44.23,39.387,0.76,0.153,✔️,0.0,False,CPU +54,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),922.0,+94/-257,0.0,48.33,47.86,35.184,14.57,0.363,✔️,0.0,False,CPU +55,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),753.0,+61/-325,0.0,53.0,52.79,44.164,4.67,0.855,➖,0.0,False,GPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),728.0,+119/-375,0.0,53.5,53.2,44.545,4.62,0.135,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),396.0,+125/-910,0.0,57.17,57.13,55.52,155.52,0.12,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-404.0,+186/-922,0.0,59.0,59.0,70.032,0.52,0.058,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip index 34217f66ba01051a307bf91c9176dc41140c7fcd..3c6497dfcf1a3204c3b5604898429d5723d41420 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ae9c565c6ed73c863db8835fda0bf33788261e08b2320fc445f66973a7e33ce6 -size 1681180 +oid sha256:5ad4cb7d35c32db3ec8bf7dbebdc7f08422c9a73a569826c4feb15dc2a8aad9e +size 1570107 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index 989b36f8b2eee94195794a42653c785b816eff5e..14d5742f08919475d3dc838b426026666e333151 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:dae14c6585f07ef55aa457d9b4e25458905605227b9e9124a927920b396f756d -size 420357 +oid sha256:f9d5779f2d691688c719eb234b74ccd4094218dec3331428e40493a4f95f9d71 +size 432744 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip index 1afadd659e2a78aa651947db4b14d4e42f201992..85aba0bb4292b91f96debcaf4587e4c35c15227b 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8c8bd8991b471e68b6234d723ce86b5fd25ade404293f508e8db540dfc88de1e -size 957317 +oid sha256:a3dd938a0ed51ac2712a04735a74b226e6bbd7660d7513559dc949c65e219f4a +size 950120 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_all/tuning-impact-elo.png.zip index 70c20091fb93d6a9f52c52792e6fd04e32a0c4ae..20e8ec13e75859bcd131d0ce3597ef6045a7abab 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c3e4fcb6b06589974a27c10b5bcd4dda673df75b64ecf4d9be106a212f66e4cd -size 120084 +oid sha256:81cad66af2c44598d9a1da6dd2897c1a3c6484e8988e89cd56e9968e05474577 +size 123446 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_regression/datasets_all/website_leaderboard.csv index 706f8247a907c2d73593e8c437c9f1c694cd6007..e45d49caf5fcb34138ef1fd4c63beebdcd8e5d43 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_all/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_all/website_leaderboard.csv @@ -1,52 +1,53 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1932.0,+314/-174,1.161,0.799,5.15,2.41,2043.44,8.369,✔️,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1857.0,+142/-62,2.562,0.647,6.85,3.93,616.43,4.041,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1854.0,+258/-95,2.472,0.729,6.92,2.28,7125.58,336.907,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1819.0,+190/-85,2.973,0.677,7.85,3.79,7125.58,38.573,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1808.0,+228/-164,2.101,0.673,8.15,3.27,2043.44,0.781,✔️,0.0,False,GPU -5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1760.0,+101/-62,4.699,0.537,9.62,5.51,1489.94,5.79,✔️,0.0,False,CPU -6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1755.0,+118/-66,4.174,0.571,9.77,8.06,4057.12,10.478,✔️,0.0,False,GPU -7,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1692.0,+182/-93,4.157,0.541,11.85,6.26,44.65,38.298,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1690.0,+216/-178,3.576,0.532,11.92,6.71,6.94,0.519,✔️,0.0,False,GPU -9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1663.0,+157/-141,6.669,0.505,12.92,5.46,3811.43,7.581,✔️,0.0,False,GPU -10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1616.0,+147/-131,5.331,0.433,14.69,10.42,4057.12,0.337,✔️,0.0,False,GPU -11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1611.0,+92/-59,7.3,0.328,14.92,12.75,686.46,5.477,✔️,0.0,False,CPU -12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1582.0,+93/-70,7.083,0.29,16.08,12.31,2895.38,1.317,✔️,0.0,False,CPU -13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1564.0,+127/-84,6.477,0.358,16.85,13.31,708.95,1.973,✔️,0.0,False,GPU -14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1551.0,+132/-103,5.748,0.311,17.38,13.53,4228.53,1.188,✔️,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1542.0,+123/-97,7.207,0.271,17.77,9.88,2895.38,0.073,✔️,0.0,False,CPU -16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1516.0,+104/-90,8.098,0.23,18.85,15.39,686.46,0.743,✔️,0.0,False,CPU -17,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1512.0,+232/-213,13.045,0.404,19.0,4.97,5.49,0.754,➖,0.0,False,GPU -18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1503.0,+118/-74,7.042,0.257,19.38,16.18,708.95,0.069,✔️,0.0,False,GPU -19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1472.0,+71/-57,8.021,0.174,20.69,19.28,848.99,0.475,✔️,0.0,False,CPU -20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1472.0,+60/-49,8.036,0.173,20.69,19.7,848.99,2.378,✔️,0.0,False,CPU -21,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1460.0,+109/-92,9.327,0.202,21.23,18.6,16.07,0.29,✔️,0.0,False,GPU -22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1444.0,+134/-95,6.582,0.191,21.92,18.71,4228.53,0.125,✔️,0.0,False,GPU -23,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1434.0,+150/-147,9.198,0.259,22.38,13.93,3811.43,0.452,✔️,0.0,False,GPU -24,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1430.0,+117/-136,9.171,0.173,22.54,18.16,8.35,0.087,✔️,0.0,False,CPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1357.0,+136/-132,8.245,0.129,25.77,22.76,13.9,0.123,✔️,0.0,False,GPU -26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1309.0,+141/-114,10.088,0.087,27.85,23.61,9.17,1.416,✔️,0.0,False,GPU -27,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1296.0,+108/-102,9.416,0.078,28.38,25.93,4452.11,0.852,✔️,0.0,False,CPU -28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1295.0,+134/-114,12.273,0.093,28.46,23.52,161.73,0.784,✔️,0.0,False,CPU -29,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1271.0,+135/-118,12.586,0.105,29.46,23.46,161.73,0.12,✔️,0.0,False,CPU -30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1241.0,+130/-136,10.034,0.064,30.69,27.38,4452.11,0.089,✔️,0.0,False,CPU -31,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1241.0,+179/-168,12.917,0.076,30.69,14.6,2.47,0.755,✔️,0.0,False,GPU -32,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1224.0,+76/-101,10.917,0.013,31.38,30.46,2.03,0.296,✔️,0.0,False,CPU -33,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1214.0,+159/-194,13.421,0.12,31.77,21.26,3017.28,0.248,✔️,0.0,False,CPU -34,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1191.0,+97/-118,11.634,0.053,32.69,30.4,2.15,0.18,✔️,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1182.0,+83/-122,13.249,0.018,33.08,31.12,526.17,0.77,✔️,0.0,False,CPU -36,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1166.0,+155/-206,13.982,0.119,33.69,15.28,3017.28,0.027,✔️,0.0,False,CPU -37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1138.0,+95/-134,14.003,0.016,34.73,32.85,526.17,0.119,✔️,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1119.0,+104/-116,14.183,0.014,35.46,33.28,0.42,0.061,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1084.0,+75/-108,14.282,0.0,36.69,35.72,527.21,2.831,✔️,0.0,False,CPU -40,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1039.0,+118/-167,15.082,0.049,38.15,35.84,10.04,0.03,✔️,0.0,False,CPU -41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1037.0,+92/-138,14.125,0.0,38.23,36.79,20.5,0.081,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1034.0,+97/-114,14.768,0.0,38.31,37.3,527.21,0.311,✔️,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+73/-90,15.317,0.0,39.35,38.95,0.63,0.064,✔️,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),887.0,+94/-166,18.821,0.0,42.31,41.7,3.08,0.293,✔️,0.0,False,CPU -45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),877.0,+134/-183,20.298,0.0,42.54,41.76,90.87,0.906,✔️,0.0,False,CPU -46,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),713.0,+109/-192,22.447,0.0,45.54,45.39,90.87,0.055,✔️,0.0,False,CPU -47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),572.0,+205/-344,29.125,0.0,47.31,46.87,0.25,0.044,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),439.0,+113/-319,36.211,0.0,48.58,48.52,177.04,0.14,✔️,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),351.0,+147/-403,36.396,0.0,49.27,49.19,177.04,0.059,✔️,0.0,False,CPU -50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),199.0,+138/-328,38.856,0.0,50.23,50.22,0.88,0.109,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1916.0,+309/-174,0.803,5.23,2.41,1.161,2043.44,8.369,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1840.0,+146/-63,0.65,7.0,3.96,2.562,616.43,4.041,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1840.0,+253/-96,0.731,7.0,2.3,2.472,7125.58,336.907,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1806.0,+176/-81,0.678,7.92,3.81,2.973,7125.58,38.573,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1792.0,+223/-156,0.677,8.31,3.34,2.101,2043.44,0.781,✔️,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1742.0,+105/-66,0.54,9.85,5.56,4.699,1489.94,5.79,✔️,0.0,False,CPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1740.0,+110/-61,0.577,9.92,8.21,4.174,4057.12,10.478,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1677.0,+174/-88,0.546,12.08,6.31,4.157,44.65,38.298,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1677.0,+210/-174,0.538,12.08,6.74,3.576,6.94,0.519,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1648.0,+157/-143,0.508,13.15,5.47,6.669,3811.43,7.581,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1603.0,+146/-131,0.436,14.92,10.53,5.331,4057.12,0.337,✔️,0.0,False,GPU +11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1596.0,+97/-56,0.331,15.23,12.9,7.3,686.46,5.477,✔️,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1570.0,+91/-68,0.293,16.31,12.42,7.083,2895.38,1.317,✔️,0.0,False,CPU +13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1550.0,+129/-89,0.364,17.15,13.46,6.477,708.95,1.973,✔️,0.0,False,GPU +14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1538.0,+128/-102,0.315,17.69,13.7,5.748,4228.53,1.188,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1529.0,+124/-96,0.272,18.08,9.95,7.207,2895.38,0.073,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1504.0,+228/-208,0.407,19.15,5.03,13.045,5.49,0.754,➖,0.0,False,GPU +17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1503.0,+109/-88,0.237,19.23,15.54,8.098,686.46,0.743,✔️,0.0,False,CPU +18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1492.0,+115/-73,0.266,19.69,16.44,7.042,708.95,0.069,✔️,0.0,False,GPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1461.0,+60/-54,0.179,21.08,19.67,8.021,848.99,0.475,✔️,0.0,False,CPU +20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1459.0,+61/-49,0.179,21.15,20.12,8.036,848.99,2.378,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1447.0,+107/-89,0.206,21.69,19.02,9.327,16.07,0.29,✔️,0.0,False,GPU +22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1434.0,+131/-93,0.2,22.31,18.96,6.582,4228.53,0.125,✔️,0.0,False,GPU +23,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1423.0,+143/-141,0.264,22.77,14.12,9.198,3811.43,0.452,✔️,0.0,False,GPU +24,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1417.0,+116/-137,0.18,23.08,18.4,9.171,8.35,0.087,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1350.0,+128/-127,0.138,26.15,23.01,8.245,13.9,0.123,✔️,0.0,False,GPU +26,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1330.0,+197/-174,0.189,27.08,14.52,9.57,18.76,6.542,➖,0.0,False,GPU +27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1301.0,+133/-114,0.085,28.38,24.35,10.088,9.17,1.416,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1290.0,+100/-102,0.082,28.85,26.3,9.416,4452.11,0.852,✔️,0.0,False,CPU +29,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1290.0,+129/-112,0.097,28.85,23.85,12.273,161.73,0.784,✔️,0.0,False,CPU +30,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1267.0,+129/-115,0.11,29.85,23.81,12.586,161.73,0.12,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1237.0,+121/-131,0.067,31.15,27.85,10.034,4452.11,0.089,✔️,0.0,False,CPU +32,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1237.0,+175/-167,0.076,31.15,14.67,12.917,2.47,0.755,✔️,0.0,False,GPU +33,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1217.0,+72/-95,0.019,32.0,30.98,10.917,2.03,0.296,✔️,0.0,False,CPU +34,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1208.0,+155/-195,0.12,32.38,21.51,13.421,3017.28,0.248,✔️,0.0,False,CPU +35,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1184.0,+96/-119,0.054,33.38,30.95,11.634,2.15,0.18,✔️,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1177.0,+77/-116,0.017,33.69,31.85,13.249,526.17,0.77,✔️,0.0,False,CPU +37,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1158.0,+152/-199,0.12,34.46,15.43,13.982,3017.28,0.027,✔️,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1135.0,+92/-128,0.015,35.35,33.55,14.003,526.17,0.119,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1112.0,+102/-114,0.018,36.23,34.07,14.183,0.42,0.061,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1078.0,+78/-110,0.002,37.46,36.4,14.282,527.21,2.831,✔️,0.0,False,CPU +41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1035.0,+119/-166,0.049,38.92,36.54,15.082,10.04,0.03,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1033.0,+85/-140,0.0,39.0,37.51,14.125,20.5,0.081,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1030.0,+94/-114,0.0,39.08,38.03,14.768,527.21,0.311,✔️,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+70/-94,0.0,40.04,39.62,15.317,0.63,0.064,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),885.0,+99/-160,0.0,43.15,42.5,18.821,3.08,0.293,✔️,0.0,False,CPU +46,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),875.0,+126/-173,0.0,43.38,42.61,20.298,90.87,0.906,✔️,0.0,False,CPU +47,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),706.0,+106/-188,0.0,46.54,46.39,22.447,90.87,0.055,✔️,0.0,False,CPU +48,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),566.0,+202/-341,0.0,48.31,47.88,29.125,0.25,0.044,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),432.0,+109/-319,0.0,49.58,49.52,36.211,177.04,0.14,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),344.0,+144/-396,0.0,50.27,50.2,36.396,177.04,0.059,✔️,0.0,False,CPU +51,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),193.0,+133/-321,0.0,51.23,51.22,38.856,0.88,0.109,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_all/winrate_matrix.png.zip index 0abb271ab9d7ba43da34fae1da3a0b95a275b467..10bd556e95866c786c0834fa7be03f8b2f337cfe 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:598843ddce77f6ae141eb8630c255d75e9b89d14e418fd31e79d4e3ad5ec8a85 -size 1471479 +oid sha256:4281c4edec2f3918af393645cd352161a25c870767546d42b6b6c9e2b893c075 +size 1602009 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index 61cbdac8d75d0cd016e9cbc7b60a48b4f55c3b49..5a6f8b9e9d3db7941b09b24aa6455d64cf144097 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:63229fad1bc6bafe6268582cc8e98ed6125dfe6157959ccd584ee7696b629589 -size 430652 +oid sha256:3bfa59b331bdda090da4cba76775a4c2fe36ff54b07d17698b8c7c17039ad7da +size 446846 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip index fabc2244a3562be0d60e00c4f6cfdd7377095f5d..caa12fcc474e3ed3eac380d017b405cce888e12a 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:64c603bd1b32c2e569f74c054f5f1acc2c14278c003610e5c45847a8ad8b9ab3 -size 917540 +oid sha256:9bd9c138cf03d30807edd7e68d9dec2fb680f0575e90771d4cd21c0157997216 +size 926558 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/tuning-impact-elo.png.zip index 5ff55a105373740553892bce0532bcf597f48421..00633be2ecac1f028eba776bc3a1890d72bdbe6b 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:af93e2729b962c9d1a179621367a88891484681549768209c815f05db2032e9f -size 119314 +oid sha256:8f7a106c36bedaed292be9585bb55b7839ae020f68df68831efabc00278d36ee +size 123795 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/website_leaderboard.csv index 46a7e63e483c8172dd9737f1097129e0df9c843a..f9b0f8f9009f7ede0577dbfc4e85a81676205597 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/website_leaderboard.csv @@ -1,52 +1,53 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),2061.0,+1014/-71,1.163,0.832,4.4,3.24,739.1,11.65,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),2027.0,+1086/-121,1.262,0.835,5.0,2.74,739.1,0.689,✔️,0.0,False,GPU -2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1950.0,+850/-36,2.284,0.699,6.6,5.71,2552.28,4.522,✔️,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1916.0,+658/-8,3.209,0.655,7.4,3.4,474.94,11.837,✔️,0.0,False,CPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1885.0,+615/-13,3.513,0.612,8.2,6.09,400.37,2.16,✔️,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1870.0,+930/-101,2.851,0.637,8.6,6.91,2068.17,1.355,✔️,0.0,False,GPU -6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1822.0,+1355/-177,2.083,0.687,10.0,2.25,4529.88,357.209,✔️,0.0,False,GPU -7,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1790.0,+1289/-174,2.39,0.65,11.0,4.14,4529.88,39.544,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1783.0,+1058/-258,2.625,0.548,11.2,6.42,1.59,0.506,✔️,0.0,False,GPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1688.0,+928/-198,4.112,0.485,14.4,10.08,2068.17,0.068,✔️,0.0,False,GPU -10,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1677.0,+780/-243,4.058,0.471,14.8,11.42,2552.28,0.201,✔️,0.0,False,GPU -11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1677.0,+503/-139,5.836,0.377,14.8,11.88,223.05,9.707,✔️,0.0,False,CPU -12,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1671.0,+1088/-218,3.203,0.526,15.0,6.19,41.5,38.298,✔️,0.0,False,GPU -13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1660.0,+548/-97,6.007,0.349,15.4,12.03,223.05,0.793,✔️,0.0,False,CPU -14,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1649.0,+524/-91,4.481,0.418,15.8,13.49,438.55,0.52,✔️,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1644.0,+510/-102,6.235,0.3,16.0,10.62,1696.82,0.538,✔️,0.0,False,CPU -16,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1617.0,+754/-226,5.187,0.336,17.0,12.43,3456.17,0.611,✔️,0.0,False,GPU -17,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1590.0,+500/-169,6.475,0.28,18.0,7.25,1696.82,0.055,✔️,0.0,False,CPU -18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1559.0,+428/-97,5.477,0.253,19.2,17.73,438.55,0.035,✔️,0.0,False,GPU -19,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1553.0,+728/-163,5.3,0.287,19.4,16.95,6.87,0.098,✔️,0.0,False,GPU -20,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1537.0,+750/-208,4.7,0.306,20.0,4.21,2.07,0.754,➖,0.0,False,GPU -21,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1521.0,+561/-275,7.061,0.27,20.6,16.66,4.99,0.047,✔️,0.0,False,CPU -22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1505.0,+528/-187,6.188,0.183,21.2,19.63,3456.17,0.037,✔️,0.0,False,GPU -23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1494.0,+356/-58,6.853,0.189,21.6,20.89,289.35,1.798,✔️,0.0,False,CPU -24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1484.0,+386/-70,6.921,0.181,22.0,21.09,289.35,0.36,✔️,0.0,False,CPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1435.0,+648/-216,7.402,0.129,23.8,21.42,5.37,0.025,✔️,0.0,False,GPU -26,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1353.0,+354/-228,8.025,0.137,26.8,24.63,0.57,0.074,✔️,0.0,False,CPU -27,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1302.0,+357/-139,8.909,0.002,28.6,27.72,2.32,0.093,✔️,0.0,False,GPU -28,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1297.0,+375/-186,8.225,0.018,28.8,28.08,0.66,0.213,✔️,0.0,False,CPU -29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1274.0,+225/-245,8.161,0.064,29.6,27.65,4108.24,0.479,✔️,0.0,False,CPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1250.0,+508/-284,9.603,0.028,30.4,28.51,6.54,0.284,✔️,0.0,False,GPU -31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1221.0,+204/-296,9.253,0.029,31.4,29.47,4108.24,0.037,✔️,0.0,False,CPU -32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1172.0,+353/-451,10.288,0.023,33.0,31.22,35.47,0.502,✔️,0.0,False,CPU -33,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1147.0,+417/-890,13.625,0.144,33.8,25.8,1820.58,0.133,✔️,0.0,False,CPU -34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1140.0,+314/-411,10.454,0.006,34.0,32.83,35.47,0.054,✔️,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1134.0,+107/-401,10.053,0.0,34.2,33.96,79.96,0.571,✔️,0.0,False,CPU -36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1124.0,+128/-390,10.217,0.0,34.5,34.09,79.96,0.053,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1061.0,+165/-350,13.469,0.0,36.4,35.28,14.05,0.029,✔️,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1033.0,+387/-674,12.12,0.006,37.2,35.12,0.18,0.054,✔️,0.0,False,CPU -39,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1026.0,+316/-953,14.242,0.1,37.4,34.8,1820.58,0.012,✔️,0.0,False,CPU -40,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+78/-318,11.35,0.0,38.1,37.96,0.42,0.053,✔️,0.0,False,CPU -41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),996.0,+363/-1057,14.681,0.12,38.2,34.58,3.47,0.01,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),871.0,+51/-752,14.269,0.0,41.2,40.95,317.48,0.719,✔️,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),842.0,+43/-769,14.827,0.0,41.8,41.54,317.48,0.127,✔️,0.0,False,CPU -44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),800.0,+94/-889,18.666,0.0,42.6,42.42,142.18,1.027,✔️,0.0,False,CPU -45,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),662.0,+74/-902,21.083,0.0,44.8,44.73,142.18,0.057,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),662.0,+107/-831,18.199,0.0,44.8,44.56,1.33,0.126,✔️,0.0,False,CPU -47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),361.0,+132/-1452,28.294,0.0,47.6,47.49,0.06,0.036,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),145.0,+92/-1782,39.482,0.0,48.8,48.8,48.35,0.054,✔️,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),-119.0,+197/-2129,39.829,0.0,50.0,49.99,48.35,0.013,✔️,0.0,False,CPU -50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),-276.0,+185/-2881,42.789,0.0,50.6,50.6,0.13,0.02,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),2034.0,+922/-55,0.839,4.4,3.24,1.163,739.1,11.65,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),2001.0,+923/-105,0.841,5.0,2.74,1.262,739.1,0.689,✔️,0.0,False,GPU +2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1924.0,+642/-41,0.707,6.6,5.71,2.284,2552.28,4.522,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1883.0,+581/-12,0.661,7.6,3.42,3.209,474.94,11.837,✔️,0.0,False,CPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1853.0,+654/-28,0.618,8.4,6.13,3.513,400.37,2.16,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1846.0,+767/-85,0.648,8.6,6.91,2.851,2068.17,1.355,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1797.0,+975/-163,0.69,10.0,2.25,2.083,4529.88,357.209,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1765.0,+911/-166,0.654,11.0,4.14,2.39,4529.88,39.544,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1759.0,+894/-242,0.556,11.2,6.42,2.625,1.59,0.506,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1666.0,+707/-190,0.489,14.4,10.08,4.112,2068.17,0.068,✔️,0.0,False,GPU +10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1649.0,+507/-132,0.386,15.0,11.94,5.836,223.05,9.707,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1649.0,+700/-222,0.474,15.0,11.66,4.058,2552.28,0.201,✔️,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1644.0,+794/-200,0.533,15.2,6.22,3.203,41.5,38.298,✔️,0.0,False,GPU +13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1633.0,+559/-100,0.358,15.6,12.08,6.007,223.05,0.793,✔️,0.0,False,CPU +14,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1622.0,+581/-74,0.433,16.0,13.61,4.481,438.55,0.52,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1617.0,+561/-93,0.312,16.2,10.68,6.235,1696.82,0.538,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1590.0,+730/-206,0.343,17.2,12.6,5.187,3456.17,0.611,✔️,0.0,False,GPU +17,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1564.0,+548/-151,0.29,18.2,7.27,6.475,1696.82,0.055,✔️,0.0,False,CPU +18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1533.0,+484/-92,0.272,19.4,17.89,5.477,438.55,0.035,✔️,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1528.0,+545/-151,0.295,19.6,17.32,5.3,6.87,0.098,✔️,0.0,False,GPU +20,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1518.0,+706/-196,0.316,20.0,4.21,4.7,2.07,0.754,➖,0.0,False,GPU +21,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1492.0,+542/-270,0.28,21.0,16.83,7.061,4.99,0.047,✔️,0.0,False,CPU +22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1482.0,+582/-167,0.196,21.4,19.92,6.188,3456.17,0.037,✔️,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1471.0,+322/-53,0.203,21.8,21.03,6.853,289.35,1.798,✔️,0.0,False,CPU +24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1461.0,+336/-66,0.196,22.2,21.22,6.921,289.35,0.36,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1415.0,+648/-201,0.137,24.0,21.66,7.402,5.37,0.025,✔️,0.0,False,GPU +26,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1331.0,+404/-207,0.14,27.2,24.88,8.025,0.57,0.074,✔️,0.0,False,CPU +27,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1288.0,+318/-106,0.006,28.8,27.84,8.909,2.32,0.093,✔️,0.0,False,GPU +28,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1283.0,+418/-166,0.03,29.0,28.27,8.225,0.66,0.213,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1261.0,+222/-185,0.067,29.8,27.82,8.161,4108.24,0.479,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1234.0,+491/-233,0.028,30.8,28.77,9.603,6.54,0.284,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1212.0,+198/-242,0.032,31.6,29.67,9.253,4108.24,0.037,✔️,0.0,False,CPU +32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1166.0,+346/-383,0.034,33.2,31.32,10.288,35.47,0.502,✔️,0.0,False,CPU +33,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1137.0,+305/-327,0.017,34.2,32.95,10.454,35.47,0.054,✔️,0.0,False,CPU +34,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1137.0,+446/-460,0.121,34.2,25.03,10.329,8.68,7.525,➖,0.0,False,GPU +35,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1131.0,+406/-883,0.147,34.4,26.06,13.625,1820.58,0.133,✔️,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1131.0,+95/-238,0.0,34.4,34.11,10.053,79.96,0.571,✔️,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1122.0,+109/-229,0.0,34.7,34.23,10.217,79.96,0.053,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1050.0,+152/-316,0.0,37.0,35.77,13.469,14.05,0.029,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1024.0,+376/-520,0.017,37.8,35.52,12.12,0.18,0.054,✔️,0.0,False,CPU +40,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1010.0,+309/-975,0.104,38.2,35.39,14.242,1820.58,0.012,✔️,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+63/-264,0.0,38.5,38.32,11.35,0.42,0.053,✔️,0.0,False,CPU +42,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),982.0,+354/-1020,0.124,39.0,35.12,14.681,3.47,0.01,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),864.0,+81/-736,0.0,42.0,41.71,14.269,317.48,0.719,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),836.0,+84/-773,0.0,42.6,42.32,14.827,317.48,0.127,✔️,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),807.0,+90/-732,0.0,43.2,42.98,18.666,142.18,1.027,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),664.0,+109/-823,0.0,45.6,45.3,18.199,1.33,0.126,✔️,0.0,False,CPU +47,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),649.0,+52/-837,0.0,45.8,45.73,21.083,142.18,0.057,✔️,0.0,False,CPU +48,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),344.0,+125/-1382,0.0,48.6,48.5,28.294,0.06,0.036,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),133.0,+78/-1763,0.0,49.8,49.8,39.482,48.35,0.054,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),-121.0,+166/-2493,0.0,51.0,50.99,39.829,48.35,0.013,✔️,0.0,False,CPU +51,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),-274.0,+170/-3339,0.0,51.6,51.6,42.789,0.13,0.02,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/winrate_matrix.png.zip index b8231e7b9275afd25fb88d64a28b618780991445..cece3d0017c238f031031a8043759b0a50f13a17 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ba0514e7743c7311010aa6fec04e9d1b34a7d7154407f7b7426e411ef6a8997a -size 1244143 +oid sha256:126017f828f9e043f4ea4d39f06211eaff5fc79c03721011108bc3ca694a95b5 +size 1348875 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 47d852ab8d1d8ae7a9407552208074dbffe7a97d..a89a619fc6d2b582e0cff5803b1515a1a2dcb156 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bd0ee128f6b06efeaa559396818531af3b65b31b97991095ecaf88916cfe3631 -size 423977 +oid sha256:83e47ed1cdc91dcef183ec643d35184d7c94c6ff01726402372387a2ad9b933b +size 436201 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip index 621e4db931809c245018e8d83905782d02c14aa4..c7b9c94347f649f233d193aec4f06a86e5b09f3a 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7740a2d9aa2c96259cfec19b7fcb846aca999e815c9b3ca168ad332318f22aec -size 987556 +oid sha256:606e5538fb047380ec63dfd1fe9d20db7f0037c1b26b6ccf28ae58796f1ddd7c +size 1010309 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_small/tuning-impact-elo.png.zip index 97bd3e8b02f71b53dc1b7fa8762ce63099d2dbcd..bf0d5cd187f688f4b55b617f8355396b69b48ae9 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:873e422985707e325535b349258bf21efa0f50f79b57eb29f98dff677fd2122a -size 118758 +oid sha256:5fea143b01af2a563171720b2e40377035fcbcb85e3035732f10cabcfec3027e +size 124560 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_regression/datasets_small/website_leaderboard.csv index fc1524f33ce240dba0cf26c627b509cb6c741e77..32fcf1724149822757f940eb1e9ac38f56cb0457 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_small/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_small/website_leaderboard.csv @@ -1,52 +1,53 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1952.0,+273/-132,2.715,0.755,5.0,2.3,9830.47,232.266,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1921.0,+505/-228,1.16,0.779,5.62,2.08,2355.62,7.55,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1910.0,+208/-53,3.338,0.694,5.88,3.59,9830.47,22.221,✔️,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1904.0,+318/-109,1.968,0.67,6.0,3.22,716.17,4.181,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1759.0,+168/-71,4.753,0.55,9.88,6.3,51.27,43.085,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1751.0,+359/-200,2.625,0.571,10.12,3.72,2355.62,0.879,✔️,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1740.0,+162/-78,5.0,0.529,10.5,9.0,4918.07,35.607,✔️,0.0,False,GPU -7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1725.0,+173/-65,5.631,0.463,11.0,8.97,4919.72,5.372,✔️,0.0,False,CPU -8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1686.0,+347/-216,4.17,0.521,12.38,6.9,8.24,0.532,✔️,0.0,False,GPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1619.0,+208/-123,6.092,0.4,14.88,10.65,4918.07,0.949,✔️,0.0,False,GPU -10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1616.0,+120/-74,8.215,0.297,15.0,13.37,1206.7,4.573,✔️,0.0,False,CPU -11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1588.0,+151/-78,7.613,0.285,16.12,13.67,3495.99,1.462,✔️,0.0,False,CPU -12,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1569.0,+199/-136,9.41,0.384,16.88,5.32,9006.35,8.016,✔️,0.0,False,GPU -13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1554.0,+166/-105,7.724,0.32,17.5,13.2,1672.82,2.447,✔️,0.0,False,GPU -14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1551.0,+231/-103,7.664,0.265,17.62,12.79,3495.99,0.148,✔️,0.0,False,CPU -15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1551.0,+184/-110,6.098,0.295,17.62,14.33,4322.16,1.598,✔️,0.0,False,GPU -16,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1534.0,+445/-282,18.26,0.466,18.38,5.6,5.86,0.71,➖,0.0,False,GPU -17,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1509.0,+143/-86,8.02,0.259,19.5,15.34,1672.82,0.097,✔️,0.0,False,GPU -18,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1500.0,+108/-62,8.708,0.169,19.88,18.29,1277.27,0.648,✔️,0.0,False,CPU -19,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1495.0,+102/-52,8.776,0.163,20.12,19.02,1277.27,3.746,✔️,0.0,False,CPU -20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1476.0,+129/-93,9.405,0.156,21.0,18.63,1206.7,0.689,✔️,0.0,False,CPU -21,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1446.0,+144/-126,11.844,0.149,22.38,19.81,25.13,0.294,✔️,0.0,False,GPU -22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1446.0,+182/-115,6.829,0.196,22.38,18.18,4322.16,0.198,✔️,0.0,False,GPU -23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1415.0,+158/-132,10.49,0.112,23.75,19.24,10.03,0.115,✔️,0.0,False,CPU -24,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1372.0,+166/-175,13.514,0.136,25.62,20.38,644.11,0.953,✔️,0.0,False,CPU -25,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1358.0,+178/-127,10.391,0.124,26.25,21.31,22.41,7.82,✔️,0.0,False,GPU -26,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1350.0,+189/-198,13.918,0.167,26.62,19.91,644.11,0.187,✔️,0.0,False,CPU -27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1342.0,+194/-144,8.773,0.13,27.0,23.69,25.53,0.154,✔️,0.0,False,GPU -28,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1339.0,+195/-178,12.41,0.127,27.12,16.15,9006.35,0.527,✔️,0.0,False,GPU -29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1328.0,+144/-144,10.201,0.087,27.62,24.95,5194.73,1.75,✔️,0.0,False,CPU -30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1269.0,+207/-194,10.523,0.086,30.25,26.22,5194.73,0.121,✔️,0.0,False,CPU -31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1263.0,+232/-223,13.293,0.105,30.5,19.16,3991.28,0.402,✔️,0.0,False,CPU -32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1243.0,+248/-243,13.82,0.131,31.38,11.32,3991.28,0.034,✔️,0.0,False,CPU -33,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1228.0,+219/-289,15.422,0.121,32.0,11.27,7.0,1.362,✔️,0.0,False,GPU -34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1219.0,+112/-184,15.247,0.029,32.38,29.57,657.99,1.135,✔️,0.0,False,CPU -35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1204.0,+91/-98,12.599,0.009,33.0,32.16,4.54,0.374,✔️,0.0,False,CPU -36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1182.0,+69/-127,14.291,0.0,33.88,33.07,2150.84,7.995,✔️,0.0,False,CPU -37,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1169.0,+101/-179,15.473,0.019,34.38,32.23,0.46,0.117,✔️,0.0,False,CPU -38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1155.0,+114/-244,16.37,0.027,34.88,32.12,657.99,0.147,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1121.0,+95/-151,14.731,0.0,36.12,35.07,2150.84,0.566,✔️,0.0,False,CPU -40,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1114.0,+92/-103,13.89,0.0,36.38,35.62,4.08,0.255,✔️,0.0,False,CPU -41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1064.0,+125/-209,15.332,0.005,38.12,36.68,12.71,0.039,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1025.0,+148/-218,14.535,0.0,39.38,37.8,22.25,0.131,✔️,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+88/-203,17.797,0.0,40.12,39.6,0.69,0.125,✔️,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),978.0,+89/-211,19.209,0.0,40.75,40.09,6.77,0.511,✔️,0.0,False,CPU -45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),909.0,+148/-323,21.318,0.0,42.5,41.35,88.79,0.572,✔️,0.0,False,CPU -46,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),724.0,+133/-310,23.299,0.0,46.0,45.81,88.79,0.052,✔️,0.0,False,CPU -47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),642.0,+210/-603,29.645,0.0,47.12,46.49,0.43,0.047,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),525.0,+94/-492,34.167,0.0,48.44,48.34,742.99,0.364,✔️,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),485.0,+125/-572,34.251,0.0,48.81,48.71,742.99,0.097,✔️,0.0,False,CPU -50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),334.0,+100/-501,36.397,0.0,50.0,49.99,3.98,0.12,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1942.0,+276/-124,0.757,5.12,2.34,2.715,9830.47,232.266,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1912.0,+495/-226,0.78,5.75,2.08,1.16,2355.62,7.55,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1900.0,+199/-51,0.693,6.0,3.63,3.338,9830.47,22.221,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1895.0,+311/-98,0.67,6.12,3.24,1.968,716.17,4.181,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1750.0,+164/-70,0.554,10.12,6.38,4.753,51.27,43.085,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1742.0,+347/-200,0.575,10.38,3.87,2.625,2355.62,0.879,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1731.0,+155/-68,0.533,10.75,9.3,5.0,4918.07,35.607,✔️,0.0,False,GPU +7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1717.0,+173/-67,0.464,11.25,9.14,5.631,4919.72,5.372,✔️,0.0,False,CPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1680.0,+346/-214,0.526,12.62,6.96,4.17,8.24,0.532,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1612.0,+207/-119,0.403,15.25,10.84,6.092,4918.07,0.949,✔️,0.0,False,GPU +10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1609.0,+127/-78,0.296,15.38,13.59,8.215,1206.7,4.573,✔️,0.0,False,CPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1585.0,+146/-76,0.281,16.38,13.83,7.613,3495.99,1.462,✔️,0.0,False,CPU +12,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1564.0,+197/-135,0.384,17.25,5.33,9.41,9006.35,8.016,✔️,0.0,False,GPU +13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1550.0,+166/-103,0.321,17.88,13.36,7.724,1672.82,2.447,✔️,0.0,False,GPU +14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1547.0,+182/-112,0.298,18.0,14.5,6.098,4322.16,1.598,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1547.0,+224/-101,0.261,18.0,12.93,7.664,3495.99,0.148,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1533.0,+438/-274,0.463,18.62,5.73,18.26,5.86,0.71,➖,0.0,False,GPU +17,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1506.0,+140/-83,0.261,19.88,15.64,8.02,1672.82,0.097,✔️,0.0,False,GPU +18,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1495.0,+100/-59,0.169,20.38,18.8,8.708,1277.27,0.648,✔️,0.0,False,CPU +19,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1487.0,+96/-46,0.164,20.75,19.6,8.776,1277.27,3.746,✔️,0.0,False,CPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1471.0,+134/-94,0.162,21.5,18.91,9.405,1206.7,0.689,✔️,0.0,False,CPU +21,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1447.0,+226/-190,0.232,22.62,11.5,9.095,25.28,0.843,➖,0.0,False,GPU +22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1442.0,+180/-119,0.202,22.88,18.41,6.829,4322.16,0.198,✔️,0.0,False,GPU +23,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1439.0,+143/-124,0.151,23.0,20.26,11.844,25.13,0.294,✔️,0.0,False,GPU +24,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1410.0,+156/-136,0.117,24.38,19.53,10.49,10.03,0.115,✔️,0.0,False,CPU +25,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1372.0,+164/-175,0.137,26.12,20.75,13.514,644.11,0.953,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1355.0,+179/-126,0.121,26.88,22.21,10.391,22.41,7.82,✔️,0.0,False,GPU +27,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1350.0,+187/-196,0.168,27.12,20.29,13.918,644.11,0.187,✔️,0.0,False,CPU +28,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1342.0,+195/-147,0.138,27.5,23.94,8.773,25.53,0.154,✔️,0.0,False,GPU +29,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1339.0,+200/-177,0.133,27.62,16.27,12.41,9006.35,0.527,✔️,0.0,False,GPU +30,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1326.0,+139/-151,0.092,28.25,25.43,10.201,5194.73,1.75,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1269.0,+203/-198,0.09,30.88,26.82,10.523,5194.73,0.121,✔️,0.0,False,CPU +32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1264.0,+227/-218,0.103,31.12,19.39,13.293,3991.28,0.402,✔️,0.0,False,CPU +33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1241.0,+244/-237,0.129,32.12,11.41,13.82,3991.28,0.034,✔️,0.0,False,CPU +34,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1230.0,+217/-283,0.12,32.62,11.32,15.422,7.0,1.362,✔️,0.0,False,GPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1215.0,+106/-178,0.027,33.25,30.59,15.247,657.99,1.135,✔️,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1200.0,+95/-103,0.012,33.88,32.96,12.599,4.54,0.374,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1181.0,+70/-136,0.003,34.62,33.72,14.291,2150.84,7.995,✔️,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1165.0,+100/-177,0.019,35.25,33.22,15.473,0.46,0.117,✔️,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1152.0,+110/-242,0.025,35.75,33.13,16.37,657.99,0.147,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1122.0,+96/-157,0.0,36.88,35.76,14.731,2150.84,0.566,✔️,0.0,False,CPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1112.0,+89/-105,0.0,37.25,36.51,13.89,4.08,0.255,✔️,0.0,False,CPU +42,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1066.0,+124/-188,0.002,38.88,37.49,15.332,12.71,0.039,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1024.0,+150/-223,0.0,40.25,38.68,14.535,22.25,0.131,✔️,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+86/-205,0.0,41.0,40.48,17.797,0.69,0.125,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),978.0,+94/-219,0.0,41.62,40.92,19.209,6.77,0.511,✔️,0.0,False,CPU +46,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),906.0,+146/-323,0.0,43.5,42.39,21.318,88.79,0.572,✔️,0.0,False,CPU +47,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),721.0,+130/-303,0.0,47.0,46.81,23.299,88.79,0.052,✔️,0.0,False,CPU +48,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),639.0,+206/-596,0.0,48.12,47.51,29.645,0.43,0.047,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),522.0,+90/-479,0.0,49.44,49.34,34.167,742.99,0.364,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),482.0,+124/-560,0.0,49.81,49.71,34.251,742.99,0.097,✔️,0.0,False,CPU +51,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),331.0,+98/-494,0.0,51.0,50.99,36.397,3.98,0.12,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_small/winrate_matrix.png.zip index 1410ef502feefb27094e9ff11e77211a0336bf57..beff34b5aa09ba29601e780d2152b33b84307e8a 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f826ab381f065d3248028b0bbc3552b60f5a3bf71f7c92f76362bfc782c5db42 -size 1315140 +oid sha256:0f8894359822d9d0cddf1d570d89758608821140977bf0eeedceae5a0742c8f2 +size 1429390 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 5ad182eeb9a05fa150081417b4ac7eb07afe2e02..a5209a5f87a066d7052e841a1b496e78b26c4e07 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8c4aec408b169814300b4c22cc8e56d15b4f3aa39ed1e28467b0b3767b055186 -size 451328 +oid sha256:e2a53ffca2c217a528b6ea58795e03afdfa80594bace9aef1285c2425829f3a9 +size 464463 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip index a5ded5f8320c08bcdd63b908c95e1140690f16d3..4b2dd6afa4bf1f17c8b12664820182e6f5748b8c 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0607851f50acc1a266062ff0463e747f9b7cfa0c714039908a98e1b563a844bc -size 1042022 +oid sha256:65e19e63f332b83a2162efd0392f892fd79f2dab202da0f3ee39ec1453b2d2b9 +size 1064576 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip index e00559d02df73399558fb66d087e66ee329ac937..0e810766756046b7e4af95ea08b00fb7f8057f73 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:31036bfbeb65c3cd11f53cce5994d4e1400b5e4b2f5fa02bbbd3b31becb1aa77 -size 120889 +oid sha256:0db701eb9b57cbc80ee921964021f038cdca0fd4690c5fb5fdae1c5545715a50 +size 129465 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/website_leaderboard.csv index 601be121811987b8fb57ec8c68320a67aca39762..d28758bde1bc29286ff5ff9b8dda819634c69945 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/website_leaderboard.csv @@ -1,56 +1,57 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1906.0,+304/-113,3.095,0.726,6.43,2.55,10054.02,249.652,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1900.0,+513/-229,1.229,0.772,6.57,2.05,2279.61,6.732,✔️,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1884.0,+400/-86,1.908,0.683,7.0,3.48,669.82,4.041,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1873.0,+180/-50,3.69,0.671,7.29,4.98,10054.02,28.583,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1780.0,+499/-189,2.584,0.597,10.14,3.32,6601.21,73.561,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1772.0,+224/-44,4.944,0.565,10.43,6.19,47.81,37.998,✔️,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1756.0,+146/-30,5.167,0.551,11.0,9.95,4840.44,33.436,✔️,0.0,False,GPU -7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1729.0,+403/-218,2.847,0.545,12.0,3.79,2279.61,0.781,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1714.0,+373/-213,4.191,0.554,12.57,7.37,7.82,0.519,✔️,0.0,False,GPU -9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1693.0,+192/-49,6.056,0.444,13.43,10.94,5390.56,5.054,✔️,0.0,False,CPU -10,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1658.0,+220/-118,6.637,0.398,14.86,7.71,105.28,2.433,✔️,0.0,False,GPU -11,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1624.0,+410/-299,19.429,0.518,16.29,5.15,5.73,0.638,➖,0.0,False,GPU -12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1605.0,+159/-48,7.956,0.287,17.14,15.34,3301.07,1.317,✔️,0.0,False,CPU -13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1605.0,+257/-127,6.481,0.39,17.14,12.18,4840.44,0.706,✔️,0.0,False,GPU -14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1574.0,+114/-57,9.035,0.246,18.57,17.2,997.12,3.787,✔️,0.0,False,CPU -15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1568.0,+226/-106,8.008,0.266,18.86,14.48,3301.07,0.125,✔️,0.0,False,CPU -16,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1558.0,+418/-227,5.384,0.406,19.29,10.76,6601.21,0.588,✔️,0.0,False,GPU -17,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1544.0,+165/-105,8.299,0.295,20.0,14.25,1709.03,2.724,✔️,0.0,False,GPU -18,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1529.0,+188/-118,6.513,0.267,20.71,17.7,4228.53,1.447,✔️,0.0,False,GPU -19,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1515.0,+180/-138,10.909,0.267,21.43,15.3,9360.97,7.658,✔️,0.0,False,GPU -20,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1512.0,+142/-77,8.482,0.251,21.57,16.24,1709.03,0.097,✔️,0.0,False,GPU -21,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1469.0,+405/-336,7.531,0.352,23.71,13.17,8.97,0.837,✔️,0.0,False,GPU -22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1466.0,+83/-44,9.543,0.098,23.86,23.13,911.22,0.527,✔️,0.0,False,CPU -23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1463.0,+78/-38,9.588,0.1,24.0,23.39,911.22,3.532,✔️,0.0,False,CPU -24,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1434.0,+126/-68,10.356,0.105,25.43,23.65,997.12,0.636,✔️,0.0,False,CPU -25,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1425.0,+196/-137,7.312,0.16,25.86,21.99,4228.53,0.173,✔️,0.0,False,GPU -26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1422.0,+220/-139,11.174,0.117,26.0,21.17,9.39,0.105,✔️,0.0,False,CPU -27,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1407.0,+139/-152,13.106,0.121,26.71,24.25,27.26,0.29,✔️,0.0,False,GPU -28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1392.0,+209/-236,14.555,0.158,27.43,21.57,613.95,0.952,✔️,0.0,False,CPU -29,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1384.0,+236/-250,14.881,0.193,27.86,20.58,613.95,0.135,✔️,0.0,False,CPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1366.0,+155/-135,11.015,0.129,28.71,23.97,22.96,8.006,✔️,0.0,False,GPU -31,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1325.0,+195/-187,9.406,0.114,30.71,27.44,17.97,0.145,✔️,0.0,False,GPU -32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1290.0,+275/-208,14.054,0.111,32.43,20.76,3778.28,0.323,✔️,0.0,False,CPU -33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1282.0,+124/-160,11.205,0.026,32.86,31.52,5937.36,1.406,✔️,0.0,False,CPU -34,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1276.0,+260/-286,16.185,0.138,33.14,10.38,7.12,1.138,✔️,0.0,False,GPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1272.0,+275/-228,14.636,0.14,33.29,14.11,3778.28,0.028,✔️,0.0,False,CPU -36,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1272.0,+146/-174,14.048,0.037,33.29,30.98,9360.97,0.522,✔️,0.0,False,GPU -37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1229.0,+155/-260,16.471,0.03,35.29,32.39,628.51,0.812,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1222.0,+185/-280,11.535,0.031,35.57,32.35,5937.36,0.119,✔️,0.0,False,CPU -39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1175.0,+138/-220,16.642,0.02,37.57,35.46,0.43,0.117,✔️,0.0,False,CPU -40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1173.0,+176/-289,17.646,0.029,37.64,34.8,628.51,0.149,✔️,0.0,False,CPU -41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1172.0,+59/-100,13.734,0.0,37.71,37.36,4.41,0.363,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1150.0,+55/-106,15.611,0.0,38.57,38.08,2163.39,7.622,✔️,0.0,False,CPU -43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1114.0,+109/-169,14.895,0.0,40.0,39.21,3.97,0.24,✔️,0.0,False,CPU -44,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1095.0,+89/-158,15.956,0.0,40.71,39.47,10.54,0.036,✔️,0.0,False,CPU -45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1095.0,+106/-175,16.062,0.0,40.71,39.78,2163.39,0.713,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1002.0,+135/-301,15.699,0.0,43.86,42.36,20.5,0.13,✔️,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+131/-217,19.276,0.0,43.93,43.18,0.67,0.118,✔️,0.0,False,CPU -48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),943.0,+77/-213,21.065,0.0,45.57,45.12,6.88,0.452,✔️,0.0,False,CPU -49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),937.0,+178/-381,22.177,0.0,45.71,44.31,86.71,0.53,✔️,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),757.0,+168/-358,24.152,0.0,49.57,49.27,86.71,0.05,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),712.0,+245/-524,30.287,0.0,50.29,49.48,0.4,0.044,✔️,0.0,False,CPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),533.0,+85/-893,36.2,0.0,52.5,52.4,753.48,0.227,✔️,0.0,False,CPU -53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),504.0,+117/-984,36.249,0.0,52.79,52.68,753.48,0.091,✔️,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),352.0,+97/-674,37.99,0.0,54.0,53.99,4.35,0.119,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1905.0,+308/-104,0.729,6.57,2.58,3.095,10054.02,249.652,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1899.0,+520/-230,0.771,6.71,2.05,1.229,2279.61,6.732,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1883.0,+380/-88,0.679,7.14,3.5,1.908,669.82,4.041,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1873.0,+173/-48,0.67,7.43,5.05,3.69,10054.02,28.583,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1782.0,+493/-185,0.593,10.29,3.32,2.584,6601.21,73.561,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1770.0,+209/-45,0.567,10.71,6.24,4.944,47.81,37.998,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1755.0,+135/-30,0.551,11.29,10.2,5.167,4840.44,33.436,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1729.0,+411/-216,0.545,12.29,3.89,2.847,2279.61,0.781,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1715.0,+376/-211,0.555,12.86,7.43,4.191,7.82,0.519,✔️,0.0,False,GPU +9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1694.0,+186/-51,0.443,13.71,11.1,6.056,5390.56,5.054,✔️,0.0,False,CPU +10,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),1653.0,+222/-120,0.391,15.43,7.88,6.637,105.28,2.433,✔️,0.0,False,GPU +11,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1627.0,+414/-295,0.518,16.57,5.24,19.429,5.73,0.638,➖,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1608.0,+152/-48,0.282,17.43,15.49,7.956,3301.07,1.317,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1605.0,+238/-130,0.388,17.57,12.35,6.481,4840.44,0.706,✔️,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1575.0,+117/-58,0.243,19.0,17.49,9.035,997.12,3.787,✔️,0.0,False,CPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1569.0,+232/-107,0.267,19.29,14.63,8.008,3301.07,0.125,✔️,0.0,False,CPU +16,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1560.0,+421/-226,0.401,19.71,10.82,5.384,6601.21,0.588,✔️,0.0,False,GPU +17,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1546.0,+164/-104,0.292,20.43,14.4,8.299,1709.03,2.724,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1532.0,+189/-120,0.268,21.14,17.91,6.513,4228.53,1.447,✔️,0.0,False,GPU +19,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1521.0,+202/-160,0.242,21.71,13.12,8.659,22.8,0.769,➖,0.0,False,GPU +20,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1518.0,+181/-140,0.27,21.86,15.39,10.909,9360.97,7.658,✔️,0.0,False,GPU +21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1516.0,+143/-78,0.248,22.0,16.51,8.482,1709.03,0.097,✔️,0.0,False,GPU +22,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1474.0,+406/-336,0.348,24.14,13.31,7.531,8.97,0.837,✔️,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1468.0,+79/-43,0.096,24.43,23.78,9.543,911.22,0.527,✔️,0.0,False,CPU +24,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1462.0,+77/-40,0.098,24.71,24.12,9.588,911.22,3.532,✔️,0.0,False,CPU +25,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1437.0,+130/-69,0.11,26.0,24.03,10.356,997.12,0.636,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1428.0,+197/-140,0.164,26.43,22.29,7.312,4228.53,0.173,✔️,0.0,False,GPU +27,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1422.0,+221/-142,0.122,26.71,21.49,11.174,9.39,0.105,✔️,0.0,False,CPU +28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1408.0,+144/-154,0.12,27.43,24.82,13.106,27.26,0.29,✔️,0.0,False,GPU +29,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1396.0,+209/-230,0.156,28.0,21.91,14.555,613.95,0.952,✔️,0.0,False,CPU +30,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1388.0,+235/-247,0.191,28.43,20.91,14.881,613.95,0.135,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1368.0,+154/-136,0.127,29.43,24.9,11.015,22.96,8.006,✔️,0.0,False,GPU +32,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1331.0,+195/-190,0.121,31.29,27.74,9.406,17.97,0.145,✔️,0.0,False,GPU +33,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1294.0,+275/-209,0.111,33.14,21.0,14.054,3778.28,0.323,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1285.0,+128/-166,0.03,33.57,32.19,11.205,5937.36,1.406,✔️,0.0,False,CPU +35,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1280.0,+261/-291,0.136,33.86,10.42,16.185,7.12,1.138,✔️,0.0,False,GPU +36,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1280.0,+151/-173,0.039,33.86,31.37,14.048,9360.97,0.522,✔️,0.0,False,GPU +37,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1274.0,+268/-232,0.138,34.14,14.24,14.636,3778.28,0.028,✔️,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1227.0,+153/-256,0.024,36.29,33.49,16.471,628.51,0.812,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1227.0,+179/-286,0.03,36.29,33.1,11.535,5937.36,0.119,✔️,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1174.0,+135/-216,0.016,38.57,36.52,16.642,0.43,0.117,✔️,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1172.0,+173/-290,0.023,38.64,35.88,17.646,628.51,0.149,✔️,0.0,False,CPU +42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1170.0,+58/-102,0.0,38.71,38.37,13.734,4.41,0.363,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1153.0,+60/-112,0.0,39.43,38.89,15.611,2163.39,7.622,✔️,0.0,False,CPU +44,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1113.0,+107/-172,0.0,41.0,40.23,14.895,3.97,0.24,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1098.0,+112/-178,0.0,41.57,40.6,16.062,2163.39,0.713,✔️,0.0,False,CPU +46,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1098.0,+94/-161,0.0,41.57,40.32,15.956,10.54,0.036,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1002.0,+133/-301,0.0,44.86,43.39,15.699,20.5,0.13,✔️,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+129/-212,0.0,44.93,44.2,19.276,0.67,0.118,✔️,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),943.0,+77/-214,0.0,46.57,46.13,21.065,6.88,0.452,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),937.0,+178/-378,0.0,46.71,45.35,22.177,86.71,0.53,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),757.0,+167/-356,0.0,50.57,50.27,24.152,86.71,0.05,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),712.0,+244/-528,0.0,51.29,50.5,30.287,0.4,0.044,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),533.0,+85/-888,0.0,53.5,53.4,36.2,753.48,0.227,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),504.0,+117/-993,0.0,53.79,53.68,36.249,753.48,0.091,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),352.0,+97/-818,0.0,55.0,54.99,37.99,4.35,0.119,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip index 76812a3f8fcb0d889d0a241484d2848ca61baa76..15c046f09724f326400a724e1cea5f9a47bd94e6 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a856b5adb7cb96846edc8c57da779beb279e2fa34af2caa3ebc687d8e33f0f98 -size 1396925 +oid sha256:ca485e4837c55ad7eea2d12a1c16b9c5698986e6cbb90c75d24c2feb16d24391 +size 1515219 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index f705ae9b51d59fd19bfe4a382b3898f3b120ceab..55c28958fd4383553e0f1d46a2fbda2b88c5cea2 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fa4f1a079bb2f2d0d3d8caf839b427701975844b55b41d11bf3932389880707e -size 496056 +oid sha256:2139836ff27309019dfa992bafdd630f92f7c9c2cf4234b07737c2deb20f2473 +size 513819 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_all/pareto_n_configs_imp.png.zip index 37a7682dd0fcf096a309a796ca41d538689b0453..4b42ecd107084489d665dd075a4c088585a90dac 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3af29ee92648265e44296ccd3884f79e0d994bf6d8218bb80b183ac19610586e -size 1071506 +oid sha256:69d0e5e543e597174456be25b458c5ae251138a883ce9725f9754d61a18edee8 +size 1044560 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_all/tuning-impact-elo.png.zip index da19d9f921a2af8c53c7ecf269ec0492c8041a34..bf6c945da83298dc5547456e78d4cd059f9008c3 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:89d2d49b05e0c90ecf46b2eab93457b446865302c70503e7e4c1648beb725fe4 -size 177418 +oid sha256:a6bd692a0df15771417ad9654d32b1166d6a1b3869c48e92b589df1a7253b8c8 +size 181173 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_all/datasets_all/website_leaderboard.csv index 133eba95641b4653bc2ec2a9002114e2d00af0b3..751bacea36ab65c945693786e3080e211fd2be4f 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_all/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_all/datasets_all/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1636.0,+60/-48,5.839,0.753,8.48,3.84,582.21,6.116,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1599.0,+78/-57,5.088,0.716,9.88,3.37,2040.22,8.92,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1554.0,+67/-50,6.062,0.658,11.83,4.65,2040.22,1.218,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1523.0,+66/-46,6.766,0.628,13.31,6.19,5.81,0.645,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1499.0,+56/-46,9.055,0.522,14.53,8.14,2950.72,11.988,✔️,0.0,False,GPU -5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1499.0,+55/-47,9.005,0.517,14.56,5.99,1735.72,2.559,✔️,0.0,False,CPU -6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1451.0,+62/-49,9.138,0.496,17.21,5.14,4907.64,286.652,✔️,0.0,False,GPU -7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1435.0,+50/-38,10.535,0.399,18.15,9.57,3285.87,1.472,✔️,0.0,False,GPU -8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1419.0,+32/-29,11.754,0.35,19.13,12.78,416.98,2.639,✔️,0.0,False,CPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1416.0,+47/-46,10.535,0.365,19.32,10.24,2950.72,0.663,✔️,0.0,False,GPU -10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1403.0,+42/-39,11.264,0.334,20.12,11.5,1658.41,0.653,✔️,0.0,False,CPU -11,🧠⚡,Foundation Model,[LimiX (default) [11.98% IMPUTED]](https://arxiv.org/abs/2509.03505),1402.0,+67/-55,12.97,0.464,20.2,6.37,3.68,0.598,➖,11.98,True,GPU -12,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1396.0,+57/-53,10.533,0.402,20.54,7.13,4907.64,39.96,✔️,0.0,False,GPU -13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1392.0,+40/-41,11.447,0.324,20.8,10.82,1658.41,0.081,✔️,0.0,False,CPU -14,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1382.0,+75/-52,11.746,0.371,21.43,8.49,4621.67,8.149,✔️,0.0,False,GPU -15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1379.0,+48/-44,11.448,0.316,21.65,11.04,3285.87,0.173,✔️,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1374.0,+31/-29,12.443,0.265,21.92,15.54,416.98,0.334,✔️,0.0,False,CPU -17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1364.0,+33/-38,12.518,0.28,22.57,14.36,693.49,1.69,✔️,0.0,False,CPU -18,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1354.0,+38/-43,12.323,0.29,23.24,12.84,6.83,0.08,✔️,0.0,False,CPU -19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1348.0,+40/-37,12.287,0.245,23.61,12.01,4621.67,0.475,✔️,0.0,False,GPU -20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1343.0,+51/-39,12.886,0.253,23.97,14.02,846.88,2.552,✔️,0.0,False,GPU -21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1342.0,+32/-34,12.838,0.23,24.04,16.93,693.49,0.308,✔️,0.0,False,CPU -22,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1338.0,+65/-65,12.658,0.382,24.27,8.06,3030.15,21.444,✔️,35.29,True,GPU -23,🧠⚡,Foundation Model,[Mitra (default) [35.29% IMPUTED]](https://arxiv.org/abs/2510.21204),1326.0,+62/-62,13.408,0.36,25.11,8.78,87.65,2.505,✔️,35.29,True,GPU -24,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1320.0,+68/-55,13.232,0.326,25.48,8.32,47.62,43.739,✔️,0.0,False,GPU -25,🧠⚡,Foundation Model,[TabICL (default) [29.41% IMPUTED]](https://arxiv.org/abs/2502.05564),1317.0,+51/-57,12.932,0.318,25.66,9.42,6.63,1.48,✔️,29.41,True,GPU -26,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1295.0,+42/-37,14.176,0.188,27.14,16.97,846.88,0.13,✔️,0.0,False,GPU -27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1287.0,+47/-44,13.952,0.219,27.69,16.45,10.49,0.132,✔️,0.0,False,GPU -28,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1281.0,+62/-58,14.316,0.285,28.06,9.86,3030.15,0.462,✔️,35.29,True,GPU -29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1280.0,+47/-45,13.735,0.153,28.17,19.25,2874.67,1.952,✔️,0.0,False,CPU -30,🧠⚡,Foundation Model,[BetaTabPFN (default) [25.49% IMPUTED]](https://arxiv.org/abs/2502.02527),1274.0,+56/-50,15.319,0.232,28.59,15.28,205.87,1.343,➖,25.49,True,GPU -31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1259.0,+40/-38,15.485,0.149,29.58,19.57,2929.85,0.422,✔️,0.0,False,CPU -32,🧠⚡,Foundation Model,[TabPFNv2 (default) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1254.0,+66/-61,15.267,0.259,29.86,13.15,3.36,0.313,✔️,35.29,True,GPU -33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1246.0,+39/-40,15.853,0.122,30.46,19.1,14.87,0.308,✔️,0.0,False,GPU -34,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1226.0,+35/-38,15.245,0.082,31.77,22.77,10.06,1.689,✔️,0.0,False,GPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1223.0,+44/-45,16.161,0.106,31.95,20.88,2929.85,0.051,✔️,0.0,False,CPU -36,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1210.0,+38/-40,15.708,0.091,32.81,24.48,1.94,0.123,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1205.0,+43/-44,15.593,0.08,33.15,23.23,2874.67,0.131,✔️,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1205.0,+42/-41,17.165,0.094,33.16,22.18,183.02,0.761,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1202.0,+48/-50,16.636,0.12,33.37,20.04,593.24,4.467,✔️,0.0,False,CPU -40,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1192.0,+44/-47,17.32,0.097,34.0,22.56,7.32,0.05,✔️,0.0,False,CPU -41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1183.0,+31/-30,16.331,0.058,34.57,29.77,1.96,0.142,✔️,0.0,False,CPU -42,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1177.0,+50/-49,18.028,0.074,34.94,22.57,373.18,0.771,✔️,0.0,False,CPU -43,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1168.0,+43/-46,18.131,0.07,35.51,22.57,183.02,0.091,✔️,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1141.0,+48/-52,18.128,0.05,37.19,24.44,593.24,0.306,✔️,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1138.0,+49/-40,18.798,0.049,37.41,22.15,373.18,0.085,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1072.0,+36/-46,19.966,0.009,41.25,33.48,9.99,0.126,✔️,0.0,False,CPU -47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1039.0,+59/-64,23.528,0.034,42.99,34.35,3.23,0.919,✔️,0.0,False,GPU -48,🧠⚡,Foundation Model,[TabFlex (default) [25.49% IMPUTED]](https://arxiv.org/abs/2506.05584),1008.0,+61/-75,25.129,0.045,44.54,31.75,0.79,0.123,➖,25.49,True,GPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1001.0,+53/-65,22.986,0.009,44.88,33.0,2.86,0.373,✔️,0.0,False,CPU -50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+44/-40,23.738,0.004,44.93,40.01,0.43,0.054,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),993.0,+62/-80,25.602,0.018,45.25,31.89,129.01,1.802,✔️,0.0,False,CPU -52,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),978.0,+58/-62,25.248,0.011,45.94,40.23,0.25,0.05,✔️,0.0,False,CPU -53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),961.0,+64/-98,31.513,0.031,46.69,35.2,237.58,0.424,✔️,0.0,False,CPU -54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),933.0,+72/-114,32.121,0.02,47.87,37.08,237.58,0.085,✔️,0.0,False,CPU -55,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),882.0,+62/-98,30.831,0.007,49.77,38.19,129.01,0.18,✔️,0.0,False,CPU -56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),851.0,+70/-120,34.849,0.012,50.8,40.6,1.19,0.12,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),627.0,+89/-100,44.451,0.0,55.71,50.58,0.19,0.036,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1633.0,+61/-47,0.75,8.66,3.87,5.993,582.21,6.116,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1597.0,+78/-57,0.715,10.06,3.43,5.284,2040.22,8.92,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1552.0,+67/-49,0.657,12.04,4.7,6.257,2040.22,1.218,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1522.0,+65/-47,0.624,13.49,6.42,6.977,5.81,0.645,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1497.0,+55/-46,0.517,14.78,8.32,9.225,2950.72,11.988,✔️,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1496.0,+54/-46,0.516,14.86,6.07,9.101,1735.72,2.559,✔️,0.0,False,CPU +6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1449.0,+62/-48,0.492,17.52,5.18,9.255,4907.64,286.652,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1433.0,+50/-39,0.396,18.47,9.67,10.651,3285.87,1.472,✔️,0.0,False,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1417.0,+32/-29,0.35,19.48,12.92,11.845,416.98,2.639,✔️,0.0,False,CPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1414.0,+48/-46,0.365,19.66,10.44,10.688,2950.72,0.663,✔️,0.0,False,GPU +10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1401.0,+41/-38,0.333,20.45,11.62,11.41,1658.41,0.653,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[LimiX (default) [11.98% IMPUTED]](https://arxiv.org/abs/2509.03505),1401.0,+67/-54,0.458,20.49,6.49,13.153,3.68,0.598,➖,11.98,True,GPU +12,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1394.0,+57/-52,0.402,20.92,7.21,10.633,4907.64,39.96,✔️,0.0,False,GPU +13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1390.0,+40/-39,0.323,21.15,10.92,11.592,1658.41,0.081,✔️,0.0,False,CPU +14,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1380.0,+74/-52,0.371,21.81,8.57,11.858,4621.67,8.149,✔️,0.0,False,GPU +15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1377.0,+48/-45,0.316,22.04,11.16,11.555,3285.87,0.173,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1372.0,+31/-29,0.266,22.31,15.73,12.535,416.98,0.334,✔️,0.0,False,CPU +17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1363.0,+33/-39,0.279,22.96,14.54,12.624,693.49,1.69,✔️,0.0,False,CPU +18,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1352.0,+38/-44,0.288,23.65,12.97,12.405,6.83,0.08,✔️,0.0,False,CPU +19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1347.0,+41/-38,0.247,24.02,12.19,12.39,4621.67,0.475,✔️,0.0,False,GPU +20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1341.0,+50/-39,0.251,24.42,14.33,13.02,846.88,2.552,✔️,0.0,False,GPU +21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1340.0,+33/-34,0.228,24.44,17.14,12.947,693.49,0.308,✔️,0.0,False,CPU +22,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1336.0,+64/-64,0.377,24.71,8.32,12.798,3030.15,21.444,✔️,35.29,True,GPU +23,🧠🔁,Neural Network,[Mitra (default) [35.29% IMPUTED]](https://arxiv.org/abs/2510.21204),1324.0,+61/-63,0.355,25.55,8.97,13.51,87.65,2.505,✔️,35.29,True,GPU +24,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1318.0,+69/-55,0.326,25.94,8.42,13.296,47.62,43.739,✔️,0.0,False,GPU +25,🧠⚡,Foundation Model,[TabICL (default) [29.41% IMPUTED]](https://arxiv.org/abs/2502.05564),1316.0,+51/-57,0.315,26.09,9.51,13.032,6.63,1.48,✔️,29.41,True,GPU +26,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1294.0,+43/-37,0.187,27.58,17.21,14.303,846.88,0.13,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1286.0,+47/-44,0.217,28.17,17.06,14.039,10.49,0.132,✔️,0.0,False,GPU +28,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1280.0,+62/-57,0.279,28.55,10.07,14.445,3030.15,0.462,✔️,35.29,True,GPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1278.0,+47/-46,0.156,28.67,19.48,13.813,2874.67,1.952,✔️,0.0,False,CPU +30,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1277.0,+54/-48,0.24,28.78,10.81,14.682,14.11,2.075,➖,0.0,False,GPU +31,🧠⚡,Foundation Model,[BetaTabPFN (default) [25.49% IMPUTED]](https://arxiv.org/abs/2502.02527),1273.0,+56/-50,0.231,29.07,15.51,15.426,205.87,1.343,➖,25.49,True,GPU +32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1258.0,+39/-38,0.148,30.09,19.95,15.578,2929.85,0.422,✔️,0.0,False,CPU +33,🧠⚡,Foundation Model,[TabPFNv2 (default) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1253.0,+67/-61,0.25,30.41,13.54,15.448,3.36,0.313,✔️,35.29,True,GPU +34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1244.0,+39/-40,0.121,31.0,19.49,15.933,14.87,0.308,✔️,0.0,False,GPU +35,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1224.0,+35/-38,0.083,32.37,23.36,15.32,10.06,1.689,✔️,0.0,False,GPU +36,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1222.0,+44/-46,0.106,32.5,21.18,16.254,2929.85,0.051,✔️,0.0,False,CPU +37,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1210.0,+38/-40,0.092,33.35,24.74,15.79,1.94,0.123,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1204.0,+43/-44,0.083,33.72,23.52,15.658,2874.67,0.131,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1204.0,+42/-41,0.093,33.74,22.49,17.281,183.02,0.761,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1200.0,+48/-50,0.12,34.01,20.41,16.701,593.24,4.467,✔️,0.0,False,CPU +41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1191.0,+44/-47,0.095,34.62,23.04,17.399,7.32,0.05,✔️,0.0,False,CPU +42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1182.0,+31/-31,0.058,35.17,30.19,16.397,1.96,0.142,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1176.0,+49/-48,0.073,35.54,22.99,18.166,373.18,0.771,✔️,0.0,False,CPU +44,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1167.0,+42/-46,0.069,36.13,22.97,18.239,183.02,0.091,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1139.0,+48/-52,0.05,37.91,25.04,18.189,593.24,0.306,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1137.0,+50/-40,0.047,38.03,23.32,18.935,373.18,0.085,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1071.0,+36/-46,0.01,41.98,34.03,20.041,9.99,0.126,✔️,0.0,False,CPU +48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1039.0,+58/-64,0.033,43.72,35.19,23.641,3.23,0.919,✔️,0.0,False,GPU +49,🧠⚡,Foundation Model,[TabFlex (default) [25.49% IMPUTED]](https://arxiv.org/abs/2506.05584),1008.0,+60/-74,0.045,45.34,32.41,25.178,0.79,0.123,➖,25.49,True,GPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1000.0,+55/-65,0.009,45.7,33.85,23.034,2.86,0.373,✔️,0.0,False,CPU +51,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+44/-40,0.003,45.71,41.04,23.811,0.43,0.054,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),993.0,+61/-78,0.019,46.05,32.33,25.658,129.01,1.802,✔️,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),978.0,+58/-61,0.011,46.74,41.01,25.307,0.25,0.05,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),960.0,+65/-98,0.031,47.56,35.79,31.592,237.58,0.424,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),932.0,+71/-112,0.021,48.75,37.75,32.184,237.58,0.085,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),881.0,+60/-96,0.008,50.67,38.8,30.879,129.01,0.18,✔️,0.0,False,CPU +57,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),850.0,+70/-117,0.012,51.71,41.24,34.895,1.19,0.12,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),625.0,+89/-101,0.0,56.7,51.45,44.484,0.19,0.036,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_all/winrate_matrix.png.zip index 0d257c5d2ab9327363b765ad4577222c487c3971..afd9d6ce27ce542f093ec1e08063b3193cc17594 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bd478c61c99e2a708ec628c9c588c5486306797858b1ec77f23f659c86f64b46 -size 2772353 +oid sha256:5f84700fdc98572a52e8bda0c3e9c3ddfeee7e2a48d934d3a2258d64afad9ac8 +size 2783485 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index 93f96399fb45917145243941d1f60bc69d16ba4b..6ed6bd06e341d74cdda566a52acc70bb857dd013 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:50ddacad8d527054ff4c75b7c0ecf25903a0849b377fd321379506a41e66c76f -size 450547 +oid sha256:5edafba46aafb690e87aa8c9eb1c77b613fbf22438354b63054f245e7e5c6d5e +size 464893 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip index 868ab6237a6adedd8d092bc8b567cb3e0140444f..f02632ef92164879bbe7266b7fbf71b8c0bb2414 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1a868df62aa7c9168d0b2c36f553073e7588803430d89b3b1b245832771a9470 -size 999272 +oid sha256:4e8cd6e8dc77f441cf4c9adca27be37387136e60f127b7f1d8a7599b27ffecbe +size 959526 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_medium/tuning-impact-elo.png.zip index 6bc982968639695a5feb05cc43f5eb3a07849a74..51990964cc3cd7a664e144b70289dbc9af74a1bf 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b847cc585de762cd70bc6171886abdb1ebfab1e9b4043a14d2fc4c42b1cd33a7 -size 142000 +oid sha256:79a0670abf2769c40b90952889a07e28737fd89b596966ab0f719f0c8d99fc45 +size 146845 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_all/datasets_medium/website_leaderboard.csv index 1c8a716163e8f3cfcba6d39b9ab23880e483014a..8b46f9afd3f29a1258e0030b5733c2ddc897b0ef 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_medium/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_all/datasets_medium/website_leaderboard.csv @@ -1,55 +1,56 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1809.0,+94/-49,2.62,0.822,6.04,2.88,289.53,3.187,✔️,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1784.0,+87/-53,2.723,0.795,6.69,3.13,432.35,4.085,✔️,0.0,False,CPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1662.0,+163/-119,5.438,0.649,10.68,3.35,735.58,11.736,✔️,0.0,False,GPU -3,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1632.0,+104/-90,5.74,0.576,11.83,7.36,1719.82,1.675,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1620.0,+120/-113,5.81,0.593,12.33,6.83,2526.28,6.013,✔️,0.0,False,GPU -5,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1607.0,+90/-64,5.642,0.536,12.89,6.9,777.59,0.25,✔️,0.0,False,CPU -6,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1606.0,+146/-108,6.152,0.582,12.92,5.31,735.58,1.386,✔️,0.0,False,GPU -7,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1598.0,+97/-65,5.838,0.538,13.3,5.73,777.59,0.052,✔️,0.0,False,CPU -8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1590.0,+75/-59,7.405,0.488,13.64,9.18,131.56,2.639,✔️,0.0,False,CPU -9,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1557.0,+110/-95,6.028,0.521,15.15,7.44,3.24,0.031,✔️,0.0,False,CPU -10,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1549.0,+87/-56,7.531,0.442,15.51,9.32,282.13,0.563,✔️,0.0,False,CPU -11,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1536.0,+146/-95,6.964,0.471,16.17,7.91,1.88,0.645,✔️,0.0,False,GPU -12,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1533.0,+114/-88,7.07,0.406,16.29,9.19,1719.82,0.081,✔️,0.0,False,GPU -13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1531.0,+149/-89,7.466,0.424,16.41,7.54,1993.14,0.619,✔️,0.0,False,GPU -14,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1519.0,+130/-121,7.078,0.482,16.97,4.24,4786.55,444.544,✔️,0.0,False,GPU -15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1502.0,+78/-60,8.625,0.346,17.84,12.32,131.56,0.132,✔️,0.0,False,CPU -16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1501.0,+70/-49,8.115,0.359,17.86,13.6,282.13,0.066,✔️,0.0,False,CPU -17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1479.0,+145/-95,8.222,0.358,18.99,8.67,1993.14,0.061,✔️,0.0,False,GPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1473.0,+104/-87,8.204,0.33,19.32,13.86,2526.28,0.349,✔️,0.0,False,GPU -19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1427.0,+112/-93,9.726,0.282,21.7,13.1,566.74,0.951,✔️,0.0,False,GPU -20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1397.0,+123/-120,9.704,0.296,23.29,8.88,4786.55,42.641,✔️,0.0,False,GPU -21,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1375.0,+70/-65,9.473,0.177,24.49,17.88,1770.56,1.069,✔️,0.0,False,CPU -22,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1374.0,+96/-92,11.424,0.206,24.56,19.3,566.74,0.06,✔️,0.0,False,GPU -23,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1366.0,+101/-111,9.978,0.228,24.99,16.75,0.49,0.053,✔️,0.0,False,CPU -24,🧠⚡,Foundation Model,[TabICL (default) [33.33% IMPUTED]](https://arxiv.org/abs/2502.05564),1341.0,+131/-109,10.315,0.276,26.37,12.29,3.97,1.596,✔️,33.33,True,GPU -25,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1328.0,+120/-136,12.499,0.225,27.08,10.23,46.62,43.739,✔️,0.0,False,GPU -26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1322.0,+132/-105,11.062,0.181,27.38,17.94,5.16,0.074,✔️,0.0,False,GPU -27,🧠⚡,Foundation Model,[LimiX (default) [40.00% IMPUTED]](https://arxiv.org/abs/2509.03505),1322.0,+138/-128,14.019,0.25,27.39,8.86,1.08,0.32,➖,40.0,True,GPU -28,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1320.0,+118/-126,12.974,0.193,27.51,18.45,1517.65,0.134,✔️,0.0,False,CPU -29,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1296.0,+76/-79,11.512,0.111,28.81,24.71,8.06,0.25,✔️,0.0,False,GPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1274.0,+85/-73,12.26,0.05,29.99,26.82,6.75,0.235,✔️,0.0,False,GPU -31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1269.0,+78/-77,11.256,0.064,30.26,25.57,1770.56,0.051,✔️,0.0,False,CPU -32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1259.0,+112/-138,13.864,0.136,30.76,23.35,1517.65,0.015,✔️,0.0,False,CPU -33,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1257.0,+57/-67,11.913,0.045,30.9,28.62,0.29,0.045,✔️,0.0,False,CPU -34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1249.0,+82/-72,12.464,0.041,31.29,25.01,66.0,0.341,✔️,0.0,False,CPU -35,🧠⚡,Foundation Model,[BetaTabPFN (default) [33.33% IMPUTED]](https://arxiv.org/abs/2502.02527),1225.0,+107/-93,14.429,0.081,32.56,25.25,66.06,0.652,➖,33.33,True,GPU -36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1213.0,+90/-83,13.175,0.042,33.14,26.81,66.0,0.024,✔️,0.0,False,CPU -37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1212.0,+101/-142,15.447,0.096,33.22,28.28,3.12,0.016,✔️,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1208.0,+70/-83,12.591,0.025,33.41,30.98,36.48,0.329,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1200.0,+110/-138,14.414,0.075,33.83,26.33,351.94,1.658,✔️,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1163.0,+61/-79,13.815,0.011,35.65,33.8,36.48,0.037,✔️,0.0,False,CPU -41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1118.0,+69/-72,15.577,0.001,37.78,36.08,2.18,0.039,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1093.0,+87/-116,16.687,0.022,38.9,34.23,351.94,0.114,✔️,0.0,False,CPU -43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1049.0,+139/-182,23.725,0.02,40.74,35.28,2.49,0.145,✔️,0.0,False,GPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1014.0,+135/-175,20.379,0.026,42.13,37.5,1.51,0.111,✔️,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+58/-88,18.741,0.0,42.67,41.79,0.2,0.021,✔️,0.0,False,CPU -46,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),989.0,+88/-119,22.293,0.006,43.08,36.37,150.3,10.071,✔️,0.0,False,CPU -47,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),948.0,+151/-218,31.147,0.005,44.5,40.65,43.64,0.081,✔️,0.0,False,CPU -48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),945.0,+111/-138,21.857,0.003,44.62,42.76,0.11,0.028,✔️,0.0,False,CPU -49,🧠⚡,Foundation Model,[TabFlex (default) [33.33% IMPUTED]](https://arxiv.org/abs/2506.05584),926.0,+72/-121,25.797,0.0,45.24,44.31,0.25,0.045,➖,33.33,True,GPU -50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),899.0,+155/-243,32.089,0.003,46.06,43.0,43.64,0.018,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),804.0,+62/-101,28.282,0.0,48.55,48.25,150.3,0.482,✔️,0.0,False,CPU -52,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),787.0,+131/-231,34.193,0.0,48.93,47.58,0.19,0.019,✔️,0.0,False,CPU -53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),560.0,+100/-154,42.236,0.0,52.4,52.31,0.1,0.072,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1806.0,+94/-52,0.822,6.1,2.88,2.62,289.53,3.187,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1781.0,+90/-54,0.796,6.75,3.14,2.723,432.35,4.085,✔️,0.0,False,CPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1659.0,+163/-118,0.651,10.78,3.35,5.438,735.58,11.736,✔️,0.0,False,GPU +3,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1631.0,+102/-90,0.58,11.9,7.37,5.74,1719.82,1.675,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1618.0,+119/-115,0.596,12.43,6.83,5.81,2526.28,6.013,✔️,0.0,False,GPU +5,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1606.0,+90/-62,0.538,12.97,6.91,5.642,777.59,0.25,✔️,0.0,False,CPU +6,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1604.0,+146/-110,0.585,13.03,5.31,6.152,735.58,1.386,✔️,0.0,False,GPU +7,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1596.0,+97/-66,0.539,13.4,5.74,5.838,777.59,0.052,✔️,0.0,False,CPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1588.0,+74/-60,0.489,13.74,9.2,7.405,131.56,2.639,✔️,0.0,False,CPU +9,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1555.0,+109/-95,0.522,15.3,7.45,6.028,3.24,0.031,✔️,0.0,False,CPU +10,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1548.0,+86/-57,0.443,15.61,9.33,7.531,282.13,0.563,✔️,0.0,False,CPU +11,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1535.0,+146/-96,0.476,16.24,7.91,6.964,1.88,0.645,✔️,0.0,False,GPU +12,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1531.0,+116/-88,0.411,16.43,9.2,7.07,1719.82,0.081,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1529.0,+150/-87,0.426,16.55,7.57,7.466,1993.14,0.619,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1517.0,+131/-122,0.486,17.14,4.24,7.078,4786.55,444.544,✔️,0.0,False,GPU +15,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1501.0,+69/-48,0.361,17.96,13.64,8.115,282.13,0.066,✔️,0.0,False,CPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1499.0,+78/-60,0.349,18.03,12.36,8.625,131.56,0.132,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1477.0,+144/-96,0.361,19.17,8.69,8.222,1993.14,0.061,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1471.0,+102/-88,0.334,19.5,13.93,8.204,2526.28,0.349,✔️,0.0,False,GPU +19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1424.0,+112/-94,0.284,21.97,13.17,9.726,566.74,0.951,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1394.0,+124/-121,0.299,23.63,8.91,9.704,4786.55,42.641,✔️,0.0,False,GPU +21,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1375.0,+69/-64,0.18,24.67,17.95,9.473,1770.56,1.069,✔️,0.0,False,CPU +22,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1374.0,+94/-92,0.209,24.76,19.41,11.424,566.74,0.06,✔️,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1365.0,+97/-111,0.227,25.23,16.81,9.978,0.49,0.053,✔️,0.0,False,CPU +24,🧠⚡,Foundation Model,[TabICL (default) [33.33% IMPUTED]](https://arxiv.org/abs/2502.05564),1339.0,+129/-109,0.278,26.66,12.33,10.315,3.97,1.596,✔️,33.33,True,GPU +25,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1324.0,+119/-136,0.231,27.52,10.28,12.499,46.62,43.739,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1321.0,+129/-105,0.181,27.7,18.03,11.062,5.16,0.074,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[LimiX (default) [40.00% IMPUTED]](https://arxiv.org/abs/2509.03505),1320.0,+135/-129,0.255,27.72,8.87,14.019,1.08,0.32,➖,40.0,True,GPU +28,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1318.0,+118/-125,0.198,27.84,18.53,12.974,1517.65,0.134,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1295.0,+76/-79,0.115,29.14,25.01,11.512,8.06,0.25,✔️,0.0,False,GPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1273.0,+86/-74,0.054,30.36,27.05,12.26,6.75,0.235,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1269.0,+79/-76,0.066,30.56,25.71,11.256,1770.56,0.051,✔️,0.0,False,CPU +32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1258.0,+111/-139,0.139,31.16,23.51,13.864,1517.65,0.015,✔️,0.0,False,CPU +33,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1255.0,+60/-68,0.046,31.33,28.89,11.913,0.29,0.045,✔️,0.0,False,CPU +34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1250.0,+82/-72,0.041,31.62,25.19,12.464,66.0,0.341,✔️,0.0,False,CPU +35,🧠⚡,Foundation Model,[BetaTabPFN (default) [33.33% IMPUTED]](https://arxiv.org/abs/2502.02527),1225.0,+103/-89,0.084,32.94,25.44,14.429,66.06,0.652,➖,33.33,True,GPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1214.0,+88/-83,0.042,33.5,26.99,13.175,66.0,0.024,✔️,0.0,False,CPU +37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1210.0,+102/-141,0.099,33.7,28.51,15.447,3.12,0.016,✔️,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1208.0,+70/-85,0.025,33.81,31.24,12.591,36.48,0.329,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1198.0,+110/-138,0.076,34.36,26.54,14.414,351.94,1.658,✔️,0.0,False,CPU +40,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1195.0,+85/-96,0.047,34.48,29.36,14.386,11.44,10.416,➖,0.0,False,GPU +41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1163.0,+63/-80,0.011,36.13,34.14,13.815,36.48,0.037,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1116.0,+69/-73,0.001,38.38,36.56,15.577,2.18,0.039,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1090.0,+88/-116,0.023,39.6,34.72,16.687,351.94,0.114,✔️,0.0,False,CPU +44,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1050.0,+138/-183,0.022,41.36,35.7,23.725,2.49,0.145,✔️,0.0,False,GPU +45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1012.0,+135/-177,0.026,42.91,38.05,20.379,1.51,0.111,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+59/-88,0.0,43.36,42.42,18.741,0.2,0.021,✔️,0.0,False,CPU +47,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),985.0,+87/-122,0.006,43.94,37.01,22.293,150.3,10.071,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),945.0,+151/-218,0.005,45.37,41.41,31.147,43.64,0.081,✔️,0.0,False,CPU +49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),943.0,+110/-142,0.003,45.41,43.41,21.857,0.11,0.028,✔️,0.0,False,CPU +50,🧠⚡,Foundation Model,[TabFlex (default) [33.33% IMPUTED]](https://arxiv.org/abs/2506.05584),925.0,+73/-125,0.0,46.03,45.04,25.797,0.25,0.045,➖,33.33,True,GPU +51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),895.0,+155/-241,0.003,46.96,43.82,32.089,43.64,0.018,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),798.0,+60/-101,0.0,49.55,49.26,28.282,150.3,0.482,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),782.0,+131/-234,0.0,49.91,48.59,34.193,0.19,0.019,✔️,0.0,False,CPU +54,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),554.0,+100/-153,0.0,53.4,53.32,42.236,0.1,0.072,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_medium/winrate_matrix.png.zip index 6a4fb7e0c7374ec76c77fad73ea32c688ff37f00..6b644390838c1920be8619f6beb8457530c4a529 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:48d4cf58c01eec79b7a840a77e8291c7df43374e8496dc8d674d46a2d966f2c7 -size 2309578 +oid sha256:0253b78f6a54f2f19f6669627b7368ab7fa47e914e5be802bd74db2438275c80 +size 2515087 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index d6078810a67604e37c20493bd883852e38e713f8..eae38e43fb01064a2d05370930d8a9ef698b7c5b 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a68bc3372dc9aa20198862597f333cf01a54677a31598ff040d2875c48a8e5a1 -size 464248 +oid sha256:d389fca70e959534d5e0cebf84f71efc52b3c53c63b8c81c5cf8e5140b3ce76c +size 477614 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_small/pareto_n_configs_imp.png.zip index 4448b456ceed313000383be8752cbca53972025c..28341be1fc48ecee13b5f829d09028d04025e90a 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b4e4c03de480d60e809f5e16dddc004ad6d4765bf8628c84fcb5fab0a4b18a7c -size 1048136 +oid sha256:3a5a080a26d3647446fb2de1e08f34f208aad0b22258deb39a62f93216f9573d +size 1037337 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_small/tuning-impact-elo.png.zip index c1c09209e32a7bb9c59e22ee32e6f172d2c49df8..198470a633d98e3820c9ee52c380cb181888cb89 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1013d6f95f51046efd9af64816ef80526d76a6afd0b3e0981194a9169102cf37 -size 193409 +oid sha256:c3b90de6ab1784ed03c6b0812cfa649c6da9ac62061f1bd6b3559902d29af524 +size 196612 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_all/datasets_small/website_leaderboard.csv index fc19183ddeb66d8f23a3f265ac756bbd4cf5fd31..71eb022599c0f1473746414392e2ea632685ac7f 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_small/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_all/datasets_small/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1612.0,+63/-54,7.181,0.715,9.5,4.46,659.63,7.444,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1610.0,+87/-62,4.943,0.738,9.55,3.38,2289.05,8.049,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1567.0,+71/-55,6.024,0.681,11.37,4.42,2289.05,1.143,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1551.0,+81/-54,6.684,0.676,12.09,5.68,6.76,0.636,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1482.0,+56/-48,10.436,0.487,15.58,8.52,3770.75,21.902,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1480.0,+89/-64,10.121,0.542,15.73,6.02,3527.3,48.6,✔️,8.33,True,GPU -6,🧠⚡,Foundation Model,[LimiX (default) [0.31% IMPUTED]](https://arxiv.org/abs/2509.03505),1459.0,+86/-69,12.533,0.544,16.89,5.69,5.08,0.645,➖,0.31,True,GPU -7,🧠⚡,Foundation Model,[Mitra (default) [8.33% IMPUTED]](https://arxiv.org/abs/2510.21204),1458.0,+68/-56,11.186,0.51,16.97,6.59,126.84,3.094,✔️,8.33,True,GPU -8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1452.0,+80/-59,9.997,0.486,17.3,5.65,5119.36,218.711,✔️,0.0,False,GPU -9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1442.0,+59/-46,11.622,0.391,17.84,9.65,3716.07,2.547,✔️,0.0,False,CPU -10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1426.0,+52/-46,11.814,0.373,18.86,10.78,3553.12,1.741,✔️,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1418.0,+73/-57,10.878,0.435,19.3,6.59,5119.36,28.351,✔️,0.0,False,GPU -12,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1399.0,+52/-41,11.979,0.331,20.5,10.75,3770.75,1.778,✔️,0.0,False,GPU -13,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1389.0,+77/-62,12.469,0.403,21.1,7.45,3527.3,0.817,✔️,8.33,True,GPU -14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1384.0,+33/-36,13.566,0.273,21.41,15.27,892.41,2.574,✔️,0.0,False,CPU -15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1364.0,+51/-49,12.792,0.282,22.68,12.45,3553.12,0.241,✔️,0.0,False,GPU -16,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1358.0,+40/-44,13.606,0.23,23.13,15.92,2476.51,0.814,✔️,0.0,False,CPU -17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1350.0,+34/-30,14.034,0.207,23.6,17.44,892.41,0.346,✔️,0.0,False,CPU -18,🧠⚡,Foundation Model,[TabPFNv2 (default) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1349.0,+76/-63,13.819,0.367,23.7,10.18,4.54,0.443,✔️,8.33,True,GPU -19,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1346.0,+45/-37,13.784,0.215,23.91,17.16,2476.51,0.101,✔️,0.0,False,CPU -20,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1337.0,+72/-68,13.538,0.35,24.45,7.71,50.32,43.709,✔️,0.0,False,GPU -21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1332.0,+44/-41,14.202,0.228,24.79,14.42,1126.71,2.941,✔️,0.0,False,GPU -22,🧠⚡,Foundation Model,[TabICL (default) [27.78% IMPUTED]](https://arxiv.org/abs/2502.05564),1328.0,+79/-106,14.023,0.327,25.08,8.57,7.15,1.47,✔️,27.78,True,GPU -23,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1326.0,+62/-62,14.22,0.266,25.22,9.46,6521.96,8.396,✔️,0.0,False,GPU -24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1324.0,+41/-41,13.989,0.193,25.36,11.38,6521.96,0.506,✔️,0.0,False,GPU -25,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1321.0,+38/-39,14.596,0.19,25.51,18.55,884.18,2.368,✔️,0.0,False,CPU -26,🧠⚡,Foundation Model,[BetaTabPFN (default) [22.22% IMPUTED]](https://arxiv.org/abs/2502.02527),1308.0,+58/-61,15.69,0.286,26.4,13.07,296.89,1.835,➖,22.22,True,GPU -27,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1305.0,+37/-38,14.946,0.175,26.6,18.42,9.64,0.128,✔️,0.0,False,CPU -28,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1305.0,+36/-40,14.805,0.151,26.61,18.85,884.18,0.393,✔️,0.0,False,CPU -29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1290.0,+54/-48,15.156,0.219,27.59,15.87,13.18,0.17,✔️,0.0,False,GPU -30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1284.0,+41/-41,15.323,0.164,27.99,16.12,1126.71,0.163,✔️,0.0,False,GPU -31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1260.0,+42/-48,15.511,0.13,29.6,19.86,3865.95,2.158,✔️,0.0,False,CPU -32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1255.0,+43/-43,16.532,0.117,29.96,19.99,3351.28,0.47,✔️,0.0,False,CPU -33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1238.0,+44/-50,17.662,0.109,31.08,17.44,16.27,0.321,✔️,0.0,False,GPU -34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1226.0,+44/-44,17.118,0.086,31.89,19.9,3351.28,0.056,✔️,0.0,False,CPU -35,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1221.0,+39/-42,16.489,0.082,32.22,21.35,15.69,4.691,✔️,0.0,False,GPU -36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1216.0,+55/-59,17.561,0.134,32.51,18.12,1678.92,8.069,✔️,0.0,False,CPU -37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1210.0,+65/-64,19.071,0.119,32.92,19.81,499.14,1.466,✔️,0.0,False,CPU -38,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1197.0,+48/-53,18.1,0.087,33.75,20.7,9.08,0.057,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1192.0,+43/-48,17.401,0.079,34.11,22.31,3865.95,0.148,✔️,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1174.0,+61/-64,19.929,0.091,35.22,19.78,499.14,0.207,✔️,0.0,False,CPU -41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1167.0,+50/-51,18.728,0.06,35.7,21.68,1678.92,0.826,✔️,0.0,False,CPU -42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1162.0,+40/-40,18.171,0.042,35.97,30.21,3.61,0.17,✔️,0.0,False,CPU -43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1162.0,+30/-36,18.096,0.019,35.98,30.28,3.29,0.253,✔️,0.0,False,CPU -44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1156.0,+57/-68,20.347,0.086,36.41,21.67,590.04,1.461,✔️,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1114.0,+55/-70,21.141,0.049,38.94,20.59,590.04,0.15,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1057.0,+44/-51,21.794,0.01,42.17,32.23,13.69,0.155,✔️,0.0,False,CPU -47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1041.0,+70/-88,23.446,0.032,43.0,33.58,5.44,1.098,✔️,0.0,False,GPU -48,🧠⚡,Foundation Model,[TabFlex (default) [22.22% IMPUTED]](https://arxiv.org/abs/2506.05584),1035.0,+80/-81,24.85,0.064,43.32,28.07,1.58,0.15,➖,22.22,True,GPU -49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+58/-77,25.82,0.006,45.05,38.62,0.51,0.089,✔️,0.0,False,CPU -50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1000.0,+51/-63,24.072,0.0,45.05,31.06,5.48,0.685,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),995.0,+71/-103,26.981,0.023,45.29,30.0,111.84,1.215,✔️,0.0,False,CPU -52,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),995.0,+71/-73,26.662,0.013,45.29,38.52,0.47,0.092,✔️,0.0,False,CPU -53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),967.0,+84/-94,31.665,0.039,46.55,32.92,520.95,0.647,✔️,0.0,False,CPU -54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),944.0,+82/-100,32.134,0.026,47.53,34.6,520.95,0.113,✔️,0.0,False,CPU -55,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),907.0,+74/-97,31.894,0.009,49.0,34.54,111.84,0.111,✔️,0.0,False,CPU -56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),866.0,+98/-119,35.122,0.017,50.45,37.69,2.59,0.138,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),654.0,+112/-150,45.374,0.0,55.42,48.52,0.37,0.033,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1608.0,+64/-52,0.71,9.72,4.52,7.399,659.63,7.444,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1608.0,+84/-62,0.735,9.76,3.46,5.22,2289.05,8.049,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1564.0,+69/-54,0.677,11.61,4.48,6.3,2289.05,1.143,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1549.0,+80/-54,0.668,12.32,5.96,6.982,6.76,0.636,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1480.0,+55/-47,0.478,15.91,8.8,10.677,3770.75,21.902,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1477.0,+88/-62,0.535,16.07,6.21,10.319,3527.3,48.6,✔️,8.33,True,GPU +6,🧠🔁,Neural Network,[LimiX (default) [0.31% IMPUTED]](https://arxiv.org/abs/2509.03505),1458.0,+84/-68,0.533,17.16,5.83,12.792,5.08,0.645,➖,0.31,True,GPU +7,🧠🔁,Neural Network,[Mitra (default) [8.33% IMPUTED]](https://arxiv.org/abs/2510.21204),1456.0,+66/-56,0.503,17.31,6.73,11.33,126.84,3.094,✔️,8.33,True,GPU +8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1450.0,+80/-58,0.478,17.66,5.71,10.163,5119.36,218.711,✔️,0.0,False,GPU +9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1440.0,+59/-46,0.389,18.24,9.93,11.759,3716.07,2.547,✔️,0.0,False,CPU +10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1423.0,+50/-46,0.366,19.25,10.94,11.978,3553.12,1.741,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1416.0,+73/-56,0.434,19.69,6.68,11.021,5119.36,28.351,✔️,0.0,False,GPU +12,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1397.0,+52/-40,0.328,20.91,11.05,12.196,3770.75,1.778,✔️,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1388.0,+76/-62,0.395,21.51,7.62,12.653,3527.3,0.817,✔️,8.33,True,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1382.0,+33/-36,0.27,21.87,15.55,13.695,892.41,2.574,✔️,0.0,False,CPU +15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1362.0,+51/-50,0.279,23.16,12.65,12.944,3553.12,0.241,✔️,0.0,False,GPU +16,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1357.0,+40/-43,0.226,23.55,16.24,13.813,2476.51,0.814,✔️,0.0,False,CPU +17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1348.0,+33/-31,0.205,24.09,17.74,14.165,892.41,0.346,✔️,0.0,False,CPU +18,🧠⚡,Foundation Model,[TabPFNv2 (default) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1347.0,+75/-62,0.355,24.19,10.5,14.075,4.54,0.443,✔️,8.33,True,GPU +19,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1344.0,+45/-36,0.211,24.37,17.49,13.99,2476.51,0.101,✔️,0.0,False,CPU +20,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1336.0,+72/-68,0.349,24.92,7.81,13.628,50.32,43.709,✔️,0.0,False,GPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1330.0,+44/-41,0.223,25.3,14.85,14.393,1126.71,2.941,✔️,0.0,False,GPU +22,🧠⚡,Foundation Model,[TabICL (default) [27.78% IMPUTED]](https://arxiv.org/abs/2502.05564),1326.0,+79/-107,0.322,25.58,8.67,14.163,7.15,1.47,✔️,27.78,True,GPU +23,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1324.0,+62/-61,0.264,25.72,9.59,14.378,6521.96,8.396,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1322.0,+41/-42,0.194,25.87,11.58,14.134,6521.96,0.506,✔️,0.0,False,GPU +25,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1322.0,+67/-56,0.317,25.89,8.54,14.806,16.22,1.278,➖,0.0,False,GPU +26,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1320.0,+38/-39,0.185,26.03,18.95,14.746,884.18,2.368,✔️,0.0,False,CPU +27,🧠⚡,Foundation Model,[BetaTabPFN (default) [22.22% IMPUTED]](https://arxiv.org/abs/2502.02527),1307.0,+58/-62,0.283,26.92,13.29,15.841,296.89,1.835,➖,22.22,True,GPU +28,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1304.0,+37/-40,0.171,27.11,18.76,15.063,9.64,0.128,✔️,0.0,False,CPU +29,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1303.0,+36/-40,0.145,27.14,19.19,14.96,884.18,0.393,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1289.0,+53/-48,0.215,28.13,16.66,15.279,13.18,0.17,✔️,0.0,False,GPU +31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1283.0,+40/-42,0.16,28.53,16.4,15.503,1126.71,0.163,✔️,0.0,False,GPU +32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1258.0,+42/-48,0.131,30.24,20.17,15.622,3865.95,2.158,✔️,0.0,False,CPU +33,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1254.0,+42/-42,0.114,30.54,20.51,16.662,3351.28,0.47,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1237.0,+44/-50,0.107,31.7,17.83,17.774,16.27,0.321,✔️,0.0,False,GPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1225.0,+44/-43,0.085,32.5,20.25,17.251,3351.28,0.056,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1219.0,+39/-42,0.081,32.91,22.02,16.595,15.69,4.691,✔️,0.0,False,GPU +37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1214.0,+54/-59,0.134,33.19,18.51,17.653,1678.92,8.069,✔️,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1209.0,+65/-62,0.116,33.58,20.11,19.235,499.14,1.466,✔️,0.0,False,CPU +39,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1196.0,+47/-53,0.082,34.43,21.23,18.212,9.08,0.057,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1190.0,+43/-48,0.082,34.79,22.65,17.492,3865.95,0.148,✔️,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1174.0,+61/-64,0.089,35.9,20.17,20.082,499.14,0.207,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1165.0,+50/-51,0.06,36.43,22.27,18.816,1678.92,0.826,✔️,0.0,False,CPU +43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1162.0,+30/-38,0.019,36.64,30.76,18.212,3.29,0.253,✔️,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1162.0,+39/-41,0.042,36.66,30.7,18.266,3.61,0.17,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1154.0,+57/-68,0.083,37.12,22.17,20.541,590.04,1.461,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1113.0,+53/-70,0.046,39.68,22.0,21.335,590.04,0.15,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1056.0,+44/-50,0.011,42.97,32.81,21.902,13.69,0.155,✔️,0.0,False,CPU +48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1041.0,+67/-87,0.029,43.78,34.58,23.605,5.44,1.098,✔️,0.0,False,GPU +49,🧠⚡,Foundation Model,[TabFlex (default) [22.22% IMPUTED]](https://arxiv.org/abs/2506.05584),1034.0,+79/-81,0.064,44.13,28.68,24.92,1.58,0.15,➖,22.22,True,GPU +50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+57/-78,0.005,45.87,39.78,25.923,0.51,0.089,✔️,0.0,False,CPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1000.0,+50/-64,0.0,45.88,31.99,24.141,5.48,0.685,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),996.0,+69/-101,0.024,46.07,30.38,27.061,111.84,1.215,✔️,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),995.0,+70/-76,0.012,46.09,39.34,26.744,0.47,0.092,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),966.0,+83/-95,0.039,47.43,33.46,31.777,520.95,0.647,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),944.0,+80/-100,0.027,48.4,35.22,32.224,520.95,0.113,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),908.0,+72/-97,0.011,49.86,35.05,31.961,111.84,0.111,✔️,0.0,False,CPU +57,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),866.0,+97/-120,0.017,51.34,38.24,35.188,2.59,0.138,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),653.0,+114/-151,0.0,56.4,49.34,45.421,0.37,0.033,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_small/winrate_matrix.png.zip index c440b334f001b575d568cd42839ebc6479e0da53..418bb58c5053583b91852d2db32d22ce8625e514 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:85e0b9600387668d2e326b6a29b3bfb48e9abd2ee98864500a52cb39ccdcb66f -size 2746120 +oid sha256:39176e18b45f52907330fd355c464bef4589288a59c82b8b41933f5f80124ee1 +size 2751715 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 4914366b2a28d3777590e98b2bc5232ccfd66701..97b35720b622927e98df5cc6f2a17aa31bb019c3 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c0e71ebe67203c7780a5e6b9ad321d726ac480167864a139ea182620c10d90a2 -size 464205 +oid sha256:445e31d24e50ef6fe649ea0f1fbaeb3f15fad1aa0c897e305a780efc36ffc45c +size 478845 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip index 0eb344392fddacc64325ef2c9759e552201eb4df..4f75c580ed8e23e7e94bfb641e432abdb80fa482 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f98ba33aed96dbeb4c223c1e82609a4ee363341edd42c5c4620fe0ff8c95bcfc -size 1085967 +oid sha256:964c5ad1dc3e820ca2fe05fec9b6b48dd5d5cfd341155a83e2075d6f96fee7a0 +size 1054996 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip index b38ea9763f0f7afd08e71288b381c8eb1d865331..d4a8d01e28091157e1a2a3b60deeb50bcda56dae 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4b128b80a98f144a4b9c30f631608ad8d94bdfd251dcdba4e53481d2562472c0 -size 145426 +oid sha256:c9dae99846c7cf99efeeb412a31b36e32534ed5ed88a94d4857d974681f8452e +size 150507 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/website_leaderboard.csv index 2282a6b130e7072a65a5583bec2a5d6423438f49..9406ad3cae50761e079b6e1ff5b9bc0ff24aa0c4 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1632.0,+93/-70,5.292,0.74,9.53,3.46,2166.18,7.945,✔️,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1628.0,+76/-55,7.717,0.711,9.71,4.55,611.55,7.01,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1594.0,+66/-62,6.928,0.714,11.11,5.37,6.14,0.586,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1589.0,+71/-61,6.39,0.686,11.29,4.35,2166.18,0.993,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1540.0,+74/-48,9.556,0.586,13.62,5.6,3899.42,55.833,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1511.0,+72/-47,10.725,0.551,15.1,6.15,136.59,3.391,✔️,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1505.0,+56/-46,11.115,0.499,15.41,8.34,3547.04,20.963,✔️,0.0,False,GPU -7,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1498.0,+74/-70,13.136,0.576,15.78,5.57,4.85,0.629,➖,0.0,False,GPU -8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1461.0,+74/-61,10.724,0.47,17.9,5.77,4980.83,207.372,✔️,0.0,False,GPU -9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1453.0,+48/-47,12.387,0.383,18.36,10.23,3376.59,2.54,✔️,0.0,False,CPU -10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1441.0,+54/-50,12.586,0.369,19.08,10.6,3372.56,1.658,✔️,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1436.0,+78/-68,11.521,0.435,19.38,6.39,4980.83,23.583,✔️,0.0,False,GPU -12,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1434.0,+82/-68,12.117,0.435,19.47,6.97,3899.42,0.983,✔️,0.0,False,GPU -13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1417.0,+57/-46,12.689,0.339,20.53,10.64,3547.04,1.57,✔️,0.0,False,GPU -14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1392.0,+38/-36,14.551,0.252,22.1,15.71,771.57,2.49,✔️,0.0,False,CPU -15,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1386.0,+68/-69,13.598,0.396,22.44,9.6,4.22,0.554,✔️,0.0,False,GPU -16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1379.0,+55/-51,13.598,0.278,22.92,12.22,3372.56,0.208,✔️,0.0,False,GPU -17,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1377.0,+43/-38,14.476,0.237,23.04,15.6,2034.85,0.805,✔️,0.0,False,CPU -18,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1363.0,+38/-34,14.675,0.219,23.94,17.08,2034.85,0.097,✔️,0.0,False,CPU -19,🧠⚡,Foundation Model,[TabICL (default) [21.21% IMPUTED]](https://arxiv.org/abs/2502.05564),1363.0,+77/-84,13.82,0.352,23.95,8.04,7.27,1.64,✔️,21.21,True,GPU -20,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1361.0,+91/-63,14.241,0.373,24.06,7.26,48.9,43.804,✔️,0.0,False,GPU -21,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1358.0,+34/-30,15.026,0.186,24.26,17.76,771.57,0.322,✔️,0.0,False,CPU -22,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1338.0,+52/-46,15.196,0.208,25.57,15.07,1105.59,2.552,✔️,0.0,False,GPU -23,🧠⚡,Foundation Model,[BetaTabPFN (default) [21.21% IMPUTED]](https://arxiv.org/abs/2502.02527),1338.0,+64/-77,16.243,0.291,25.61,12.57,303.08,1.792,➖,21.21,True,GPU -24,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1334.0,+37/-36,15.598,0.182,25.82,19.14,828.74,2.312,✔️,0.0,False,CPU -25,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1334.0,+45/-48,14.832,0.172,25.85,12.27,6147.69,0.485,✔️,0.0,False,GPU -26,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1334.0,+73/-66,15.247,0.252,25.86,11.65,6147.69,8.149,✔️,0.0,False,GPU -27,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1322.0,+48/-44,15.903,0.177,26.64,18.03,8.51,0.123,✔️,0.0,False,CPU -28,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1316.0,+34/-34,15.81,0.142,26.99,19.24,828.74,0.336,✔️,0.0,False,CPU -29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1309.0,+55/-53,15.975,0.22,27.51,15.35,12.24,0.154,✔️,0.0,False,GPU -30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1293.0,+45/-41,16.334,0.154,28.52,16.9,1105.59,0.13,✔️,0.0,False,GPU -31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1278.0,+47/-44,17.376,0.114,29.53,19.36,2929.85,0.433,✔️,0.0,False,CPU -32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1271.0,+45/-54,16.4,0.122,30.01,21.32,3704.3,2.074,✔️,0.0,False,CPU -33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1266.0,+46/-41,17.996,0.117,30.36,16.68,16.16,0.307,✔️,0.0,False,GPU -34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1248.0,+51/-48,17.977,0.081,31.51,19.23,2929.85,0.054,✔️,0.0,False,CPU -35,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1243.0,+40/-38,17.096,0.081,31.85,20.77,15.48,4.535,✔️,0.0,False,GPU -36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1234.0,+51/-60,18.45,0.137,32.42,17.55,1459.62,8.056,✔️,0.0,False,CPU -37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1216.0,+49/-51,18.863,0.072,33.57,20.75,8.47,0.054,✔️,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1211.0,+58/-55,20.343,0.1,33.91,22.7,416.39,1.393,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1201.0,+42/-46,18.313,0.065,34.56,22.3,3704.3,0.143,✔️,0.0,False,CPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1180.0,+54/-59,19.638,0.06,35.83,21.57,1459.62,0.89,✔️,0.0,False,CPU -41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1177.0,+57/-56,21.176,0.069,36.06,23.58,416.39,0.179,✔️,0.0,False,CPU -42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1172.0,+34/-37,19.218,0.015,36.37,31.15,3.05,0.241,✔️,0.0,False,CPU -43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1158.0,+35/-41,19.446,0.021,37.24,32.62,3.39,0.158,✔️,0.0,False,CPU -44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1143.0,+54/-53,21.823,0.051,38.11,26.04,572.67,1.421,✔️,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1103.0,+45/-45,22.6,0.018,40.42,26.25,572.67,0.143,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1067.0,+50/-59,22.778,0.011,42.4,31.83,11.82,0.147,✔️,0.0,False,CPU -47,🧠⚡,Foundation Model,[TabFlex (default) [21.21% IMPUTED]](https://arxiv.org/abs/2506.05584),1056.0,+82/-88,25.63,0.069,43.0,27.1,1.19,0.15,➖,21.21,True,GPU -48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1054.0,+70/-86,24.777,0.035,43.09,33.03,5.39,1.034,✔️,0.0,False,GPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1009.0,+51/-66,24.915,0.0,45.3,30.44,5.18,0.652,✔️,0.0,False,CPU -50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1006.0,+82/-85,27.446,0.014,45.46,38.22,0.42,0.078,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1000.0,+72/-88,28.256,0.014,45.71,33.26,100.58,1.106,✔️,0.0,False,CPU -52,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+53/-69,26.69,0.002,45.73,39.57,0.47,0.069,✔️,0.0,False,CPU -53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1000.0,+71/-104,31.374,0.043,45.75,31.75,504.17,0.629,✔️,0.0,False,CPU -54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),977.0,+72/-104,31.806,0.028,46.76,33.44,504.17,0.103,✔️,0.0,False,CPU -55,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),904.0,+78/-101,33.41,0.0,49.65,35.57,100.58,0.101,✔️,0.0,False,CPU -56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),899.0,+80/-125,34.29,0.018,49.8,36.54,2.51,0.137,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),677.0,+96/-161,45.738,0.0,55.29,47.88,0.29,0.028,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1629.0,+92/-69,0.735,9.75,3.53,5.593,2166.18,7.945,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1624.0,+76/-54,0.705,9.94,4.61,7.954,611.55,7.01,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1591.0,+65/-62,0.706,11.36,5.64,7.252,6.14,0.586,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1586.0,+72/-61,0.682,11.55,4.42,6.69,2166.18,0.993,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1537.0,+74/-48,0.58,13.94,5.78,9.769,3899.42,55.833,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),1509.0,+72/-47,0.545,15.42,6.28,10.881,136.59,3.391,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1502.0,+55/-44,0.49,15.77,8.62,11.376,3547.04,20.963,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1497.0,+73/-69,0.565,16.06,5.7,13.417,4.85,0.629,➖,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1459.0,+74/-61,0.463,18.28,5.84,10.903,4980.83,207.372,✔️,0.0,False,GPU +9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1450.0,+47/-47,0.381,18.78,10.5,12.535,3376.59,2.54,✔️,0.0,False,CPU +10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1439.0,+53/-51,0.362,19.5,10.76,12.763,3372.56,1.658,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1434.0,+78/-68,0.433,19.79,6.48,11.675,4980.83,23.583,✔️,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1432.0,+80/-68,0.428,19.88,7.12,12.316,3899.42,0.983,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1415.0,+57/-45,0.334,20.95,10.96,12.924,3547.04,1.57,✔️,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1390.0,+38/-35,0.249,22.59,16.02,14.69,771.57,2.49,✔️,0.0,False,CPU +15,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1385.0,+68/-69,0.384,22.93,9.9,13.876,4.22,0.554,✔️,0.0,False,GPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1377.0,+55/-51,0.273,23.43,12.42,13.762,3372.56,0.208,✔️,0.0,False,GPU +17,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1376.0,+43/-38,0.232,23.49,15.93,14.7,2034.85,0.805,✔️,0.0,False,CPU +18,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1362.0,+38/-33,0.214,24.42,17.43,14.898,2034.85,0.097,✔️,0.0,False,CPU +19,🧠⚡,Foundation Model,[TabICL (default) [21.21% IMPUTED]](https://arxiv.org/abs/2502.05564),1361.0,+76/-84,0.348,24.45,8.14,13.972,7.27,1.64,✔️,21.21,True,GPU +20,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1360.0,+90/-63,0.372,24.54,7.36,14.337,48.9,43.804,✔️,0.0,False,GPU +21,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1358.0,+65/-67,0.342,24.68,8.22,15.317,14.11,1.17,➖,0.0,False,GPU +22,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1356.0,+35/-29,0.184,24.77,18.1,15.167,771.57,0.322,✔️,0.0,False,CPU +23,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1336.0,+50/-45,0.203,26.12,15.58,15.403,1105.59,2.552,✔️,0.0,False,GPU +24,🧠⚡,Foundation Model,[BetaTabPFN (default) [21.21% IMPUTED]](https://arxiv.org/abs/2502.02527),1336.0,+64/-78,0.287,26.15,12.79,16.406,303.08,1.792,➖,21.21,True,GPU +25,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1332.0,+45/-48,0.172,26.38,12.53,14.988,6147.69,0.485,✔️,0.0,False,GPU +26,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1332.0,+38/-36,0.178,26.38,19.61,15.76,828.74,2.312,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1332.0,+72/-65,0.251,26.38,11.85,15.418,6147.69,8.149,✔️,0.0,False,GPU +28,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1320.0,+47/-44,0.173,27.19,18.39,16.029,8.51,0.123,✔️,0.0,False,CPU +29,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1315.0,+35/-33,0.137,27.56,19.62,15.977,828.74,0.336,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1308.0,+54/-52,0.214,28.07,16.14,16.108,12.24,0.154,✔️,0.0,False,GPU +31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1292.0,+45/-40,0.15,29.09,17.22,16.529,1105.59,0.13,✔️,0.0,False,GPU +32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1277.0,+47/-43,0.11,30.13,19.88,17.517,2929.85,0.433,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1269.0,+45/-54,0.123,30.67,21.7,16.52,3704.3,2.074,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1264.0,+45/-40,0.115,30.97,17.06,18.117,16.16,0.307,✔️,0.0,False,GPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1247.0,+50/-47,0.079,32.14,19.58,18.12,2929.85,0.054,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1241.0,+40/-38,0.08,32.55,21.45,17.211,15.48,4.535,✔️,0.0,False,GPU +37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1232.0,+50/-60,0.136,33.13,17.94,18.549,1459.62,8.056,✔️,0.0,False,CPU +38,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1215.0,+48/-51,0.066,34.26,21.33,18.983,8.47,0.054,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1210.0,+58/-56,0.097,34.61,23.12,20.521,416.39,1.393,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1200.0,+42/-46,0.066,35.27,22.66,18.411,3704.3,0.143,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1179.0,+54/-60,0.059,36.58,22.2,19.732,1459.62,0.89,✔️,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1176.0,+57/-56,0.066,36.77,24.17,21.341,416.39,0.179,✔️,0.0,False,CPU +43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1171.0,+34/-38,0.017,37.08,31.68,19.343,3.05,0.241,✔️,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1157.0,+35/-42,0.021,37.97,33.22,19.548,3.39,0.158,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1142.0,+54/-55,0.049,38.88,26.8,22.034,572.67,1.421,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1103.0,+46/-46,0.017,41.21,28.79,22.81,572.67,0.143,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1066.0,+50/-58,0.012,43.23,32.44,22.893,11.82,0.147,✔️,0.0,False,CPU +48,🧠⚡,Foundation Model,[TabFlex (default) [21.21% IMPUTED]](https://arxiv.org/abs/2506.05584),1055.0,+80/-87,0.07,43.85,27.72,25.704,1.19,0.15,➖,21.21,True,GPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1054.0,+70/-84,0.032,43.91,34.06,24.949,5.39,1.034,✔️,0.0,False,GPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1009.0,+51/-66,0.0,46.15,31.38,24.988,5.18,0.652,✔️,0.0,False,CPU +51,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1006.0,+80/-85,0.013,46.27,39.04,27.535,0.42,0.078,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1001.0,+72/-85,0.014,46.51,33.74,28.342,100.58,1.106,✔️,0.0,False,CPU +53,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+53/-70,0.002,46.57,40.81,26.801,0.47,0.069,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),999.0,+71/-105,0.043,46.61,32.26,31.494,504.17,0.629,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),977.0,+72/-105,0.029,47.62,34.03,31.903,504.17,0.103,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),904.0,+77/-102,0.001,50.52,36.14,33.482,100.58,0.101,✔️,0.0,False,CPU +57,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),900.0,+78/-122,0.018,50.67,37.06,34.361,2.51,0.137,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),677.0,+96/-160,0.0,56.27,48.7,45.788,0.29,0.028,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/winrate_matrix.png.zip index f48a2908ffceb98098d756a8723595572c59c836..df578b94c15f9ccb88a7dcb6873d6ff419c08db1 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9d3b10a349a171a8cc51b9e1fbbf6310568b5b656b4f33c6dc82f0ca7cb5a6dc -size 2779156 +oid sha256:bf5779cc35aa0991976ec4d82b93cdc57199c888b80b52f6e68aaa30ca7240bd +size 2766042 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index f7b50c516c1e2a80e80eb514d62623dd6d72d60e..9b259c7bd309cc7a3c3fb50c073cfeb2f7ffd45d 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ca1a4a5000190b6609c6289e0d55e78e4e8a9e616eaf69dac0a38c6f912c4ee5 -size 475350 +oid sha256:22ed382fe444d21ecdc6b10c2e36067e98bc775a2c38fb8e98ef9d3a33f6bec5 +size 490641 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip index f4a28cc5820061ead99710faeacdcb3e8e2cffcb..9f4e590cbc46266540705458105105c42f224701 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:72b0f5f36dc4f78e8d6104ee66a3adeb5372df60a2424c994ac53e98125cd778 -size 1054118 +oid sha256:407d252eaaa739685e078afff6eda3d70707791f4b8db78dbe8d5f529718a07e +size 1016467 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_all/tuning-impact-elo.png.zip index 72c3816066eb22d8a5c2a167d8faefeec4953e7a..ad659b7b7f828ed2c3b6aa99d16146513b4a1788 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:56559afa1e3f99d8c9d1778046fb174af7c451ba01a34032ba8c801fcb37b099 -size 176724 +oid sha256:15c4d0e8c1ee92652bb4bc6632235312d84e05e2de03bdb7c5d9fad9774c2ee8 +size 187424 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_binary/datasets_all/website_leaderboard.csv index 5461becbda8817cebaf2b4803e70c2d0cce327cf..52253f2a1bcb6e5a56ef68e0bbedb60ceadc4d4b 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_all/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_all/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1623.0,+89/-65,6.145,0.747,8.95,3.5,442.09,6.381,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1554.0,+100/-58,6.413,0.642,11.87,4.35,1923.88,9.103,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1529.0,+90/-51,7.496,0.639,13.04,6.02,4.91,0.694,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1521.0,+94/-54,7.523,0.594,13.46,5.24,1923.88,1.329,✔️,0.0,False,GPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1484.0,+71/-60,9.411,0.511,15.42,5.71,1152.95,1.784,✔️,0.0,False,CPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1472.0,+48/-54,10.519,0.455,16.12,10.09,2775.44,8.849,✔️,0.0,False,GPU -6,🧠⚡,Foundation Model,[TabICL (default) [3.33% IMPUTED]](https://arxiv.org/abs/2502.05564),1448.0,+65/-54,9.774,0.468,17.52,6.71,8.06,1.743,✔️,3.33,True,GPU -7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1448.0,+66/-50,11.17,0.407,17.54,8.17,2229.7,1.387,✔️,0.0,False,GPU -8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1414.0,+51/-37,12.663,0.337,19.61,12.21,328.96,1.102,✔️,0.0,False,CPU -9,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1405.0,+66/-55,9.517,0.391,20.18,6.96,4982.66,358.753,✔️,0.0,False,GPU -10,🧠⚡,Foundation Model,[LimiX (default) [17.04% IMPUTED]](https://arxiv.org/abs/2509.03505),1398.0,+82/-76,13.11,0.469,20.62,6.79,1.92,0.489,➖,17.04,True,GPU -11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1393.0,+72/-56,12.081,0.326,20.99,9.69,2229.7,0.133,✔️,0.0,False,GPU -12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1386.0,+60/-51,12.442,0.309,21.41,11.06,1055.6,0.482,✔️,0.0,False,CPU -13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1382.0,+61/-52,12.536,0.317,21.68,10.08,1055.6,0.063,✔️,0.0,False,CPU -14,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1380.0,+54/-51,12.413,0.327,21.79,10.5,4.27,0.064,✔️,0.0,False,CPU -15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1379.0,+49/-54,12.311,0.271,21.9,11.37,2775.44,0.415,✔️,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1365.0,+38/-30,13.52,0.234,22.8,15.34,328.96,0.126,✔️,0.0,False,CPU -17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1364.0,+62/-51,13.566,0.28,22.87,12.42,472.99,0.621,✔️,0.0,False,CPU -18,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1362.0,+62/-55,14.431,0.323,22.98,11.51,250.36,1.708,➖,0.0,False,GPU -19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1349.0,+46/-56,13.197,0.236,23.86,11.46,3383.91,0.51,✔️,0.0,False,GPU -20,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1349.0,+90/-92,13.238,0.348,23.88,9.54,3383.91,8.396,✔️,0.0,False,GPU -21,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1343.0,+59/-52,11.206,0.272,24.26,8.57,4982.66,42.199,✔️,0.0,False,GPU -22,🧠⚡,Foundation Model,[Mitra (default) [33.33% IMPUTED]](https://arxiv.org/abs/2510.21204),1336.0,+87/-81,14.039,0.395,24.76,8.22,103.86,2.616,✔️,33.33,True,GPU -23,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1332.0,+47/-40,14.081,0.206,24.99,15.8,472.99,0.13,✔️,0.0,False,CPU -24,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1330.0,+86/-76,14.004,0.346,25.18,8.69,3031.01,27.044,✔️,33.33,True,GPU -25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1315.0,+56/-53,14.2,0.181,26.15,15.32,642.33,2.767,✔️,0.0,False,GPU -26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1302.0,+49/-40,13.82,0.175,27.06,18.1,2241.65,2.415,✔️,0.0,False,CPU -27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1298.0,+64/-63,14.773,0.226,27.36,14.59,8.72,0.126,✔️,0.0,False,GPU -28,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1293.0,+48/-43,15.686,0.152,27.68,18.0,1560.02,0.404,✔️,0.0,False,CPU -29,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1288.0,+46/-56,15.473,0.142,28.06,16.42,642.33,0.146,✔️,0.0,False,GPU -30,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1282.0,+80/-75,15.828,0.276,28.42,8.94,3031.01,0.514,✔️,33.33,True,GPU -31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1269.0,+62/-70,14.57,0.222,29.33,11.21,49.67,43.824,✔️,0.0,False,GPU -32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1261.0,+48/-47,16.397,0.101,29.92,18.94,1560.02,0.052,✔️,0.0,False,CPU -33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1257.0,+46/-40,16.239,0.124,30.14,16.66,13.5,0.311,✔️,0.0,False,GPU -34,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1256.0,+58/-56,15.841,0.138,30.25,18.4,559.38,4.08,✔️,0.0,False,CPU -35,🧠⚡,Foundation Model,[TabPFNv2 (default) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1256.0,+80/-80,17.22,0.275,30.26,11.7,2.71,0.276,✔️,33.33,True,GPU -36,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1253.0,+43/-43,15.156,0.094,30.44,20.02,9.33,1.116,✔️,0.0,False,GPU -37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1236.0,+48/-49,17.45,0.102,31.6,19.41,4.98,0.049,✔️,0.0,False,CPU -38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1221.0,+66/-56,16.712,0.096,32.61,22.15,1.24,0.11,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1215.0,+43/-37,16.017,0.066,32.99,22.6,2241.65,0.136,✔️,0.0,False,CPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1186.0,+57/-60,17.87,0.052,34.94,22.15,559.38,0.276,✔️,0.0,False,CPU -41,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1183.0,+48/-45,18.346,0.045,35.09,25.49,125.38,0.674,✔️,0.0,False,CPU -42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1173.0,+46/-35,17.489,0.045,35.75,30.57,0.81,0.093,✔️,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1153.0,+50/-58,19.69,0.032,37.05,25.7,169.75,0.603,✔️,0.0,False,CPU -44,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1144.0,+50/-49,19.5,0.027,37.62,27.86,125.38,0.069,✔️,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1119.0,+42/-57,20.419,0.011,39.16,26.32,169.75,0.059,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1093.0,+53/-47,20.405,0.012,40.68,31.46,4.95,0.091,✔️,0.0,False,CPU -47,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1056.0,+61/-77,21.791,0.015,42.74,27.51,2.56,0.267,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1050.0,+83/-99,27.109,0.053,43.03,29.05,162.68,0.405,✔️,0.0,False,CPU -49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1041.0,+102/-102,25.867,0.076,43.53,26.39,0.85,0.129,➖,0.0,False,GPU -50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1028.0,+61/-93,24.522,0.017,44.19,31.56,139.84,3.917,✔️,0.0,False,CPU -51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1022.0,+87/-118,27.802,0.034,44.49,31.08,162.68,0.054,✔️,0.0,False,CPU -52,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+62/-69,24.603,0.007,45.57,38.74,0.25,0.029,✔️,0.0,False,CPU -53,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),975.0,+72/-97,26.891,0.015,46.69,38.16,2.8,0.564,✔️,0.0,False,GPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),962.0,+77/-111,29.736,0.02,47.26,34.13,0.84,0.094,✔️,0.0,False,CPU -55,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),946.0,+72/-94,26.072,0.008,47.92,41.57,0.2,0.037,✔️,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),918.0,+76/-128,29.619,0.0,49.05,35.34,139.84,0.193,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),593.0,+109/-189,44.093,0.0,56.3,52.0,0.15,0.023,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1622.0,+89/-65,0.744,9.1,3.52,6.178,442.09,6.381,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1553.0,+101/-58,0.642,12.06,4.41,6.446,1923.88,9.103,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1530.0,+91/-51,0.635,13.2,6.16,7.53,4.91,0.694,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1521.0,+94/-55,0.594,13.66,5.27,7.555,1923.88,1.329,✔️,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1483.0,+72/-60,0.512,15.72,5.78,9.442,1152.95,1.784,✔️,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1471.0,+48/-54,0.45,16.4,10.21,10.551,2775.44,8.849,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabICL (default) [3.33% IMPUTED]](https://arxiv.org/abs/2502.05564),1448.0,+63/-54,0.465,17.8,6.78,9.808,8.06,1.743,✔️,3.33,True,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1447.0,+66/-50,0.403,17.85,8.23,11.202,2229.7,1.387,✔️,0.0,False,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1414.0,+52/-37,0.337,19.94,12.33,12.695,328.96,1.102,✔️,0.0,False,CPU +9,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1405.0,+66/-56,0.386,20.51,6.99,9.55,4982.66,358.753,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[LimiX (default) [17.04% IMPUTED]](https://arxiv.org/abs/2509.03505),1399.0,+82/-74,0.461,20.91,6.91,13.142,1.92,0.489,➖,17.04,True,GPU +11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1392.0,+73/-56,0.326,21.38,9.78,12.114,2229.7,0.133,✔️,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1387.0,+61/-51,0.309,21.72,11.15,12.473,1055.6,0.482,✔️,0.0,False,CPU +13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1382.0,+60/-53,0.318,22.03,10.14,12.568,1055.6,0.063,✔️,0.0,False,CPU +14,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1381.0,+54/-52,0.325,22.12,10.58,12.444,4.27,0.064,✔️,0.0,False,CPU +15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1378.0,+50/-53,0.273,22.27,11.46,12.341,2775.44,0.415,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1365.0,+38/-30,0.235,23.2,15.51,13.55,328.96,0.126,✔️,0.0,False,CPU +17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1364.0,+62/-49,0.281,23.27,12.54,13.598,472.99,0.621,✔️,0.0,False,CPU +18,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1363.0,+60/-54,0.321,23.33,11.66,14.462,250.36,1.708,➖,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1349.0,+90/-92,0.35,24.28,9.59,13.268,3383.91,8.396,✔️,0.0,False,GPU +20,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1349.0,+46/-57,0.238,24.28,11.66,13.228,3383.91,0.51,✔️,0.0,False,GPU +21,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1343.0,+59/-53,0.274,24.7,8.63,11.238,4982.66,42.199,✔️,0.0,False,GPU +22,🧠🔁,Neural Network,[Mitra (default) [33.33% IMPUTED]](https://arxiv.org/abs/2510.21204),1336.0,+85/-81,0.392,25.19,8.37,14.071,103.86,2.616,✔️,33.33,True,GPU +23,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1332.0,+47/-40,0.207,25.42,15.95,14.112,472.99,0.13,✔️,0.0,False,CPU +24,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1329.0,+86/-76,0.342,25.66,8.82,14.037,3031.01,27.044,✔️,33.33,True,GPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1315.0,+57/-52,0.181,26.62,15.61,14.232,642.33,2.767,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1302.0,+48/-40,0.179,27.55,18.31,13.85,2241.65,2.415,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1298.0,+64/-63,0.224,27.84,15.26,14.804,8.72,0.126,✔️,0.0,False,GPU +28,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1294.0,+48/-42,0.151,28.12,18.34,15.717,1560.02,0.404,✔️,0.0,False,CPU +29,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1288.0,+46/-56,0.143,28.5,16.61,15.503,642.33,0.146,✔️,0.0,False,GPU +30,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1283.0,+60/-58,0.196,28.88,12.54,16.251,12.86,3.167,➖,0.0,False,GPU +31,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1282.0,+79/-74,0.271,28.91,9.11,15.859,3031.01,0.514,✔️,33.33,True,GPU +32,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1269.0,+62/-70,0.224,29.87,11.33,14.603,49.67,43.824,✔️,0.0,False,GPU +33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1261.0,+47/-45,0.1,30.39,19.21,16.428,1560.02,0.052,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1257.0,+45/-40,0.123,30.67,16.95,16.27,13.5,0.311,✔️,0.0,False,GPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1255.0,+58/-55,0.139,30.82,18.73,15.872,559.38,4.08,✔️,0.0,False,CPU +36,🧠⚡,Foundation Model,[TabPFNv2 (default) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1255.0,+80/-80,0.265,30.82,11.94,17.252,2.71,0.276,✔️,33.33,True,GPU +37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1252.0,+43/-44,0.096,31.02,20.55,15.186,9.33,1.116,✔️,0.0,False,GPU +38,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1236.0,+48/-48,0.098,32.15,19.88,17.481,4.98,0.049,✔️,0.0,False,CPU +39,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1222.0,+66/-56,0.098,33.12,22.35,16.741,1.24,0.11,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1215.0,+44/-38,0.069,33.59,22.92,16.046,2241.65,0.136,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1185.0,+57/-59,0.053,35.61,22.79,17.902,559.38,0.276,✔️,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1183.0,+47/-46,0.045,35.73,25.84,18.377,125.38,0.674,✔️,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1174.0,+47/-36,0.044,36.37,30.97,17.518,0.81,0.093,✔️,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1153.0,+50/-57,0.031,37.69,26.14,19.721,169.75,0.603,✔️,0.0,False,CPU +45,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1143.0,+50/-48,0.025,38.33,28.36,19.532,125.38,0.069,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1119.0,+42/-57,0.011,39.84,28.41,20.45,169.75,0.059,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1093.0,+53/-48,0.013,41.43,31.96,20.434,4.95,0.091,✔️,0.0,False,CPU +48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1056.0,+60/-77,0.015,43.52,28.3,21.82,2.56,0.267,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1050.0,+83/-98,0.053,43.85,29.51,27.138,162.68,0.405,✔️,0.0,False,CPU +50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1040.0,+102/-103,0.077,44.38,26.98,25.899,0.85,0.129,➖,0.0,False,GPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1028.0,+61/-93,0.017,45.01,32.01,24.552,139.84,3.917,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1022.0,+87/-119,0.035,45.32,31.62,27.83,162.68,0.054,✔️,0.0,False,CPU +53,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+63/-69,0.006,46.39,39.96,24.634,0.25,0.029,✔️,0.0,False,CPU +54,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),976.0,+72/-99,0.013,47.53,38.88,26.921,2.8,0.564,✔️,0.0,False,GPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),962.0,+76/-110,0.02,48.15,34.66,29.763,0.84,0.094,✔️,0.0,False,CPU +56,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),947.0,+71/-95,0.008,48.76,42.35,26.102,0.2,0.037,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),917.0,+76/-127,0.001,49.94,35.87,29.648,139.84,0.193,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),593.0,+110/-191,0.0,57.28,52.85,44.116,0.15,0.023,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_all/winrate_matrix.png.zip index 7b76b25a74a9c1f16d3326e8b8483f059be79525..91ba988092cab2bd5a52b545f153d039acf39fab 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c89e5ad4a6eee54e4760b89a82157cbe28b63f8e0263ee8da1d9b07a9cd34b29 -size 2760741 +oid sha256:d0a668574979001b16557139565bdc4af0e5982e8ef56f9c080dafd0572c1005 +size 2756384 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index 2b28a717a097c6ba349ae48981e7d3e448bd2aab..3da507f1b75d836df350eb1acf5bea57ea1e8e93 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:70f3377e354ad69aef50ae476d5d3f30ac41432724ac61f7970814a078181872 -size 430512 +oid sha256:dc00846a6609a718d64484eebc3ec143d6d2d93ee3a4f9338020df27b08f3f35 +size 446030 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip index 46890467250e8e4eaa06739f177d4287b0e2cdfc..78dab23977984111418a7519d9cd0d5f10b3d389 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:40f924465c075a5bcdf59f115279d92fa0b195ecc6ed43b4e2de6f33004a2d00 -size 937488 +oid sha256:7a262d023cdf7dbdc06467ae1f2e4f3d9f1d50253f09390aadb34786e5351d50 +size 917303 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/tuning-impact-elo.png.zip index 7055df9e05fc9288f8bde2f4b850b754ac91584b..fc94329969b9e24080782534b857f5fc06a8669c 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7c884aafd87b05486a559e8b56e19ec4ddd5898ff247f1fc2bebd8d261949043 -size 134896 +oid sha256:05f5ff2c2606c96ebb106e60a336942ddf9f741bfacdfd5d799519bd8f38381e +size 138075 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/website_leaderboard.csv index 8600c4c711836435bd5453accf4ca2d24c2526fe..5fc6559664ce3fa2fa919223a478ae747350aa4f 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/website_leaderboard.csv @@ -1,55 +1,56 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1952.0,+156/-57,2.22,0.904,5.02,2.43,275.07,2.448,✔️,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1887.0,+175/-63,2.498,0.85,6.53,2.71,325.91,2.156,✔️,0.0,False,CPU -2,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1744.0,+147/-88,4.733,0.63,11.19,5.89,619.15,0.198,✔️,0.0,False,CPU -3,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1739.0,+142/-86,5.026,0.636,11.38,4.9,619.15,0.016,✔️,0.0,False,CPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1736.0,+118/-39,5.448,0.572,11.51,8.06,1633.8,2.053,✔️,0.0,False,GPU -5,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1722.0,+131/-79,4.771,0.63,12.09,5.58,2.21,0.025,✔️,0.0,False,CPU -6,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1705.0,+146/-89,7.516,0.544,12.8,7.1,110.13,0.386,✔️,0.0,False,CPU -7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1688.0,+146/-81,7.932,0.524,13.49,8.59,86.85,0.668,✔️,0.0,False,CPU -8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1668.0,+198/-135,7.278,0.512,14.38,5.61,804.65,0.71,✔️,0.0,False,GPU -9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1667.0,+146/-138,6.915,0.543,14.46,7.14,2139.17,6.41,✔️,0.0,False,GPU -10,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1650.0,+204/-120,8.389,0.489,15.23,5.4,842.24,15.119,✔️,0.0,False,GPU -11,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1638.0,+126/-65,7.036,0.372,15.81,8.84,1633.8,0.088,✔️,0.0,False,GPU -12,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1628.0,+117/-76,8.384,0.419,16.27,11.67,110.13,0.061,✔️,0.0,False,CPU -13,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1612.0,+196/-154,7.986,0.456,17.02,6.32,804.65,0.063,✔️,0.0,False,GPU -14,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1600.0,+193/-114,9.041,0.42,17.65,6.9,842.24,1.465,✔️,0.0,False,GPU -15,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1572.0,+156/-126,8.638,0.417,19.02,8.0,2.3,0.896,✔️,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1568.0,+158/-86,9.749,0.323,19.23,12.24,86.85,0.078,✔️,0.0,False,CPU -17,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1556.0,+165/-162,9.22,0.42,19.88,8.63,8.53,3.578,✔️,0.0,False,GPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1542.0,+120/-80,9.79,0.254,20.59,13.66,2139.17,0.349,✔️,0.0,False,GPU -19,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1527.0,+170/-152,9.027,0.346,21.37,7.59,4860.88,475.791,✔️,0.0,False,GPU -20,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1515.0,+96/-110,9.128,0.247,22.0,14.66,1156.69,1.558,✔️,0.0,False,CPU -21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1480.0,+146/-104,10.686,0.195,23.86,15.28,572.82,1.935,✔️,0.0,False,GPU -22,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1465.0,+127/-110,12.878,0.171,24.69,18.15,270.2,0.192,✔️,0.0,False,CPU -23,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1451.0,+121/-141,10.869,0.227,25.47,15.18,0.49,0.053,✔️,0.0,False,CPU -24,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1444.0,+132/-115,12.706,0.161,25.86,19.53,572.82,0.109,✔️,0.0,False,GPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1443.0,+149/-178,10.498,0.243,25.88,14.7,5.5,0.074,✔️,0.0,False,GPU -26,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1422.0,+105/-126,11.44,0.124,27.06,20.82,371.59,2.386,✔️,0.0,False,CPU -27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1420.0,+83/-65,10.931,0.079,27.19,24.01,6.41,0.247,✔️,0.0,False,GPU -28,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1407.0,+118/-135,13.968,0.136,27.88,20.37,96.61,0.754,➖,0.0,False,GPU -29,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1393.0,+126/-107,14.005,0.093,28.67,24.4,270.2,0.015,✔️,0.0,False,CPU -30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1371.0,+97/-128,11.539,0.072,29.85,23.72,1156.69,0.054,✔️,0.0,False,CPU -31,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1365.0,+128/-130,12.77,0.085,30.14,19.07,4860.88,42.641,✔️,0.0,False,GPU -32,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1351.0,+127/-111,15.839,0.052,30.89,26.74,1.07,0.018,✔️,0.0,False,CPU -33,🧠⚡,Foundation Model,[LimiX (default) [55.56% IMPUTED]](https://arxiv.org/abs/2509.03505),1348.0,+200/-227,18.235,0.233,31.07,13.41,0.63,0.019,➖,55.56,True,GPU -34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1337.0,+97/-131,12.469,0.067,31.63,28.44,0.29,0.033,✔️,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1313.0,+52/-59,14.156,0.002,32.88,31.05,42.95,0.226,✔️,0.0,False,CPU -36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1308.0,+86/-82,13.961,0.009,33.14,30.03,8.34,0.25,✔️,0.0,False,GPU -37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1275.0,+59/-86,13.465,0.0,34.77,32.44,40.36,0.318,✔️,0.0,False,CPU -38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1274.0,+76/-73,15.136,0.005,34.81,31.22,42.95,0.021,✔️,0.0,False,CPU -39,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1274.0,+171/-184,17.027,0.032,34.82,21.22,47.03,45.876,✔️,0.0,False,GPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1272.0,+81/-116,15.039,0.037,34.91,29.62,371.59,0.102,✔️,0.0,False,CPU -41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1220.0,+68/-96,15.181,0.0,37.36,35.66,1.86,0.046,✔️,0.0,False,CPU -42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1214.0,+64/-68,15.182,0.0,37.63,35.91,40.36,0.028,✔️,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1199.0,+131/-210,18.054,0.043,38.31,32.82,1.52,0.115,✔️,0.0,False,CPU -44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1157.0,+147/-229,24.041,0.008,40.12,35.81,43.64,0.085,✔️,0.0,False,CPU -45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1138.0,+96/-184,20.737,0.011,40.89,31.9,513.06,78.02,✔️,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1107.0,+165/-222,25.035,0.005,42.07,38.55,43.64,0.022,✔️,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+83/-194,22.283,0.0,45.68,44.94,0.12,0.014,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),983.0,+131/-215,26.738,0.0,46.17,44.55,0.22,0.024,✔️,0.0,False,CPU -49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),969.0,+198/-439,28.52,0.025,46.56,40.45,2.62,0.186,✔️,0.0,False,GPU -50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),916.0,+95/-189,30.36,0.0,47.91,47.06,0.24,0.036,➖,0.0,False,GPU -51,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),912.0,+93/-235,24.149,0.0,48.0,47.54,0.1,0.021,✔️,0.0,False,CPU -52,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),884.0,+82/-127,27.802,0.0,48.62,48.24,513.06,2.568,✔️,0.0,False,CPU -53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),508.0,+104/-387,45.598,0.0,53.27,53.24,0.11,0.087,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1954.0,+158/-59,0.905,5.04,2.43,2.22,275.07,2.448,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1888.0,+177/-63,0.852,6.57,2.71,2.498,325.91,2.156,✔️,0.0,False,CPU +2,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1747.0,+146/-88,0.634,11.21,5.89,4.733,619.15,0.198,✔️,0.0,False,CPU +3,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1741.0,+143/-86,0.64,11.43,4.9,5.026,619.15,0.016,✔️,0.0,False,CPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1738.0,+117/-40,0.577,11.54,8.06,5.448,1633.8,2.053,✔️,0.0,False,GPU +5,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1724.0,+130/-78,0.634,12.13,5.58,4.771,2.21,0.025,✔️,0.0,False,CPU +6,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1707.0,+148/-88,0.546,12.85,7.1,7.516,110.13,0.386,✔️,0.0,False,CPU +7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1691.0,+147/-80,0.527,13.54,8.6,7.932,86.85,0.668,✔️,0.0,False,CPU +8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1669.0,+200/-135,0.515,14.52,5.62,7.278,804.65,0.71,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1667.0,+148/-140,0.547,14.62,7.15,6.915,2139.17,6.41,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1650.0,+207/-120,0.491,15.4,5.4,8.389,842.24,15.119,✔️,0.0,False,GPU +11,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1639.0,+125/-66,0.378,15.94,8.85,7.036,1633.8,0.088,✔️,0.0,False,GPU +12,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1631.0,+117/-75,0.422,16.33,11.69,8.384,110.13,0.061,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1613.0,+198/-157,0.46,17.2,6.32,7.986,804.65,0.063,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1600.0,+197/-114,0.423,17.84,6.91,9.041,842.24,1.465,✔️,0.0,False,GPU +15,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1574.0,+158/-128,0.42,19.14,8.0,8.638,2.3,0.896,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1569.0,+159/-87,0.328,19.43,12.28,9.749,86.85,0.078,✔️,0.0,False,CPU +17,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1556.0,+164/-164,0.423,20.12,8.65,9.22,8.53,3.578,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1542.0,+121/-82,0.258,20.83,13.71,9.79,2139.17,0.349,✔️,0.0,False,GPU +19,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1526.0,+171/-154,0.35,21.65,7.61,9.027,4860.88,475.791,✔️,0.0,False,GPU +20,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1516.0,+97/-111,0.251,22.2,14.71,9.128,1156.69,1.558,✔️,0.0,False,CPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1482.0,+144/-102,0.197,24.1,15.33,10.686,572.82,1.935,✔️,0.0,False,GPU +22,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1468.0,+127/-108,0.178,24.83,18.2,12.878,270.2,0.192,✔️,0.0,False,CPU +23,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1454.0,+121/-141,0.23,25.65,15.21,10.869,0.49,0.053,✔️,0.0,False,CPU +24,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1448.0,+132/-113,0.164,25.98,19.56,12.706,572.82,0.109,✔️,0.0,False,GPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1444.0,+152/-176,0.246,26.2,14.75,10.498,5.5,0.074,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1423.0,+106/-128,0.126,27.38,20.92,11.44,371.59,2.386,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1421.0,+82/-64,0.083,27.49,24.18,10.931,6.41,0.247,✔️,0.0,False,GPU +28,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1409.0,+121/-134,0.14,28.16,20.48,13.968,96.61,0.754,➖,0.0,False,GPU +29,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1396.0,+126/-104,0.094,28.89,24.54,14.005,270.2,0.015,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1372.0,+98/-130,0.074,30.23,23.87,11.539,1156.69,0.054,✔️,0.0,False,CPU +31,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1364.0,+128/-130,0.089,30.65,19.2,12.77,4860.88,42.641,✔️,0.0,False,GPU +32,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1353.0,+128/-112,0.054,31.25,26.91,15.839,1.07,0.018,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[LimiX (default) [55.56% IMPUTED]](https://arxiv.org/abs/2509.03505),1347.0,+203/-227,0.234,31.56,13.47,18.235,0.63,0.019,➖,55.56,True,GPU +34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1338.0,+97/-132,0.069,32.09,28.7,12.469,0.29,0.033,✔️,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1314.0,+49/-57,0.002,33.31,31.42,14.156,42.95,0.226,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1309.0,+88/-83,0.01,33.6,30.32,13.961,8.34,0.25,✔️,0.0,False,GPU +37,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1299.0,+69/-106,0.012,34.12,30.92,15.997,16.41,16.24,➖,0.0,False,GPU +38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1276.0,+75/-72,0.005,35.28,31.55,15.136,42.95,0.021,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1275.0,+59/-86,0.0,35.32,32.86,13.465,40.36,0.318,✔️,0.0,False,CPU +40,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1274.0,+171/-188,0.038,35.4,21.37,17.027,47.03,45.876,✔️,0.0,False,GPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1272.0,+80/-118,0.038,35.48,29.97,15.039,371.59,0.102,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1221.0,+71/-97,0.0,37.96,36.16,15.181,1.86,0.046,✔️,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1213.0,+66/-70,0.0,38.31,36.48,15.182,40.36,0.028,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1199.0,+132/-210,0.044,38.99,33.24,18.054,1.52,0.115,✔️,0.0,False,CPU +45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1155.0,+148/-230,0.008,40.91,36.46,24.041,43.64,0.085,✔️,0.0,False,CPU +46,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1134.0,+98/-186,0.01,41.77,32.46,20.737,513.06,78.02,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1106.0,+163/-220,0.004,42.9,39.28,25.035,43.64,0.022,✔️,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+86/-199,0.0,46.52,45.72,22.283,0.12,0.014,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),979.0,+130/-214,0.0,47.15,45.55,26.738,0.22,0.024,✔️,0.0,False,CPU +50,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),971.0,+197/-439,0.024,47.37,41.15,28.52,2.62,0.186,✔️,0.0,False,GPU +51,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),912.0,+98/-240,0.0,48.88,48.38,24.149,0.1,0.021,✔️,0.0,False,CPU +52,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),912.0,+95/-189,0.0,48.88,48.02,30.36,0.24,0.036,➖,0.0,False,GPU +53,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),880.0,+81/-124,0.0,49.62,49.24,27.802,513.06,2.568,✔️,0.0,False,CPU +54,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),506.0,+104/-386,0.0,54.27,54.24,45.598,0.11,0.087,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/winrate_matrix.png.zip index ff057e97e1002941df9cb2c12eabeaf6c999273b..331d702328759d00717dac14de3ab30c95edf137 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e1f3549cd30e501817685fc7855e5995bfaf4bd76a68595093515a9924971853 -size 2264366 +oid sha256:b381e58b0213884ae21a22fc58b81600d65c3a1615f9d80061e5f0c8268d0cc7 +size 2468063 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index ce386c9526dbe7732ad2ebfffe93a5b6c4539aeb..fff844d5b93a0363578e495c1ef83a14e363c381 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ea778aab0e2d8b7d4f5d32deed403739b54a9b2e2d552726af97d1bddfae6b25 -size 482352 +oid sha256:619f76ba9eaba1be6416f33e13c135aec9e22c3ad9673210ce7860b671a9a1a8 +size 496497 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip index c99661ff7e4dd0b302acd30f15b22c73e11465d9..8c6802ef045f62620f762f3dc92b0d3163d74e42 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2897ea1ced63aaca1e16be267bd05e09763eef8947f6898124bb07315d3249fb -size 1038898 +oid sha256:c0ec9ae17518e67e2aa8628137fd6a52eb69c623e518f9ee0c39ea0d3c71350c +size 1032867 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_small/tuning-impact-elo.png.zip index fa00e8ecc45f77952bb95a01ec33ff667fa57090..e6d873c22883e9af5e90800f3cfdb12ca6a4a393 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:159170cbc24b8b4875788bf77a9b0a9165ee5a58d198217e75684db9128995a1 -size 190540 +oid sha256:f4fa3ea15c5cd58d848f71b5d4523043fe358ca18c258c4a0afd49f8d3810a84 +size 193633 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_binary/datasets_small/website_leaderboard.csv index ce1c168e8451c3bf5ad5147a53098f47666dcd67..2eb55ff17814c948a636983c7293ac1946ec1ccf 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_small/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_small/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1573.0,+135/-64,5.566,0.7,10.43,4.02,2052.28,8.252,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1573.0,+93/-51,7.007,0.72,10.43,5.44,5.96,0.554,✔️,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1568.0,+103/-72,7.828,0.675,10.63,4.33,593.57,8.084,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1545.0,+122/-58,6.873,0.658,11.64,4.75,2052.28,1.266,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[Mitra (default) [4.76% IMPUTED]](https://arxiv.org/abs/2510.21204),1479.0,+89/-54,10.506,0.564,14.91,6.08,143.16,3.761,✔️,4.76,True,GPU -5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1466.0,+105/-56,10.456,0.494,15.6,6.44,3422.16,45.847,✔️,4.76,True,GPU -6,🧠⚡,Foundation Model,[LimiX (default) [0.53% IMPUTED]](https://arxiv.org/abs/2509.03505),1466.0,+94/-65,10.913,0.565,15.65,5.59,3.78,0.54,➖,0.53,True,GPU -7,🧠⚡,Foundation Model,[TabICL (default) [4.76% IMPUTED]](https://arxiv.org/abs/2502.05564),1450.0,+72/-70,10.012,0.477,16.52,6.13,7.78,1.706,✔️,4.76,True,GPU -8,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1423.0,+68/-46,12.692,0.391,18.1,11.32,2950.72,15.595,✔️,0.0,False,GPU -9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1410.0,+74/-55,12.838,0.353,18.88,10.15,2457.54,1.658,✔️,0.0,False,GPU -10,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1404.0,+80/-70,12.373,0.36,19.23,10.86,1921.63,1.718,✔️,0.0,False,CPU -11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1398.0,+79/-72,9.728,0.394,19.65,6.71,5351.68,268.579,✔️,0.0,False,GPU -12,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1388.0,+125/-63,13.062,0.394,20.24,6.64,3422.16,0.983,✔️,4.76,True,GPU -13,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1382.0,+76/-66,14.629,0.387,20.62,9.69,317.91,1.983,➖,0.0,False,GPU -14,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1366.0,+84/-73,10.535,0.344,21.63,6.93,5351.68,37.621,✔️,0.0,False,GPU -15,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1357.0,+44/-36,14.691,0.244,22.23,14.91,512.57,1.18,✔️,0.0,False,CPU -16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1351.0,+73/-54,13.836,0.26,22.59,12.56,2457.54,0.178,✔️,0.0,False,GPU -17,🧠⚡,Foundation Model,[TabPFNv2 (default) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1349.0,+99/-81,15.05,0.393,22.76,8.85,4.05,0.421,✔️,4.76,True,GPU -18,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1325.0,+37/-34,15.136,0.178,24.31,17.21,512.57,0.243,✔️,0.0,False,CPU -19,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1322.0,+74/-59,14.571,0.209,24.48,12.95,2950.72,1.225,✔️,0.0,False,GPU -20,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1312.0,+80/-64,14.657,0.215,25.21,10.71,4806.04,0.512,✔️,0.0,False,GPU -21,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1303.0,+47/-43,15.746,0.159,25.78,17.73,1096.5,0.584,✔️,0.0,False,CPU -22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1301.0,+59/-44,15.688,0.185,25.92,16.88,4.81,0.08,✔️,0.0,False,CPU -23,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1299.0,+55/-42,15.755,0.168,26.08,18.41,1096.5,0.074,✔️,0.0,False,CPU -24,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1294.0,+86/-77,13.517,0.289,26.4,9.3,51.83,43.615,✔️,0.0,False,GPU -25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1284.0,+46/-66,15.706,0.169,27.09,15.33,711.84,2.983,✔️,0.0,False,GPU -26,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1282.0,+44/-47,16.16,0.155,27.18,18.29,693.49,1.398,✔️,0.0,False,CPU -27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1273.0,+67/-61,16.605,0.208,27.82,14.52,10.08,0.152,✔️,0.0,False,GPU -28,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1272.0,+102/-109,15.948,0.254,27.92,11.14,4806.04,8.461,✔️,0.0,False,GPU -29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1260.0,+46/-48,16.89,0.123,28.71,17.9,1634.54,0.422,✔️,0.0,False,CPU -30,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1260.0,+43/-37,16.522,0.1,28.73,18.62,693.49,0.194,✔️,0.0,False,CPU -31,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1259.0,+59/-64,17.215,0.166,28.77,13.98,14.37,0.334,✔️,0.0,False,GPU -32,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1258.0,+49/-61,16.659,0.127,28.81,15.35,711.84,0.162,✔️,0.0,False,GPU -33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1252.0,+77/-56,15.831,0.133,29.21,20.12,2665.36,2.619,✔️,0.0,False,CPU -34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1239.0,+47/-48,17.421,0.095,30.14,17.24,1634.54,0.054,✔️,0.0,False,CPU -35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1219.0,+60/-79,17.727,0.136,31.5,17.51,644.55,5.974,✔️,0.0,False,CPU -36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1218.0,+46/-53,18.14,0.11,31.57,17.33,7.32,0.054,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1215.0,+52/-53,16.966,0.084,31.76,18.67,10.89,2.249,✔️,0.0,False,GPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1178.0,+69/-47,17.936,0.059,34.19,22.12,2665.36,0.143,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1170.0,+61/-84,19.083,0.055,34.69,19.95,644.55,0.398,✔️,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1163.0,+65/-80,20.437,0.064,35.18,23.33,203.15,0.968,✔️,0.0,False,CPU -41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1158.0,+56/-58,19.215,0.033,35.52,27.53,1.94,0.121,✔️,0.0,False,CPU -42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1127.0,+62/-79,21.351,0.038,37.43,25.36,203.15,0.091,✔️,0.0,False,CPU -43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1127.0,+40/-53,19.64,0.02,37.45,31.55,2.44,0.121,✔️,0.0,False,CPU -44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1106.0,+57/-92,22.062,0.042,38.77,23.91,484.22,1.025,✔️,0.0,False,CPU -45,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1073.0,+106/-126,23.941,0.109,40.71,22.04,1.3,0.15,➖,0.0,False,GPU -46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1070.0,+52/-89,22.683,0.011,40.88,24.6,484.22,0.086,✔️,0.0,False,CPU -47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1052.0,+62/-82,22.644,0.017,41.89,29.86,9.39,0.13,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1020.0,+92/-135,28.424,0.067,43.63,26.7,278.97,0.873,✔️,0.0,False,CPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1011.0,+74/-90,23.393,0.0,44.09,25.59,2.97,0.388,✔️,0.0,False,CPU -50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+56/-113,25.597,0.01,44.63,36.01,0.42,0.041,✔️,0.0,False,CPU -51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),997.0,+93/-135,28.987,0.044,44.79,28.48,278.97,0.103,✔️,0.0,False,CPU -52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),988.0,+78/-117,26.144,0.019,45.2,31.26,90.71,1.623,✔️,0.0,False,CPU -53,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),984.0,+80/-114,26.193,0.01,45.39,36.59,3.71,0.919,✔️,0.0,False,GPU -54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),962.0,+82/-136,26.896,0.011,46.44,38.56,0.33,0.043,✔️,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),951.0,+89/-163,31.021,0.029,46.94,30.75,1.48,0.139,✔️,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),922.0,+83/-149,30.397,0.001,48.17,31.33,90.71,0.125,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),616.0,+111/-175,43.448,0.0,55.88,50.04,0.22,0.02,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1572.0,+94/-51,0.713,10.61,5.61,7.055,5.96,0.554,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1571.0,+135/-67,0.698,10.63,4.09,5.613,2052.28,8.252,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1567.0,+103/-72,0.672,10.84,4.35,7.874,593.57,8.084,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1544.0,+122/-60,0.656,11.85,4.78,6.918,2052.28,1.266,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[Mitra (default) [4.76% IMPUTED]](https://arxiv.org/abs/2510.21204),1478.0,+90/-54,0.56,15.16,6.18,10.552,143.16,3.761,✔️,4.76,True,GPU +5,🧠🔁,Neural Network,[LimiX (default) [0.53% IMPUTED]](https://arxiv.org/abs/2509.03505),1466.0,+93/-64,0.554,15.86,5.71,10.959,3.78,0.54,➖,0.53,True,GPU +6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1465.0,+104/-56,0.489,15.93,6.54,10.503,3422.16,45.847,✔️,4.76,True,GPU +7,🧠⚡,Foundation Model,[TabICL (default) [4.76% IMPUTED]](https://arxiv.org/abs/2502.05564),1449.0,+72/-70,0.47,16.81,6.2,10.06,7.78,1.706,✔️,4.76,True,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1422.0,+68/-46,0.382,18.47,11.52,12.737,2950.72,15.595,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1409.0,+73/-56,0.343,19.26,10.27,12.884,2457.54,1.658,✔️,0.0,False,GPU +10,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1403.0,+80/-70,0.361,19.64,11.2,12.418,1921.63,1.718,✔️,0.0,False,CPU +11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1397.0,+79/-72,0.385,20.0,6.75,9.774,5351.68,268.579,✔️,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1388.0,+124/-63,0.387,20.58,6.77,13.105,3422.16,0.983,✔️,4.76,True,GPU +13,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1381.0,+76/-66,0.383,21.01,9.83,14.674,317.91,1.983,➖,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1365.0,+84/-72,0.345,22.05,6.98,10.581,5351.68,37.621,✔️,0.0,False,GPU +15,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1356.0,+46/-38,0.242,22.69,15.15,14.736,512.57,1.18,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1350.0,+73/-54,0.257,23.07,12.77,13.883,2457.54,0.178,✔️,0.0,False,GPU +17,🧠⚡,Foundation Model,[TabPFNv2 (default) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1348.0,+99/-81,0.379,23.21,9.03,15.096,4.05,0.421,✔️,4.76,True,GPU +18,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1324.0,+38/-35,0.177,24.79,17.48,15.179,512.57,0.243,✔️,0.0,False,CPU +19,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1322.0,+72/-59,0.209,24.96,13.12,14.615,2950.72,1.225,✔️,0.0,False,GPU +20,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1311.0,+79/-64,0.216,25.71,10.95,14.701,4806.04,0.512,✔️,0.0,False,GPU +21,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1303.0,+47/-44,0.157,26.21,18.06,15.791,1096.5,0.584,✔️,0.0,False,CPU +22,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1301.0,+66/-72,0.274,26.36,9.98,16.359,9.05,1.763,➖,0.0,False,GPU +23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1301.0,+60/-45,0.18,26.38,17.16,15.732,4.81,0.08,✔️,0.0,False,CPU +24,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1298.0,+54/-43,0.167,26.55,18.69,15.8,1096.5,0.074,✔️,0.0,False,CPU +25,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1293.0,+86/-78,0.289,26.93,9.41,13.564,51.83,43.615,✔️,0.0,False,GPU +26,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1282.0,+47/-66,0.167,27.65,15.74,15.751,711.84,2.983,✔️,0.0,False,GPU +27,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1281.0,+45/-47,0.154,27.73,18.66,16.205,693.49,1.398,✔️,0.0,False,CPU +28,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1272.0,+67/-62,0.203,28.37,15.48,16.65,10.08,0.152,✔️,0.0,False,GPU +29,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1271.0,+101/-110,0.254,28.42,11.24,15.99,4806.04,8.461,✔️,0.0,False,GPU +30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1259.0,+45/-49,0.12,29.27,18.36,16.934,1634.54,0.422,✔️,0.0,False,CPU +31,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1258.0,+42/-37,0.098,29.31,18.89,16.567,693.49,0.194,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1258.0,+60/-66,0.162,29.33,14.25,17.26,14.37,0.334,✔️,0.0,False,GPU +33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1257.0,+49/-61,0.126,29.39,15.57,16.701,711.84,0.162,✔️,0.0,False,GPU +34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1251.0,+77/-56,0.135,29.83,20.45,15.873,2665.36,2.619,✔️,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1238.0,+46/-46,0.092,30.72,17.53,17.466,1634.54,0.054,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1217.0,+60/-78,0.136,32.17,17.91,17.772,644.55,5.974,✔️,0.0,False,CPU +37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1217.0,+46/-54,0.102,32.2,17.83,18.185,7.32,0.054,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1213.0,+53/-53,0.084,32.45,19.29,17.009,10.89,2.249,✔️,0.0,False,GPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1177.0,+68/-47,0.061,34.88,22.5,17.978,2665.36,0.143,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1169.0,+60/-84,0.056,35.41,20.63,19.128,644.55,0.398,✔️,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1162.0,+64/-79,0.062,35.87,23.66,20.483,203.15,0.968,✔️,0.0,False,CPU +42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1157.0,+57/-57,0.033,36.18,27.93,19.258,1.94,0.121,✔️,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1127.0,+39/-54,0.018,38.14,32.03,19.682,2.44,0.121,✔️,0.0,False,CPU +44,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1127.0,+61/-76,0.036,38.15,25.84,21.396,203.15,0.091,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1105.0,+57/-91,0.041,39.5,24.36,22.107,484.22,1.025,✔️,0.0,False,CPU +46,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1072.0,+107/-128,0.11,41.53,22.55,23.987,1.3,0.15,➖,0.0,False,GPU +47,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1070.0,+51/-89,0.008,41.65,27.18,22.728,484.22,0.086,✔️,0.0,False,CPU +48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1051.0,+63/-83,0.019,42.7,30.36,22.686,9.39,0.13,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1019.0,+92/-136,0.067,44.46,27.1,28.465,278.97,0.873,✔️,0.0,False,CPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1011.0,+74/-90,0.0,44.91,26.47,23.434,2.97,0.388,✔️,0.0,False,CPU +51,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+56/-113,0.008,45.45,37.33,25.641,0.42,0.041,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),997.0,+93/-135,0.046,45.62,28.96,29.029,278.97,0.103,✔️,0.0,False,CPU +53,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),989.0,+78/-116,0.019,45.99,31.67,26.187,90.71,1.623,✔️,0.0,False,CPU +54,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),984.0,+80/-115,0.008,46.25,37.3,26.235,3.71,0.919,✔️,0.0,False,GPU +55,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),962.0,+83/-132,0.012,47.26,39.31,26.939,0.33,0.043,✔️,0.0,False,CPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),951.0,+88/-163,0.028,47.79,31.16,31.06,1.48,0.139,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),922.0,+84/-144,0.001,49.02,31.76,30.44,90.71,0.125,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),616.0,+112/-175,0.0,56.86,50.83,43.481,0.22,0.02,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_small/winrate_matrix.png.zip index 9c6aa5d338241b9108cb80ed3f8095eb0fa0365d..15b480a1c0956c064c4f784fdf7ae70e10af733c 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:954c29c843e4042c9765e5e6b356f70c8d3944a0e732ba6b684df6b369b532c6 -size 2725579 +oid sha256:6af8ffe010ba9279109a345279e9c8537101d8b8c3ffb16b584fc5f643eb934c +size 2746721 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index ce465ded0d2895af4a661e734b3907ad4601eec8..743c34ed370ff429dd4598b1de4eb33c03335185 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f64bdfe903509a16e33d5258f330a5a2c1b23c43062254218bd3d7941080cbf5 -size 483576 +oid sha256:d83968c004fea351a6ca5bacff713114a76e1339a9fae62ccface1449e50e0a7 +size 498546 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip index 083bd614a024246ef718acd3b04043fbf8c06841..8f0e2d08b893420eb0489bc66a4811ee15ce1ada 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:05ac7e35d5f3c73493f338740b208a7632e5ec2d446fd4b75de22099d1c55d63 -size 1037072 +oid sha256:cea277c5a38542647c4a708867dfbce2ef79b409594369cea10c6d778dd2b3d0 +size 1017812 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip index 8d9030cdf3201065790e098bb8d367c0f860f984..89a231ac3942fcd0d0e68e3c16c1c29e2c6678bc 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:658f1e0d88cc47bb0a0389366adf2dae5f864cae524e6392896a0fdcd344dd86 -size 131179 +oid sha256:d42d10234dacd5720d7286486da40b2c8e08f46cf4157ac4a473578b19166c5b +size 136669 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/website_leaderboard.csv index 27e234f4851f3594c157236e6b1e39eca32dae81..44a8f600299a84276935de595a073267406f6fbb 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1600.0,+91/-54,7.184,0.729,10.36,5.33,5.52,0.529,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1588.0,+113/-69,5.826,0.685,10.87,4.46,2046.25,8.114,✔️,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1584.0,+88/-56,8.183,0.662,11.04,4.67,592.15,8.082,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1569.0,+104/-64,7.089,0.657,11.74,4.79,2046.25,1.254,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1510.0,+92/-60,10.733,0.585,14.67,5.88,144.27,3.772,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1497.0,+79/-54,10.68,0.512,15.33,6.23,3445.6,51.274,✔️,0.0,False,GPU -6,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1486.0,+98/-79,11.297,0.565,15.96,5.68,3.76,0.527,➖,0.0,False,GPU -7,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1479.0,+79/-67,10.213,0.493,16.35,5.93,8.06,1.714,✔️,0.0,False,GPU -8,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1442.0,+65/-50,13.145,0.385,18.49,11.6,2879.46,14.626,✔️,0.0,False,GPU -9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1438.0,+75/-59,13.186,0.362,18.78,9.99,2379.18,1.595,✔️,0.0,False,GPU -10,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1430.0,+69/-59,14.423,0.406,19.25,9.32,316.0,1.931,➖,0.0,False,GPU -11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1419.0,+76/-58,12.9,0.339,19.93,11.82,1828.68,1.648,✔️,0.0,False,CPU -12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1419.0,+84/-68,10.011,0.391,19.94,6.64,5319.2,249.314,✔️,0.0,False,GPU -13,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1415.0,+108/-75,13.416,0.407,20.2,6.42,3445.6,0.995,✔️,0.0,False,GPU -14,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1399.0,+104/-71,10.637,0.361,21.18,6.68,5319.2,29.488,✔️,0.0,False,GPU -15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1381.0,+86/-58,14.163,0.273,22.36,12.32,2379.18,0.176,✔️,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1376.0,+45/-44,15.222,0.233,22.67,15.32,449.53,1.149,✔️,0.0,False,CPU -17,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1372.0,+96/-85,15.503,0.405,22.91,8.63,3.73,0.488,✔️,0.0,False,GPU -18,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1352.0,+65/-52,14.896,0.22,24.25,12.67,2879.46,0.944,✔️,0.0,False,GPU -19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1347.0,+63/-58,14.824,0.225,24.59,10.35,4575.49,0.51,✔️,0.0,False,GPU -20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1347.0,+38/-37,15.649,0.17,24.6,17.27,449.53,0.215,✔️,0.0,False,CPU -21,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1326.0,+105/-79,13.65,0.303,25.96,8.98,51.44,43.709,✔️,0.0,False,GPU -22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1326.0,+61/-49,16.157,0.189,25.97,16.69,4.7,0.08,✔️,0.0,False,CPU -23,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1323.0,+56/-53,16.287,0.151,26.23,17.99,1081.93,0.556,✔️,0.0,False,CPU -24,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1316.0,+56/-45,16.312,0.159,26.64,19.09,1081.93,0.074,✔️,0.0,False,CPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1308.0,+67/-67,16.789,0.219,27.19,14.07,9.81,0.146,✔️,0.0,False,GPU -26,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1302.0,+53/-58,16.244,0.162,27.58,15.63,592.34,2.767,✔️,0.0,False,GPU -27,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1298.0,+52/-52,16.778,0.138,27.86,19.56,645.57,1.032,✔️,0.0,False,CPU -28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1295.0,+66/-71,17.352,0.174,28.07,13.53,14.36,0.311,✔️,0.0,False,GPU -29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1292.0,+50/-50,17.207,0.129,28.32,17.46,1618.46,0.415,✔️,0.0,False,CPU -30,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1290.0,+97/-103,16.484,0.254,28.39,11.16,4575.49,8.396,✔️,0.0,False,GPU -31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1286.0,+69/-58,15.992,0.14,28.68,19.64,2576.18,2.415,✔️,0.0,False,CPU -32,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1280.0,+54/-58,17.188,0.127,29.14,15.16,592.34,0.146,✔️,0.0,False,GPU -33,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1275.0,+42/-42,17.137,0.084,29.47,19.6,645.57,0.172,✔️,0.0,False,CPU -34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1271.0,+52/-55,17.71,0.1,29.74,16.79,1618.46,0.054,✔️,0.0,False,CPU -35,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1254.0,+54/-58,18.247,0.115,30.88,16.81,6.86,0.052,✔️,0.0,False,CPU -36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1253.0,+67/-78,17.887,0.143,30.93,17.0,618.9,5.468,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1250.0,+55/-56,17.097,0.088,31.15,18.15,10.47,1.987,✔️,0.0,False,GPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1213.0,+63/-50,17.997,0.062,33.59,21.56,2576.18,0.136,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1206.0,+67/-74,19.212,0.058,34.09,19.4,618.9,0.352,✔️,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1183.0,+84/-72,21.088,0.067,35.56,23.19,199.83,0.865,✔️,0.0,False,CPU -41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1172.0,+64/-60,19.888,0.025,36.23,29.08,1.77,0.119,✔️,0.0,False,CPU -42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1150.0,+78/-78,21.933,0.04,37.6,25.09,199.83,0.086,✔️,0.0,False,CPU -43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1144.0,+40/-43,20.296,0.018,38.0,32.13,2.2,0.12,✔️,0.0,False,CPU -44,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1110.0,+95/-131,23.531,0.115,40.03,21.41,1.24,0.143,➖,0.0,False,GPU -45,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1108.0,+65/-85,22.968,0.02,40.14,26.17,440.56,0.933,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1086.0,+75/-74,22.66,0.018,41.44,29.26,8.39,0.127,✔️,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1078.0,+52/-79,23.518,0.004,41.88,24.91,440.56,0.085,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1057.0,+98/-123,28.113,0.071,43.04,26.02,273.27,0.715,✔️,0.0,False,CPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1043.0,+78/-87,23.546,0.0,43.76,24.97,2.84,0.381,✔️,0.0,False,CPU -50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1034.0,+94/-126,28.584,0.047,44.19,27.79,273.27,0.098,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1020.0,+83/-108,26.329,0.02,44.93,30.67,89.68,1.55,✔️,0.0,False,CPU -52,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1002.0,+98/-112,26.937,0.011,45.79,36.55,2.96,0.796,✔️,0.0,False,GPU -53,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+63/-105,26.578,0.003,45.87,37.86,0.42,0.04,✔️,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),989.0,+102/-139,30.404,0.03,46.39,30.04,1.44,0.138,✔️,0.0,False,CPU -55,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),976.0,+93/-114,27.729,0.012,46.96,38.72,0.31,0.042,✔️,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),953.0,+86/-138,30.595,0.001,47.95,30.7,89.68,0.12,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),639.0,+130/-215,43.912,0.0,55.9,49.79,0.21,0.019,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1599.0,+90/-55,0.723,10.53,5.5,7.231,5.52,0.529,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1587.0,+114/-70,0.683,11.08,4.49,5.873,2046.25,8.114,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1583.0,+88/-56,0.658,11.25,4.69,8.229,592.15,8.082,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1567.0,+106/-65,0.656,11.95,4.83,7.134,2046.25,1.254,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),1509.0,+91/-59,0.583,14.91,5.98,10.778,144.27,3.772,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1495.0,+80/-55,0.508,15.65,6.32,10.726,3445.6,51.274,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1486.0,+98/-78,0.555,16.17,5.79,11.342,3.76,0.527,➖,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1478.0,+80/-66,0.489,16.63,6.0,10.261,8.06,1.714,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1441.0,+65/-51,0.377,18.88,11.78,13.189,2879.46,14.626,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1436.0,+74/-57,0.355,19.16,10.11,13.231,2379.18,1.595,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1429.0,+70/-59,0.403,19.6,9.45,14.468,316.0,1.931,➖,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1418.0,+85/-67,0.384,20.3,6.67,10.056,5319.2,249.314,✔️,0.0,False,GPU +12,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1417.0,+77/-58,0.341,20.36,12.1,12.945,1828.68,1.648,✔️,0.0,False,CPU +13,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1414.0,+107/-75,0.401,20.53,6.55,13.459,3445.6,0.995,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1398.0,+105/-70,0.362,21.58,6.73,10.683,5319.2,29.488,✔️,0.0,False,GPU +15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1379.0,+86/-58,0.269,22.84,12.52,14.209,2379.18,0.176,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1374.0,+44/-43,0.233,23.14,15.56,15.267,449.53,1.149,✔️,0.0,False,CPU +17,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1371.0,+97/-83,0.393,23.35,8.81,15.549,3.73,0.488,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1351.0,+65/-52,0.22,24.7,12.84,14.939,2879.46,0.944,✔️,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1346.0,+63/-58,0.227,25.06,10.58,14.868,4575.49,0.51,✔️,0.0,False,GPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1345.0,+38/-37,0.172,25.11,17.55,15.693,449.53,0.215,✔️,0.0,False,CPU +21,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1326.0,+61/-49,0.187,26.43,16.97,16.201,4.7,0.08,✔️,0.0,False,CPU +22,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1325.0,+106/-80,0.304,26.47,9.08,13.696,51.44,43.709,✔️,0.0,False,GPU +23,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1323.0,+83/-71,0.282,26.63,10.3,16.884,8.81,1.55,➖,0.0,False,GPU +24,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1322.0,+56/-53,0.151,26.66,18.33,16.332,1081.93,0.556,✔️,0.0,False,CPU +25,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1316.0,+55/-43,0.16,27.12,19.39,16.356,1081.93,0.074,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1307.0,+67/-67,0.213,27.72,15.01,16.834,9.81,0.146,✔️,0.0,False,GPU +27,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1301.0,+53/-58,0.161,28.16,16.07,16.289,592.34,2.767,✔️,0.0,False,GPU +28,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1297.0,+51/-52,0.139,28.42,19.99,16.823,645.57,1.032,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1294.0,+66/-71,0.17,28.61,13.79,17.396,14.36,0.311,✔️,0.0,False,GPU +30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1291.0,+49/-50,0.126,28.86,17.91,17.251,1618.46,0.415,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1290.0,+95/-103,0.256,28.89,11.23,16.525,4575.49,8.396,✔️,0.0,False,GPU +32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1285.0,+69/-59,0.142,29.28,19.96,16.034,2576.18,2.415,✔️,0.0,False,CPU +33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1278.0,+55/-57,0.128,29.72,15.38,17.23,592.34,0.146,✔️,0.0,False,GPU +34,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1273.0,+40/-43,0.083,30.07,19.91,17.181,645.57,0.172,✔️,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1270.0,+51/-55,0.097,30.3,17.07,17.754,1618.46,0.054,✔️,0.0,False,CPU +36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1253.0,+54/-58,0.107,31.5,17.3,18.291,6.86,0.052,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1251.0,+67/-78,0.143,31.58,17.4,17.932,618.9,5.468,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1248.0,+55/-56,0.088,31.83,18.75,17.14,10.47,1.987,✔️,0.0,False,GPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1212.0,+63/-51,0.064,34.26,21.93,18.039,2576.18,0.136,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1204.0,+67/-73,0.058,34.79,20.06,19.257,618.9,0.352,✔️,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1182.0,+84/-72,0.065,36.24,23.51,21.134,199.83,0.865,✔️,0.0,False,CPU +42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1172.0,+64/-62,0.028,36.89,29.5,19.93,1.77,0.119,✔️,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1150.0,+78/-74,0.038,38.3,25.55,21.978,199.83,0.086,✔️,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1144.0,+40/-44,0.019,38.69,32.62,20.337,2.2,0.12,✔️,0.0,False,CPU +45,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1109.0,+94/-132,0.115,40.84,21.9,23.577,1.24,0.143,➖,0.0,False,GPU +46,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1108.0,+65/-83,0.021,40.89,26.68,23.012,440.56,0.933,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1085.0,+76/-74,0.02,42.24,29.74,22.701,8.39,0.127,✔️,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1078.0,+52/-80,0.004,42.65,27.65,23.562,440.56,0.085,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1056.0,+97/-122,0.071,43.86,26.41,28.155,273.27,0.715,✔️,0.0,False,CPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1043.0,+79/-88,0.0,44.57,25.84,23.586,2.84,0.381,✔️,0.0,False,CPU +51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1034.0,+95/-124,0.048,45.02,28.26,28.625,273.27,0.098,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1020.0,+84/-108,0.02,45.71,31.07,26.372,89.68,1.55,✔️,0.0,False,CPU +53,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1002.0,+98/-114,0.008,46.63,37.26,26.979,2.96,0.796,✔️,0.0,False,GPU +54,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+63/-102,0.004,46.71,39.28,26.621,0.42,0.04,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),989.0,+101/-141,0.03,47.23,30.44,30.443,1.44,0.138,✔️,0.0,False,CPU +56,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),977.0,+92/-114,0.012,47.78,39.45,27.771,0.31,0.042,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),954.0,+87/-138,0.001,48.79,31.12,30.638,89.68,0.12,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),640.0,+129/-216,0.0,56.88,50.58,43.946,0.21,0.019,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip index 4e908b6552c60b47211b113a80a3d20aaf093bae..74ebf9872068fd203248c17f69638a6eee097f35 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d6aee50ebbd6d0dd5d4fd9e28423ac60c0db044a477c56d377dc5cfc80758256 -size 2690375 +oid sha256:86c9fe539d1046475161fd06224850bbe79b076b09109bdbc633221924e86aab +size 2706479 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index 2840613bbd8ce9d773d8bb5c2c3a2aa293705fba..ac796fce730575a0873ab18b716b8448ea38778b 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0b0821532cd551daad0cc0c07140a2b7d022729611c88c5c1857eaa3909cbe3f -size 481972 +oid sha256:b3c81212cd232aa133611f2eac410fb28ee0e475278e554d6bda33b05b524be7 +size 498250 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip index 4814ffbb620a4c2ea86e473937d54e0115b8953a..ce083c3fe20002ccc9c42decfdb031f7b09937e5 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:38efb78b76aa096acde6bad709cca51e27be2a6c1e64a7886690446c24ada01e -size 1035298 +oid sha256:e14ac02a16f7c737e2f13b5e0e91c6b8ad30538ab771583fb354cd4a6321eab7 +size 1024258 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_all/tuning-impact-elo.png.zip index f6d80f3af39a262ca0a78f3024251e3ff7f8eb23..31cd6b18fb5329b7d0b13cb00f4df1021a8cecd0 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:891d6d61fddc25cc518f270a2cf82e3ac0deca2ddd0eeedf07ee86e5714b3f36 -size 183321 +oid sha256:48ded57a568091889c7d515cad23f6198dda025e26b86e7193556921ea6a5254 +size 181693 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_classification/datasets_all/website_leaderboard.csv index 548b97b2fc4f61d4b27068e6623257e5bb661690..7435b23f15eee20aa3d0e74a6d15a449fbf54121 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_all/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_all/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1638.0,+69/-56,6.824,0.765,8.56,3.69,545.23,6.469,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1577.0,+79/-72,6.302,0.681,10.99,3.71,2046.25,8.976,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1550.0,+72/-63,7.349,0.643,12.21,4.72,2046.25,1.329,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1535.0,+66/-61,7.777,0.643,12.96,5.76,5.76,0.794,✔️,0.0,False,GPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1485.0,+66/-53,10.48,0.493,15.62,5.79,1684.49,1.97,✔️,0.0,False,CPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1472.0,+56/-46,11.041,0.462,16.32,9.49,2879.46,12.49,✔️,0.0,False,GPU -6,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1445.0,+61/-42,12.372,0.405,17.94,8.68,2466.21,1.502,✔️,0.0,False,GPU -7,🧠⚡,Foundation Model,[TabICL (default) [5.26% IMPUTED]](https://arxiv.org/abs/2502.05564),1417.0,+62/-60,12.417,0.427,19.7,7.44,8.68,1.743,✔️,5.26,True,GPU -8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1416.0,+46/-33,13.471,0.333,19.76,12.62,382.05,1.488,✔️,0.0,False,CPU -9,🧠⚡,Foundation Model,[LimiX (default) [16.08% IMPUTED]](https://arxiv.org/abs/2509.03505),1415.0,+71/-72,12.898,0.47,19.82,6.63,3.45,0.504,➖,16.08,True,GPU -10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1404.0,+76/-50,11.426,0.39,20.47,7.33,4940.61,307.751,✔️,0.0,False,GPU -11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1394.0,+57/-54,13.269,0.329,21.15,9.99,2466.21,0.176,✔️,0.0,False,GPU -12,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1393.0,+49/-43,12.692,0.305,21.19,11.07,2879.46,0.598,✔️,0.0,False,GPU -13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1392.0,+54/-41,12.971,0.308,21.23,11.95,1372.94,0.556,✔️,0.0,False,CPU -14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1384.0,+54/-44,13.175,0.303,21.76,11.08,1372.94,0.074,✔️,0.0,False,CPU -15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1375.0,+44/-31,14.193,0.246,22.4,15.39,382.05,0.254,✔️,0.0,False,CPU -16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1365.0,+48/-44,14.314,0.271,23.04,13.33,685.87,1.455,✔️,0.0,False,CPU -17,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1362.0,+46/-44,13.81,0.29,23.26,11.95,5.72,0.076,✔️,0.0,False,CPU -18,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1357.0,+79/-74,14.612,0.398,23.59,7.8,3008.22,20.849,✔️,31.58,True,GPU -19,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1356.0,+51/-48,15.62,0.312,23.63,12.55,282.72,1.886,➖,0.0,False,GPU -20,🧠⚡,Foundation Model,[Mitra (default) [31.58% IMPUTED]](https://arxiv.org/abs/2510.21204),1355.0,+79/-79,15.045,0.41,23.72,8.41,121.91,2.776,✔️,31.58,True,GPU -21,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1353.0,+81/-75,13.855,0.319,23.8,10.42,4879.89,8.744,✔️,0.0,False,GPU -22,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1352.0,+55/-37,13.898,0.226,23.92,11.59,4879.89,0.525,✔️,0.0,False,GPU -23,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1345.0,+61/-52,13.143,0.284,24.39,8.86,4940.61,41.606,✔️,0.0,False,GPU -24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1341.0,+46/-40,14.692,0.214,24.64,16.3,685.87,0.205,✔️,0.0,False,CPU -25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1319.0,+43/-45,15.322,0.197,26.1,15.62,934.1,3.063,✔️,0.0,False,GPU -26,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1305.0,+73/-76,16.473,0.307,27.09,8.8,3008.22,0.514,✔️,31.58,True,GPU -27,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1300.0,+42/-36,16.365,0.16,27.4,18.06,2686.11,0.47,✔️,0.0,False,CPU -28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1299.0,+44/-41,15.242,0.162,27.48,18.83,2389.22,2.158,✔️,0.0,False,CPU -29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1299.0,+50/-48,16.043,0.229,27.49,15.1,10.21,0.138,✔️,0.0,False,GPU -30,🧠⚡,Foundation Model,[TabPFNv2 (default) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1282.0,+72/-71,17.271,0.292,28.68,11.75,3.37,0.315,✔️,31.58,True,GPU -31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1278.0,+53/-55,16.823,0.146,28.9,16.76,934.1,0.169,✔️,0.0,False,GPU -32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1266.0,+43/-36,17.087,0.111,29.75,19.24,2686.11,0.054,✔️,0.0,False,CPU -33,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1265.0,+62/-60,16.444,0.221,29.82,10.77,49.21,43.824,✔️,0.0,False,GPU -34,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1254.0,+60/-57,17.607,0.153,30.58,17.25,618.9,4.766,✔️,0.0,False,CPU -35,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1243.0,+41/-43,18.158,0.105,31.3,18.36,14.78,0.346,✔️,0.0,False,GPU -36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1240.0,+44/-44,18.322,0.112,31.51,20.18,6.86,0.057,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1230.0,+40/-37,17.431,0.074,32.15,22.08,10.47,1.707,✔️,0.0,False,GPU -38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1221.0,+49/-53,17.518,0.09,32.76,23.28,1.77,0.117,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1217.0,+48/-42,17.465,0.077,33.01,22.74,2389.22,0.152,✔️,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1206.0,+50/-47,19.045,0.086,33.72,21.84,189.76,0.743,✔️,0.0,False,CPU -41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1188.0,+52/-58,19.443,0.062,34.94,21.46,618.9,0.298,✔️,0.0,False,CPU -42,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1183.0,+59/-58,19.941,0.085,35.25,20.57,323.74,0.743,✔️,0.0,False,CPU -43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1179.0,+45/-38,18.579,0.064,35.49,29.61,1.79,0.12,✔️,0.0,False,CPU -44,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1166.0,+55/-51,20.27,0.062,36.33,22.54,189.76,0.079,✔️,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1146.0,+66/-57,20.807,0.056,37.59,19.77,323.74,0.076,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1084.0,+46/-55,22.16,0.01,41.26,32.08,6.83,0.147,✔️,0.0,False,CPU -47,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1037.0,+84/-85,29.833,0.041,43.74,31.33,252.58,0.528,✔️,0.0,False,CPU -48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1037.0,+68/-69,24.506,0.012,43.74,29.99,2.91,0.368,✔️,0.0,False,CPU -49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1023.0,+66/-93,27.652,0.023,44.46,29.66,139.84,3.615,✔️,0.0,False,CPU -50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1011.0,+85/-92,28.785,0.06,45.04,29.33,1.11,0.189,➖,0.0,False,GPU -51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1008.0,+78/-89,30.589,0.027,45.18,33.3,252.58,0.089,✔️,0.0,False,CPU -52,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+56/-57,26.919,0.006,45.56,39.42,0.38,0.037,✔️,0.0,False,CPU -53,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),998.0,+74/-88,27.409,0.022,45.68,37.75,3.61,0.939,✔️,0.0,False,GPU -54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),938.0,+69/-71,29.159,0.006,48.2,42.58,0.25,0.041,✔️,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),929.0,+84/-111,33.418,0.016,48.58,36.92,1.29,0.131,✔️,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),904.0,+83/-104,33.842,0.009,49.5,36.12,139.84,0.193,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),555.0,+111/-114,49.665,0.0,56.61,53.11,0.18,0.029,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1636.0,+70/-56,0.762,8.74,3.72,7.026,545.23,6.469,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1576.0,+81/-71,0.679,11.18,3.79,6.561,2046.25,8.976,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1549.0,+72/-63,0.641,12.42,4.78,7.607,2046.25,1.329,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1534.0,+65/-62,0.638,13.13,6.02,8.056,5.76,0.794,✔️,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1482.0,+65/-53,0.494,15.95,5.88,10.605,1684.49,1.97,✔️,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1471.0,+56/-45,0.456,16.62,9.75,11.265,2879.46,12.49,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1443.0,+59/-42,0.401,18.27,8.77,12.523,2466.21,1.502,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabICL (default) [5.26% IMPUTED]](https://arxiv.org/abs/2502.05564),1415.0,+63/-60,0.422,20.05,7.51,12.546,8.68,1.743,✔️,5.26,True,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1414.0,+46/-33,0.334,20.11,12.76,13.589,382.05,1.488,✔️,0.0,False,CPU +9,🧠🔁,Neural Network,[LimiX (default) [16.08% IMPUTED]](https://arxiv.org/abs/2509.03505),1414.0,+71/-70,0.461,20.14,6.79,13.139,3.45,0.504,➖,16.08,True,GPU +10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1403.0,+75/-50,0.386,20.84,7.37,11.579,4940.61,307.751,✔️,0.0,False,GPU +11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1392.0,+56/-53,0.328,21.55,10.1,13.41,2466.21,0.176,✔️,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1392.0,+54/-41,0.308,21.57,12.09,13.162,1372.94,0.556,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1392.0,+49/-42,0.306,21.58,11.33,12.894,2879.46,0.598,✔️,0.0,False,GPU +14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1383.0,+54/-44,0.303,22.13,11.19,13.366,1372.94,0.074,✔️,0.0,False,CPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1373.0,+44/-31,0.247,22.81,15.59,14.313,382.05,0.254,✔️,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1364.0,+48/-43,0.271,23.44,13.48,14.452,685.87,1.455,✔️,0.0,False,CPU +17,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1361.0,+47/-44,0.289,23.63,12.06,13.917,5.72,0.076,✔️,0.0,False,CPU +18,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1355.0,+51/-48,0.31,24.04,12.74,15.759,282.72,1.886,➖,0.0,False,GPU +19,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1355.0,+79/-74,0.393,24.07,8.05,14.795,3008.22,20.849,✔️,31.58,True,GPU +20,🧠🔁,Neural Network,[Mitra (default) [31.58% IMPUTED]](https://arxiv.org/abs/2510.21204),1353.0,+78/-79,0.406,24.18,8.57,15.178,121.91,2.776,✔️,31.58,True,GPU +21,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1352.0,+81/-75,0.32,24.21,10.51,14.001,4879.89,8.744,✔️,0.0,False,GPU +22,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1350.0,+56/-38,0.228,24.35,11.78,14.032,4879.89,0.525,✔️,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1343.0,+62/-52,0.285,24.85,8.93,13.274,4940.61,41.606,✔️,0.0,False,GPU +24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1340.0,+46/-38,0.214,25.06,16.49,14.835,685.87,0.205,✔️,0.0,False,CPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1318.0,+43/-44,0.195,26.59,15.93,15.499,934.1,3.063,✔️,0.0,False,GPU +26,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1303.0,+72/-77,0.301,27.6,9.0,16.643,3008.22,0.514,✔️,31.58,True,GPU +27,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1300.0,+42/-36,0.16,27.85,18.41,16.484,2686.11,0.47,✔️,0.0,False,CPU +28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1298.0,+45/-42,0.165,27.99,19.06,15.343,2389.22,2.158,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1298.0,+50/-50,0.228,27.99,15.76,16.156,10.21,0.138,✔️,0.0,False,GPU +30,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1292.0,+54/-65,0.238,28.41,10.21,16.572,13.7,2.033,➖,0.0,False,GPU +31,🧠⚡,Foundation Model,[TabPFNv2 (default) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1280.0,+72/-71,0.282,29.24,12.11,17.509,3.37,0.315,✔️,31.58,True,GPU +32,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1278.0,+53/-54,0.145,29.39,16.97,16.99,934.1,0.169,✔️,0.0,False,GPU +33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1266.0,+44/-36,0.111,30.24,19.51,17.209,2686.11,0.054,✔️,0.0,False,CPU +34,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1264.0,+63/-60,0.223,30.37,10.86,16.525,49.21,43.824,✔️,0.0,False,GPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1252.0,+61/-56,0.153,31.16,17.57,17.69,618.9,4.766,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1242.0,+41/-43,0.104,31.85,18.68,18.26,14.78,0.346,✔️,0.0,False,GPU +37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1239.0,+44/-43,0.109,32.06,20.62,18.424,6.86,0.057,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1229.0,+41/-37,0.075,32.77,22.66,17.528,10.47,1.707,✔️,0.0,False,GPU +39,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1222.0,+49/-53,0.091,33.27,23.5,17.624,1.77,0.117,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1216.0,+49/-41,0.081,33.61,23.04,17.548,2389.22,0.152,✔️,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1206.0,+50/-46,0.085,34.33,22.09,19.197,189.76,0.743,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1186.0,+52/-57,0.062,35.62,22.02,19.522,618.9,0.298,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1183.0,+58/-57,0.084,35.86,20.95,20.121,323.74,0.743,✔️,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1179.0,+46/-39,0.063,36.12,30.01,18.664,1.79,0.12,✔️,0.0,False,CPU +45,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1165.0,+54/-50,0.061,36.99,22.85,20.41,189.76,0.079,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1146.0,+66/-56,0.055,38.23,20.97,20.987,323.74,0.076,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1083.0,+45/-55,0.011,42.01,32.62,22.258,6.83,0.147,✔️,0.0,False,CPU +48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1037.0,+66/-68,0.012,44.54,30.82,24.567,2.91,0.368,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1036.0,+83/-83,0.042,44.57,31.85,29.935,252.58,0.528,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1023.0,+66/-95,0.024,45.25,30.02,27.724,139.84,3.615,✔️,0.0,False,CPU +51,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1010.0,+86/-92,0.061,45.88,29.96,28.847,1.11,0.189,➖,0.0,False,GPU +52,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1007.0,+77/-86,0.028,46.02,33.89,30.671,252.58,0.089,✔️,0.0,False,CPU +53,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+56/-58,0.005,46.37,40.54,27.013,0.38,0.037,✔️,0.0,False,CPU +54,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),998.0,+74/-86,0.02,46.48,38.51,27.556,3.61,0.939,✔️,0.0,False,GPU +55,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),939.0,+68/-72,0.006,49.02,43.33,29.233,0.25,0.041,✔️,0.0,False,CPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),928.0,+83/-113,0.016,49.47,37.49,33.477,1.29,0.131,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),904.0,+82/-104,0.01,50.36,36.62,33.902,139.84,0.193,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),555.0,+112/-115,0.0,57.59,53.99,49.706,0.18,0.029,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_all/winrate_matrix.png.zip index 8d3246e083a35048500bce2ae86f44141998f569..79fa359aa41177fd6f826f3f7b6a4f11be9b2b19 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c14a7249165162fda9e0dd900ccf4213d91bd623896d7428aef03cacdd7bb3f0 -size 2678386 +oid sha256:7cda3e2873ae7d434bf682d74c28287673383d51c88731d895d55bb531a00696 +size 2674003 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index 7a4318cd81da4b247f9711528d464ded18762c9b..1c8e36ec212c9c484ac2921825e335c97f5bb9d6 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:15cd9675844d68edb113b6f82a5262e07f7635d39d63eefa21a950fead1b055e -size 437424 +oid sha256:d6cdedc208dcb30ba98d90993436f8e340f56ec6902bded5409da186abc73829 +size 452212 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip index 32239e698e2456c7da88dd78df0400e1f8857a40..24fe1f84c12886df71ac4b7b0db556e101d4f82e 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4c7e93928ceada5ce77c33aab027034f339f9591c89c79fb527ea57439da632b -size 957078 +oid sha256:087ac716c9418d75f3d8dabf829bf006e116dd3b1e2a409224e490298fc72555 +size 928381 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/tuning-impact-elo.png.zip index 148af277601af416712c5832032400705cb17f03..b66e8c2d6c95b0174e99f29cae0f544aa56c6de5 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:80e128c092bdfbb917128af98332ef36cb7adda9ca14883791913b113fa3d777 -size 134430 +oid sha256:6e69c127e58610588e097cc946888596d26beac23841d6f77e59fc3190f7a902 +size 138335 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/website_leaderboard.csv index ad80d991c6cb5300d9d1df4fb398e6a63a400a57..b1925060b0a69cba38ec59f15f77092126fe6589 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/website_leaderboard.csv @@ -1,55 +1,56 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1917.0,+147/-60,2.185,0.906,4.82,2.46,275.97,2.818,✔️,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1858.0,+176/-74,2.418,0.858,6.13,2.65,314.17,2.249,✔️,0.0,False,CPU -2,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1695.0,+135/-72,5.413,0.618,11.37,6.22,697.8,0.199,✔️,0.0,False,CPU -3,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1694.0,+143/-70,5.658,0.624,11.43,5.19,697.8,0.023,✔️,0.0,False,CPU -4,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1665.0,+131/-67,5.6,0.609,12.62,5.98,2.44,0.027,✔️,0.0,False,CPU -5,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1662.0,+128/-94,7.881,0.542,12.72,7.4,207.65,0.398,✔️,0.0,False,CPU -6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1658.0,+143/-72,8.139,0.529,12.88,8.42,93.21,0.758,✔️,0.0,False,CPU -7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1639.0,+120/-89,7.26,0.515,13.72,8.72,1624.54,1.763,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1637.0,+216/-124,7.552,0.54,13.82,3.83,783.29,13.79,✔️,0.0,False,GPU -9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1616.0,+142/-128,7.563,0.529,14.77,7.59,2338.19,14.248,✔️,0.0,False,GPU -10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1594.0,+182/-116,8.64,0.46,15.79,6.1,1398.9,0.645,✔️,0.0,False,GPU -11,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1589.0,+114/-75,8.696,0.427,16.03,11.85,207.65,0.056,✔️,0.0,False,CPU -12,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1585.0,+204/-117,8.38,0.468,16.22,6.11,783.29,1.924,✔️,0.0,False,GPU -13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1555.0,+126/-87,8.734,0.335,17.7,9.55,1624.54,0.083,✔️,0.0,False,GPU -14,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1544.0,+166/-78,9.818,0.346,18.22,11.76,93.21,0.087,✔️,0.0,False,CPU -15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1544.0,+180/-146,9.353,0.41,18.26,6.86,1398.9,0.062,✔️,0.0,False,GPU -16,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1533.0,+146/-118,9.128,0.415,18.82,8.42,2.3,0.82,✔️,0.0,False,GPU -17,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1512.0,+160/-142,9.75,0.413,19.9,9.15,7.51,3.146,✔️,0.0,False,GPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1500.0,+109/-79,10.252,0.264,20.5,14.09,2338.19,0.54,✔️,0.0,False,GPU -19,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1478.0,+154/-137,9.97,0.325,21.66,8.15,4805.66,461.797,✔️,0.0,False,GPU -20,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1467.0,+86/-76,10.088,0.234,22.26,15.27,1169.3,1.442,✔️,0.0,False,CPU -21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1455.0,+142/-105,12.545,0.214,22.91,15.36,893.93,0.205,✔️,0.0,False,CPU -22,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1430.0,+148/-126,10.933,0.255,24.28,14.98,0.47,0.052,✔️,0.0,False,CPU -23,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1413.0,+135/-129,12.445,0.175,25.19,16.24,709.85,2.929,✔️,0.0,False,GPU -24,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1388.0,+158/-107,13.595,0.141,26.62,20.06,893.93,0.016,✔️,0.0,False,CPU -25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1378.0,+132/-132,14.515,0.145,27.13,20.54,709.85,0.19,✔️,0.0,False,GPU -26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1376.0,+145/-149,12.68,0.218,27.26,15.69,5.67,0.081,✔️,0.0,False,GPU -27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1351.0,+84/-82,13.54,0.071,28.67,25.09,6.98,0.233,✔️,0.0,False,GPU -28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1349.0,+97/-138,14.46,0.112,28.74,21.98,389.62,2.114,✔️,0.0,False,CPU -29,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1341.0,+136/-132,15.921,0.122,29.18,21.44,84.36,0.919,➖,0.0,False,GPU -30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1338.0,+83/-84,12.269,0.076,29.33,23.8,1169.3,0.053,✔️,0.0,False,CPU -31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1334.0,+125/-102,15.629,0.086,29.6,25.47,2.09,0.019,✔️,0.0,False,CPU -32,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1322.0,+111/-117,13.703,0.077,30.22,19.83,4805.66,43.211,✔️,0.0,False,GPU -33,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1321.0,+116/-99,13.699,0.062,30.27,22.3,52.45,0.219,✔️,0.0,False,CPU -34,🧠⚡,Foundation Model,[LimiX (default) [60.00% IMPUTED]](https://arxiv.org/abs/2509.03505),1303.0,+188/-187,18.746,0.21,31.21,14.25,0.44,0.016,➖,60.0,True,GPU -35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1289.0,+90/-106,13.714,0.061,31.97,28.98,0.28,0.033,✔️,0.0,False,CPU -36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1284.0,+124/-111,14.614,0.063,32.2,24.11,52.45,0.021,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1277.0,+79/-71,14.632,0.009,32.57,29.74,10.5,0.606,✔️,0.0,False,GPU -38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1259.0,+71/-86,13.754,0.026,33.5,30.99,38.42,0.32,✔️,0.0,False,CPU -39,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1241.0,+122/-150,17.572,0.029,34.41,21.89,46.9,45.623,✔️,0.0,False,GPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1214.0,+80/-141,17.64,0.033,35.73,30.58,389.62,0.112,✔️,0.0,False,CPU -41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1201.0,+75/-84,15.542,0.012,36.32,34.32,38.42,0.028,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1176.0,+63/-70,16.74,0.0,37.5,35.95,1.87,0.046,✔️,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1139.0,+109/-210,21.377,0.038,39.1,33.8,1.61,0.112,✔️,0.0,False,CPU -44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1099.0,+142/-227,26.806,0.007,40.78,36.67,43.46,0.087,✔️,0.0,False,CPU -45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1068.0,+95/-197,24.396,0.009,41.96,33.16,331.68,44.947,✔️,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1048.0,+150/-236,28.029,0.004,42.71,39.35,43.46,0.021,✔️,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+99/-176,22.389,0.0,44.35,43.24,0.12,0.013,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),928.0,+135/-232,29.601,0.0,46.49,44.98,0.2,0.023,✔️,0.0,False,CPU -49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),914.0,+201/-459,31.192,0.022,46.88,41.22,2.8,0.224,✔️,0.0,False,GPU -50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),880.0,+87/-199,26.755,0.0,47.71,47.28,0.09,0.019,✔️,0.0,False,CPU -51,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),859.0,+109/-176,32.973,0.0,48.2,47.41,0.2,0.033,➖,0.0,False,GPU -52,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),818.0,+67/-150,32.101,0.0,49.06,48.67,331.68,1.626,✔️,0.0,False,CPU -53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),447.0,+126/-378,49.305,0.0,53.34,53.31,0.12,0.101,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1920.0,+148/-61,0.907,4.83,2.46,2.185,275.97,2.818,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1860.0,+178/-76,0.86,6.17,2.65,2.418,314.17,2.249,✔️,0.0,False,CPU +2,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1698.0,+136/-72,0.621,11.39,6.22,5.413,697.8,0.199,✔️,0.0,False,CPU +3,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1696.0,+143/-70,0.627,11.47,5.19,5.658,697.8,0.023,✔️,0.0,False,CPU +4,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1668.0,+130/-66,0.613,12.65,5.98,5.6,2.44,0.027,✔️,0.0,False,CPU +5,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1665.0,+127/-92,0.544,12.77,7.4,7.881,207.65,0.398,✔️,0.0,False,CPU +6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1661.0,+144/-72,0.533,12.92,8.42,8.139,93.21,0.758,✔️,0.0,False,CPU +7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1641.0,+122/-91,0.519,13.83,8.73,7.26,1624.54,1.763,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1638.0,+216/-124,0.542,13.97,3.83,7.552,783.29,13.79,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1617.0,+145/-129,0.533,14.91,7.6,7.563,2338.19,14.248,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1596.0,+184/-118,0.463,15.91,6.11,8.64,1398.9,0.645,✔️,0.0,False,GPU +11,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1592.0,+115/-74,0.431,16.09,11.87,8.696,207.65,0.056,✔️,0.0,False,CPU +12,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1586.0,+203/-117,0.471,16.39,6.11,8.38,783.29,1.924,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1555.0,+129/-89,0.34,17.91,9.57,8.734,1624.54,0.083,✔️,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1546.0,+168/-78,0.351,18.4,11.79,9.818,93.21,0.087,✔️,0.0,False,CPU +15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1545.0,+183/-146,0.414,18.43,6.86,9.353,1398.9,0.062,✔️,0.0,False,GPU +16,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1536.0,+148/-119,0.419,18.92,8.42,9.128,2.3,0.82,✔️,0.0,False,GPU +17,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1513.0,+160/-143,0.417,20.12,9.17,9.75,7.51,3.146,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1501.0,+112/-82,0.269,20.71,14.13,10.252,2338.19,0.54,✔️,0.0,False,GPU +19,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1479.0,+156/-138,0.329,21.91,8.17,9.97,4805.66,461.797,✔️,0.0,False,GPU +20,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1469.0,+87/-76,0.239,22.43,15.32,10.088,1169.3,1.442,✔️,0.0,False,CPU +21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1458.0,+143/-105,0.22,23.03,15.4,12.545,893.93,0.205,✔️,0.0,False,CPU +22,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1433.0,+148/-126,0.258,24.44,15.01,10.933,0.47,0.052,✔️,0.0,False,CPU +23,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1414.0,+136/-127,0.178,25.5,16.3,12.445,709.85,2.929,✔️,0.0,False,GPU +24,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1391.0,+158/-107,0.143,26.82,20.15,13.595,893.93,0.016,✔️,0.0,False,CPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1382.0,+132/-131,0.147,27.33,20.61,14.515,709.85,0.19,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1376.0,+148/-151,0.221,27.64,15.75,12.68,5.67,0.081,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1352.0,+85/-85,0.075,29.04,25.29,13.54,6.98,0.233,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1350.0,+97/-141,0.113,29.13,22.1,14.46,389.62,2.114,✔️,0.0,False,CPU +29,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1343.0,+134/-130,0.126,29.53,21.59,15.921,84.36,0.919,➖,0.0,False,GPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1340.0,+85/-85,0.078,29.68,23.94,12.269,1169.3,0.053,✔️,0.0,False,CPU +31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1336.0,+126/-104,0.088,29.92,25.61,15.629,2.09,0.019,✔️,0.0,False,CPU +32,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1323.0,+116/-98,0.061,30.66,22.47,13.699,52.45,0.219,✔️,0.0,False,CPU +33,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1322.0,+112/-119,0.08,30.69,19.96,13.703,4805.66,43.211,✔️,0.0,False,GPU +34,🧠🔁,Neural Network,[LimiX (default) [60.00% IMPUTED]](https://arxiv.org/abs/2509.03505),1304.0,+188/-188,0.211,31.71,14.31,18.746,0.44,0.016,➖,60.0,True,GPU +35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1289.0,+92/-110,0.062,32.48,29.28,13.714,0.28,0.033,✔️,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1286.0,+123/-114,0.063,32.62,24.29,14.614,52.45,0.021,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1279.0,+79/-73,0.011,32.99,30.01,14.632,10.5,0.606,✔️,0.0,False,GPU +38,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1262.0,+67/-88,0.011,33.9,31.0,16.64,14.85,13.554,➖,0.0,False,GPU +39,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1260.0,+73/-86,0.026,34.0,31.34,13.754,38.42,0.32,✔️,0.0,False,CPU +40,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1241.0,+122/-149,0.034,34.98,22.06,17.572,46.9,45.623,✔️,0.0,False,GPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1214.0,+81/-142,0.034,36.34,30.97,17.64,389.62,0.112,✔️,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1201.0,+77/-85,0.013,36.93,34.78,15.542,38.42,0.028,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1176.0,+63/-72,0.0,38.14,36.49,16.74,1.87,0.046,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1139.0,+109/-212,0.039,39.81,34.26,21.377,1.61,0.112,✔️,0.0,False,CPU +45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1097.0,+143/-229,0.007,41.59,37.35,26.806,43.46,0.087,✔️,0.0,False,CPU +46,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1065.0,+95/-197,0.009,42.84,33.75,24.396,331.68,44.947,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1046.0,+151/-234,0.004,43.55,40.11,28.029,43.46,0.021,✔️,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+99/-170,0.0,45.17,44.0,22.389,0.12,0.013,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),924.0,+135/-230,0.0,47.47,45.99,29.601,0.2,0.023,✔️,0.0,False,CPU +50,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),915.0,+200/-468,0.022,47.71,41.94,31.192,2.8,0.224,✔️,0.0,False,GPU +51,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),880.0,+89/-204,0.0,48.6,48.15,26.755,0.09,0.019,✔️,0.0,False,CPU +52,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),856.0,+108/-175,0.0,49.17,48.37,32.973,0.2,0.033,➖,0.0,False,GPU +53,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),814.0,+66/-149,0.0,50.06,49.68,32.101,331.68,1.626,✔️,0.0,False,CPU +54,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),444.0,+126/-380,0.0,54.34,54.31,49.305,0.12,0.101,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/winrate_matrix.png.zip index 2562a435e29faa379d30baa5908d592bc6822686..7b6511a3bd9f89704870a7f28947777c1118a834 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:966440a682d3728d229ce7afad6e2b477f1602d0c52d078d8abd3c63ba14b249 -size 2273190 +oid sha256:fca7527d95e885d7fa3d7ef3884104fb3a252e72fb881a3a00744c65f430152d +size 2472965 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 06d1138c7a0aad011f2d81542d300b1763490154..2cbcde2deb30bf47e94543aacce6b8aa245b44ca 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4d74b100ca6a413321c9dae6b8f21950982695910125c0986581e1a71bb4d896 -size 471502 +oid sha256:16cba423274831765d5a5977eb34602b45036df69ed389886b3b8a3dc0e0c9e9 +size 486581 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip index 7b5ef8762e095faabeada40af45c311c551fb4ab..66e6313fa364b7c147675f6244b199bd0a005ba4 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:aae23739517f2abc4c3e7df8ecf238bca412375257f9eadb70b119f3c6d56c03 -size 1027096 +oid sha256:b3a6e759e42b2af60a360f937563d1b0731b0b702e00380c495f0ecdebcefb62 +size 1003280 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_small/tuning-impact-elo.png.zip index 4f9f9f634a114934f1653999e3d2739dd786a2ef..05139da6959a8dd51565d43242ca8c3bb5fb2da3 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0b5b5befc5edc9bfc71c0ebc812c59ec31733b3b120334cb71792f85f85334e0 -size 190557 +oid sha256:72b6e17057a6835f7d1dcf65d7b98f89af29e3b5f22b501b591e36ed45f072bc +size 190634 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_classification/datasets_small/website_leaderboard.csv index d3a4f497c8b258dfe78a765eefae142f48f71581..92add9208a3ea5d24907aea614e10efe20042f6a 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_small/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_small/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1604.0,+78/-60,8.48,0.711,9.9,4.49,610.76,8.082,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1602.0,+88/-82,5.856,0.725,9.98,3.67,2289.05,8.114,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1583.0,+75/-67,6.98,0.697,10.77,4.36,2289.05,1.254,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1582.0,+78/-79,7.294,0.713,10.84,5.18,6.07,0.762,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[LimiX (default) [0.40% IMPUTED]](https://arxiv.org/abs/2509.03505),1489.0,+85/-70,10.809,0.559,15.35,5.56,4.62,0.633,➖,0.4,True,GPU -5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1488.0,+85/-75,11.831,0.54,15.43,6.02,3302.86,45.811,✔️,7.14,True,GPU -6,🧠⚡,Foundation Model,[Mitra (default) [7.14% IMPUTED]](https://arxiv.org/abs/2510.21204),1484.0,+92/-78,12.423,0.556,15.65,6.52,144.27,3.772,✔️,7.14,True,GPU -7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1457.0,+68/-52,12.392,0.433,17.16,9.79,3422.82,18.936,✔️,0.0,False,GPU -8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1431.0,+68/-56,13.705,0.375,18.68,10.21,3170.35,1.694,✔️,0.0,False,GPU -9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1426.0,+79/-52,13.359,0.359,19.01,10.04,3143.8,1.884,✔️,0.0,False,CPU -10,🧠⚡,Foundation Model,[TabICL (default) [7.14% IMPUTED]](https://arxiv.org/abs/2502.05564),1416.0,+82/-60,13.369,0.421,19.62,6.98,8.89,1.714,✔️,7.14,True,GPU -11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1409.0,+67/-62,11.946,0.399,20.04,7.07,5271.34,231.986,✔️,0.0,False,GPU -12,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1407.0,+106/-83,14.357,0.416,20.18,6.83,3302.86,0.929,✔️,7.14,True,GPU -13,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1389.0,+67/-50,15.512,0.368,21.32,10.91,330.95,2.359,➖,0.0,False,GPU -14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1376.0,+58/-60,14.668,0.291,22.09,11.93,3170.35,0.241,✔️,0.0,False,GPU -15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1375.0,+72/-61,12.943,0.352,22.17,7.4,5271.34,33.986,✔️,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1374.0,+46/-36,15.375,0.252,22.22,15.36,735.1,1.901,✔️,0.0,False,CPU -17,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1373.0,+58/-52,14.106,0.28,22.32,11.72,3422.82,1.534,✔️,0.0,False,GPU -18,🧠⚡,Foundation Model,[TabPFNv2 (default) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1372.0,+79/-73,15.443,0.396,22.38,9.29,4.15,0.42,✔️,7.14,True,GPU -19,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1349.0,+44/-29,15.756,0.196,23.88,17.3,735.1,0.311,✔️,0.0,False,CPU -20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1336.0,+41/-35,15.67,0.187,24.74,17.8,1714.62,0.768,✔️,0.0,False,CPU -21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1330.0,+51/-49,15.2,0.2,25.11,10.89,6047.72,0.525,✔️,0.0,False,GPU -22,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1325.0,+39/-32,15.86,0.178,25.44,18.66,1714.62,0.098,✔️,0.0,False,CPU -23,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1313.0,+55/-58,16.35,0.201,26.25,15.38,1053.84,3.063,✔️,0.0,False,GPU -24,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1306.0,+45/-47,16.611,0.163,26.73,18.68,797.9,2.228,✔️,0.0,False,CPU -25,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1302.0,+78/-77,16.103,0.234,27.03,12.02,6047.72,8.744,✔️,0.0,False,GPU -26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1301.0,+44/-38,16.742,0.164,27.05,18.55,7.19,0.118,✔️,0.0,False,CPU -27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1298.0,+59/-61,17.244,0.224,27.3,14.87,10.78,0.17,✔️,0.0,False,GPU -28,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1291.0,+42/-42,16.833,0.126,27.72,18.81,797.9,0.353,✔️,0.0,False,CPU -29,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1291.0,+77/-81,16.042,0.279,27.73,9.1,51.44,43.709,✔️,0.0,False,GPU -30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1275.0,+41/-37,17.729,0.125,28.82,19.23,3351.28,0.544,✔️,0.0,False,CPU -31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1268.0,+55/-60,17.648,0.141,29.25,15.68,1053.84,0.169,✔️,0.0,False,GPU -32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1267.0,+54/-50,17.082,0.126,29.33,20.53,2977.49,2.415,✔️,0.0,False,CPU -33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1248.0,+46/-42,18.335,0.092,30.63,18.92,3351.28,0.064,✔️,0.0,False,CPU -34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1246.0,+54/-57,19.416,0.13,30.78,16.14,15.54,0.346,✔️,0.0,False,GPU -35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1242.0,+70/-62,18.731,0.161,30.99,15.99,1423.89,9.208,✔️,0.0,False,CPU -36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1228.0,+57/-49,19.284,0.109,31.94,18.75,8.03,0.059,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1209.0,+46/-41,18.821,0.063,33.18,21.12,14.89,4.433,✔️,0.0,False,GPU -38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1200.0,+81/-70,20.935,0.105,33.77,19.75,399.55,1.466,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1194.0,+44/-56,19.321,0.072,34.22,22.35,2977.49,0.156,✔️,0.0,False,CPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1192.0,+58/-53,20.087,0.07,34.32,19.34,1423.89,0.749,✔️,0.0,False,CPU -41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1171.0,+41/-48,19.87,0.025,35.67,28.99,2.79,0.238,✔️,0.0,False,CPU -42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1163.0,+82/-76,21.958,0.078,36.19,20.05,399.55,0.174,✔️,0.0,False,CPU -43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1156.0,+50/-53,20.316,0.054,36.6,29.76,3.15,0.141,✔️,0.0,False,CPU -44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1150.0,+76/-89,22.17,0.091,36.98,19.99,571.65,1.39,✔️,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1110.0,+75/-74,23.019,0.051,39.41,18.55,571.65,0.129,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1059.0,+60/-66,24.096,0.013,42.3,30.74,11.29,0.204,✔️,0.0,False,CPU -47,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1044.0,+91/-103,27.29,0.082,43.07,25.61,2.59,0.533,➖,0.0,False,GPU -48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1025.0,+75/-110,26.058,0.022,44.09,36.08,4.95,1.012,✔️,0.0,False,GPU -49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1023.0,+78/-96,30.914,0.051,44.16,29.56,483.02,0.874,✔️,0.0,False,CPU -50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1009.0,+67/-72,25.623,0.0,44.85,28.65,5.04,0.646,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1008.0,+86/-111,28.815,0.027,44.91,28.44,121.17,1.63,✔️,0.0,False,CPU -52,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+67/-78,28.537,0.008,45.29,37.69,0.46,0.068,✔️,0.0,False,CPU -53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),998.0,+76/-98,31.504,0.033,45.36,31.3,483.02,0.113,✔️,0.0,False,CPU -54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),959.0,+78/-102,30.018,0.009,47.15,40.28,0.42,0.074,✔️,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),923.0,+98/-136,34.781,0.022,48.6,34.38,2.39,0.143,✔️,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),920.0,+105/-116,34.463,0.012,48.72,32.71,121.17,0.136,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),582.0,+114/-167,49.793,0.0,56.35,51.76,0.28,0.023,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1601.0,+78/-59,0.706,10.13,4.55,8.755,610.76,8.082,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1599.0,+89/-83,0.721,10.19,3.78,6.207,2289.05,8.114,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1580.0,+76/-68,0.693,10.98,4.44,7.33,2289.05,1.254,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1579.0,+77/-78,0.703,11.03,5.46,7.673,6.07,0.762,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[LimiX (default) [0.40% IMPUTED]](https://arxiv.org/abs/2509.03505),1487.0,+83/-68,0.547,15.6,5.71,11.136,4.62,0.633,➖,0.4,True,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1484.0,+85/-75,0.533,15.78,6.21,12.079,3302.86,45.811,✔️,7.14,True,GPU +6,🧠🔁,Neural Network,[Mitra (default) [7.14% IMPUTED]](https://arxiv.org/abs/2510.21204),1481.0,+90/-77,0.551,15.98,6.64,12.603,144.27,3.772,✔️,7.14,True,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1454.0,+67/-52,0.423,17.53,10.17,12.696,3422.82,18.936,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1428.0,+67/-55,0.368,19.08,10.37,13.91,3170.35,1.694,✔️,0.0,False,GPU +9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1422.0,+78/-53,0.359,19.45,10.39,13.529,3143.8,1.884,✔️,0.0,False,CPU +10,🧠⚡,Foundation Model,[TabICL (default) [7.14% IMPUTED]](https://arxiv.org/abs/2502.05564),1413.0,+83/-60,0.414,20.02,7.06,13.544,8.89,1.714,✔️,7.14,True,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1407.0,+67/-63,0.391,20.44,7.13,12.154,5271.34,231.986,✔️,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1405.0,+105/-82,0.408,20.57,6.99,14.588,3302.86,0.929,✔️,7.14,True,GPU +13,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1386.0,+67/-50,0.364,21.74,11.09,15.701,330.95,2.359,➖,0.0,False,GPU +14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1374.0,+57/-61,0.288,22.57,12.15,14.859,3170.35,0.241,✔️,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1373.0,+72/-60,0.352,22.62,7.45,13.121,5271.34,33.986,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1372.0,+46/-35,0.251,22.67,15.62,15.535,735.1,1.901,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1371.0,+58/-52,0.279,22.77,12.11,14.38,3422.82,1.534,✔️,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabPFNv2 (default) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1369.0,+79/-72,0.383,22.85,9.59,15.767,4.15,0.42,✔️,7.14,True,GPU +19,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1347.0,+44/-30,0.195,24.36,17.62,15.918,735.1,0.311,✔️,0.0,False,CPU +20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1334.0,+41/-35,0.185,25.19,18.22,15.93,1714.62,0.768,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1328.0,+51/-50,0.202,25.61,11.11,15.382,6047.72,0.525,✔️,0.0,False,GPU +22,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1324.0,+38/-31,0.176,25.93,19.06,16.119,1714.62,0.098,✔️,0.0,False,CPU +23,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1320.0,+72/-69,0.319,26.2,8.23,16.547,13.27,1.624,➖,0.0,False,GPU +24,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1311.0,+56/-59,0.196,26.81,15.77,16.59,1053.84,3.063,✔️,0.0,False,GPU +25,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1304.0,+44/-49,0.161,27.26,19.08,16.799,797.9,2.228,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1300.0,+78/-76,0.234,27.53,12.17,16.301,6047.72,8.744,✔️,0.0,False,GPU +27,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1300.0,+44/-38,0.162,27.54,18.92,16.887,7.19,0.118,✔️,0.0,False,CPU +28,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1296.0,+58/-61,0.221,27.85,15.73,17.397,10.78,0.17,✔️,0.0,False,GPU +29,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1290.0,+78/-81,0.279,28.26,9.18,16.152,51.44,43.709,✔️,0.0,False,GPU +30,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1290.0,+40/-42,0.123,28.27,19.15,17.027,797.9,0.353,✔️,0.0,False,CPU +31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1273.0,+41/-36,0.123,29.38,19.75,17.891,3351.28,0.544,✔️,0.0,False,CPU +32,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1267.0,+56/-60,0.139,29.83,15.93,17.874,1053.84,0.169,✔️,0.0,False,GPU +33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1265.0,+53/-51,0.127,29.96,20.88,17.22,2977.49,2.415,✔️,0.0,False,CPU +34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1246.0,+46/-42,0.09,31.22,19.25,18.5,3351.28,0.064,✔️,0.0,False,CPU +35,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1244.0,+53/-57,0.128,31.38,16.45,19.556,15.54,0.346,✔️,0.0,False,GPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1240.0,+70/-61,0.161,31.64,16.35,18.844,1423.89,9.208,✔️,0.0,False,CPU +37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1227.0,+56/-49,0.104,32.57,19.24,19.423,8.03,0.059,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1207.0,+44/-41,0.063,33.9,21.79,18.953,14.89,4.433,✔️,0.0,False,GPU +39,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1199.0,+80/-70,0.102,34.42,19.98,21.141,399.55,1.466,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1192.0,+44/-57,0.075,34.9,22.71,19.433,2977.49,0.156,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1190.0,+57/-52,0.07,35.03,19.9,20.194,1423.89,0.749,✔️,0.0,False,CPU +42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1170.0,+41/-47,0.025,36.31,29.42,20.014,2.79,0.238,✔️,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1162.0,+81/-74,0.076,36.87,20.33,22.149,399.55,0.174,✔️,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1155.0,+49/-53,0.052,37.26,30.2,20.432,3.15,0.141,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1149.0,+76/-88,0.089,37.67,20.44,22.415,571.65,1.39,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1109.0,+74/-74,0.048,40.13,19.97,23.263,571.65,0.129,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1058.0,+60/-66,0.014,43.08,31.28,24.229,11.29,0.204,✔️,0.0,False,CPU +48,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1044.0,+91/-101,0.082,43.87,26.17,27.374,2.59,0.533,➖,0.0,False,GPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1024.0,+75/-108,0.019,44.88,36.85,26.257,4.95,1.012,✔️,0.0,False,GPU +50,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1022.0,+76/-95,0.051,45.0,30.04,31.052,483.02,0.874,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1009.0,+86/-112,0.028,45.66,28.74,28.912,121.17,1.63,✔️,0.0,False,CPU +52,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1009.0,+66/-74,0.0,45.67,29.57,25.706,5.04,0.646,✔️,0.0,False,CPU +53,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+66/-75,0.006,46.1,38.9,28.664,0.46,0.068,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),998.0,+76/-98,0.034,46.2,31.86,31.614,483.02,0.113,✔️,0.0,False,CPU +55,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),960.0,+76/-103,0.009,47.95,41.01,30.118,0.42,0.074,✔️,0.0,False,CPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),923.0,+96/-134,0.021,49.45,34.85,34.861,2.39,0.143,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),921.0,+103/-114,0.014,49.54,33.12,34.544,121.17,0.136,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),582.0,+115/-170,0.0,57.33,52.6,49.849,0.28,0.023,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_small/winrate_matrix.png.zip index 8fa40c958c736d9746d8bab938eed53d05843a32..8637dae7f6644c64e0092efcfefe1c43a5b74eb2 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c6a7cc9e39c35919c1c2185fb2db054f6f1882a257f6ac4e2301f96bafaa7c11 -size 2759671 +oid sha256:7c2bee50e67fec85a2468a5137363a87e89d871a08b823a76249406eec3e3a9f +size 2751943 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index b9279ecfb3085e261d89d00749f14ed2f29c0fa1..8c56951c703847f31b248ea429457b528ad2d3d6 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:856bc4aeeccd3e684e795eff61eda1466cbd64477131ba8be24f024afddb5ec7 -size 471650 +oid sha256:7ccb49b7d5cabfba7369edc4126e8d3f3a44b625984da8b9e1e5313bdb2880a9 +size 485745 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip index bad1869aae7da54d4b36405eb45cf5d9111f49a3..1ba96aeaee4b954ee7afbe21f4444674028d8768 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:605635e0ca32888ba4fade93b2d53a3e50a4081fa32f16d340a54937efec630c -size 1051061 +oid sha256:60ae278d4e63ee43297ac15c87407e5aaba3709e7bb1aa36d6bbffc23b519a8b +size 1031647 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip index b4e32d90960bb8de9907240843440b65ce427ea0..501dbee39c29832d0c424ccf298230c98c82edbe 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8b5c0420a632ab0b5df292d9576706ec365a5dc6fbb02c7621f8c6cee483107b -size 120230 +oid sha256:527fd0e1f3d7ab8be1a6b0ea290217c94cc5a7d930b16adda100e0b55f543b55 +size 121995 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/website_leaderboard.csv index 9eb8e140ebb7cc10688c97ebfcf83baa97abecd1..1f9cf6227b8613a285e17df780d5726301fd6b67 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1634.0,+83/-79,6.234,0.728,9.78,3.8,2168.17,7.961,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1631.0,+70/-57,7.553,0.747,9.87,4.93,5.99,0.633,✔️,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1625.0,+85/-64,9.074,0.701,10.13,4.67,604.44,8.082,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1614.0,+81/-69,7.377,0.702,10.59,4.34,2168.17,1.23,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1544.0,+69/-56,11.21,0.576,13.78,5.67,3445.6,48.236,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1536.0,+80/-59,11.857,0.593,14.17,6.16,146.65,4.101,✔️,0.0,False,GPU -6,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1518.0,+87/-73,11.42,0.58,15.09,5.58,4.37,0.585,➖,0.0,False,GPU -7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1486.0,+61/-44,13.101,0.447,16.86,9.64,3169.9,16.793,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1459.0,+71/-61,12.877,0.447,18.45,6.6,8.89,1.743,✔️,0.0,False,GPU -9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1454.0,+71/-46,14.494,0.373,18.73,10.01,2828.45,1.596,✔️,0.0,False,GPU -10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1452.0,+82/-72,13.931,0.442,18.89,6.45,3445.6,0.995,✔️,0.0,False,GPU -11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1444.0,+66/-55,14.093,0.356,19.32,10.43,2898.23,1.884,✔️,0.0,False,CPU -12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1425.0,+77/-70,12.671,0.388,20.47,6.97,5119.36,231.986,✔️,0.0,False,GPU -13,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1424.0,+60/-53,15.951,0.369,20.56,10.58,330.95,2.023,➖,0.0,False,GPU -14,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1410.0,+84/-72,15.11,0.421,21.42,8.85,4.06,0.436,✔️,0.0,False,GPU -15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1400.0,+56/-54,14.802,0.297,22.04,11.42,3169.9,1.446,✔️,0.0,False,GPU -16,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1400.0,+82/-72,13.572,0.355,22.07,7.06,5119.36,26.967,✔️,0.0,False,GPU -17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1399.0,+72/-51,15.476,0.29,22.12,11.71,2828.45,0.216,✔️,0.0,False,GPU -18,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1388.0,+42/-35,16.363,0.23,22.83,15.77,647.56,1.72,✔️,0.0,False,CPU -19,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1362.0,+37/-30,16.741,0.175,24.46,17.6,647.56,0.284,✔️,0.0,False,CPU -20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1356.0,+50/-36,16.602,0.186,24.9,17.76,1465.86,0.692,✔️,0.0,False,CPU -21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1355.0,+56/-50,15.889,0.195,24.97,11.03,5944.88,0.516,✔️,0.0,False,GPU -22,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1342.0,+48/-35,16.823,0.173,25.77,18.98,1465.86,0.091,✔️,0.0,False,CPU -23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1326.0,+64/-56,18.007,0.23,26.84,14.3,10.42,0.155,✔️,0.0,False,GPU -24,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1325.0,+46/-41,17.617,0.157,26.9,19.33,766.06,1.917,✔️,0.0,False,CPU -25,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1323.0,+95/-83,17.015,0.242,27.06,11.78,5944.88,8.396,✔️,0.0,False,GPU -26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1320.0,+45/-44,17.726,0.159,27.23,18.32,5.72,0.11,✔️,0.0,False,CPU -27,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1320.0,+87/-95,16.755,0.3,27.25,8.62,50.32,43.824,✔️,0.0,False,GPU -28,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1320.0,+50/-53,17.396,0.172,27.25,16.63,934.1,2.767,✔️,0.0,False,GPU -29,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1309.0,+39/-36,17.841,0.119,27.97,19.24,766.06,0.278,✔️,0.0,False,CPU -30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1297.0,+46/-41,18.633,0.117,28.77,18.8,2686.11,0.47,✔️,0.0,False,CPU -31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1294.0,+56/-54,17.813,0.135,28.96,20.03,2862.05,2.158,✔️,0.0,False,CPU -32,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1283.0,+45/-58,19.49,0.14,29.68,15.39,14.8,0.337,✔️,0.0,False,GPU -33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1277.0,+50/-50,18.738,0.12,30.07,16.88,934.1,0.165,✔️,0.0,False,GPU -34,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1275.0,+77/-87,19.42,0.174,30.22,15.28,1358.63,8.067,✔️,0.0,False,CPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1269.0,+45/-48,19.24,0.083,30.63,18.46,2686.11,0.056,✔️,0.0,False,CPU -36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1244.0,+50/-47,20.124,0.089,32.25,19.07,7.4,0.057,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1243.0,+44/-39,19.291,0.068,32.34,20.31,13.83,3.953,✔️,0.0,False,GPU -38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1219.0,+64/-78,20.796,0.076,33.88,19.08,1358.63,0.897,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1213.0,+53/-43,20.109,0.061,34.25,22.19,2862.05,0.152,✔️,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1199.0,+60/-67,22.25,0.076,35.15,24.0,370.85,1.466,✔️,0.0,False,CPU -41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1184.0,+45/-43,21.0,0.019,36.07,30.03,2.4,0.218,✔️,0.0,False,CPU -42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1160.0,+62/-70,23.266,0.046,37.55,25.98,370.85,0.161,✔️,0.0,False,CPU -43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1153.0,+36/-44,21.599,0.027,37.95,32.6,2.9,0.133,✔️,0.0,False,CPU -44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1133.0,+70/-69,23.712,0.043,39.17,25.14,527.42,1.39,✔️,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1093.0,+61/-65,24.551,0.011,41.45,24.97,527.42,0.123,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1081.0,+56/-63,24.892,0.014,42.1,30.05,10.38,0.188,✔️,0.0,False,CPU -47,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1071.0,+83/-130,27.865,0.088,42.63,24.72,2.43,0.495,➖,0.0,False,GPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1058.0,+100/-110,30.113,0.054,43.29,28.54,436.81,0.874,✔️,0.0,False,CPU -49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1034.0,+89/-103,27.438,0.024,44.5,35.91,4.4,0.974,✔️,0.0,False,GPU -50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1033.0,+101/-105,30.656,0.036,44.53,30.27,436.81,0.106,✔️,0.0,False,CPU -51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1032.0,+72/-81,26.107,0.0,44.59,27.79,4.73,0.623,✔️,0.0,False,CPU -52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1008.0,+77/-109,30.107,0.015,45.72,32.67,113.26,1.55,✔️,0.0,False,CPU -53,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+61/-83,29.212,0.003,46.1,38.93,0.45,0.066,✔️,0.0,False,CPU -54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),969.0,+75/-112,30.601,0.009,47.45,40.18,0.4,0.07,✔️,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),958.0,+91/-127,33.436,0.023,47.9,33.34,2.16,0.139,✔️,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),909.0,+87/-143,36.035,0.001,49.71,34.2,113.26,0.12,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),600.0,+121/-261,49.89,0.0,56.33,51.43,0.26,0.023,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1630.0,+84/-80,0.722,10.0,3.9,6.61,2168.17,7.961,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1628.0,+70/-56,0.738,10.07,5.21,7.958,5.99,0.633,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1620.0,+83/-64,0.696,10.38,4.74,9.368,604.44,8.082,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1610.0,+80/-71,0.697,10.81,4.41,7.752,2168.17,1.23,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1540.0,+70/-55,0.57,14.09,5.86,11.475,3445.6,48.236,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),1533.0,+79/-59,0.59,14.47,6.28,12.049,146.65,4.101,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1516.0,+88/-72,0.568,15.35,5.73,11.77,4.37,0.585,➖,0.0,False,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1482.0,+62/-44,0.437,17.25,10.01,13.427,3169.9,16.793,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1456.0,+72/-61,0.442,18.82,6.68,13.063,8.89,1.743,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1450.0,+70/-46,0.366,19.14,10.17,14.712,2828.45,1.596,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1449.0,+81/-72,0.436,19.25,6.6,14.177,3445.6,0.995,✔️,0.0,False,GPU +11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1440.0,+63/-54,0.357,19.77,10.75,14.274,2898.23,1.884,✔️,0.0,False,CPU +12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1422.0,+77/-69,0.38,20.9,7.02,12.892,5119.36,231.986,✔️,0.0,False,GPU +13,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1421.0,+60/-51,0.365,20.98,10.75,16.153,330.95,2.023,➖,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1407.0,+83/-71,0.408,21.87,9.13,15.457,4.06,0.436,✔️,0.0,False,GPU +15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1397.0,+55/-54,0.293,22.5,11.81,15.095,3169.9,1.446,✔️,0.0,False,GPU +16,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1397.0,+83/-72,0.355,22.53,7.11,13.762,5119.36,26.967,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1396.0,+71/-50,0.286,22.61,11.93,15.679,2828.45,0.216,✔️,0.0,False,GPU +18,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1385.0,+41/-34,0.23,23.31,16.06,16.533,647.56,1.72,✔️,0.0,False,CPU +19,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1360.0,+36/-29,0.175,24.97,17.95,16.914,647.56,0.284,✔️,0.0,False,CPU +20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1354.0,+49/-35,0.183,25.37,18.19,16.88,1465.86,0.692,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1352.0,+55/-50,0.195,25.47,11.28,16.083,5944.88,0.516,✔️,0.0,False,GPU +22,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1349.0,+77/-82,0.339,25.69,8.08,17.238,12.28,1.411,➖,0.0,False,GPU +23,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1340.0,+46/-34,0.17,26.28,19.43,17.1,1465.86,0.091,✔️,0.0,False,CPU +24,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1323.0,+64/-56,0.225,27.4,15.15,18.17,10.42,0.155,✔️,0.0,False,GPU +25,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1322.0,+44/-41,0.156,27.46,19.79,17.817,766.06,1.917,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1321.0,+95/-83,0.243,27.58,11.91,17.226,5944.88,8.396,✔️,0.0,False,GPU +27,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1318.0,+47/-42,0.157,27.75,18.7,17.881,5.72,0.11,✔️,0.0,False,CPU +28,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1318.0,+87/-95,0.301,27.79,8.7,16.871,50.32,43.824,✔️,0.0,False,GPU +29,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1317.0,+50/-52,0.169,27.84,17.12,17.652,934.1,2.767,✔️,0.0,False,GPU +30,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1306.0,+39/-35,0.117,28.55,19.61,18.047,766.06,0.278,✔️,0.0,False,CPU +31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1295.0,+46/-40,0.114,29.34,19.33,18.806,2686.11,0.47,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1291.0,+57/-54,0.137,29.6,20.38,17.96,2862.05,2.158,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1281.0,+46/-57,0.138,30.25,15.68,19.639,14.8,0.337,✔️,0.0,False,GPU +34,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1275.0,+50/-49,0.119,30.68,17.18,18.979,934.1,0.165,✔️,0.0,False,GPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1272.0,+77/-86,0.173,30.88,15.62,19.54,1358.63,8.067,✔️,0.0,False,CPU +36,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1267.0,+44/-46,0.08,31.22,18.79,19.416,2686.11,0.056,✔️,0.0,False,CPU +37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1242.0,+49/-45,0.083,32.89,19.61,20.271,7.4,0.057,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1240.0,+44/-39,0.068,33.04,20.96,19.431,13.83,3.953,✔️,0.0,False,GPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1216.0,+64/-77,0.075,34.6,19.66,20.91,1358.63,0.897,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1211.0,+52/-43,0.063,34.95,22.57,20.228,2862.05,0.152,✔️,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1197.0,+60/-66,0.073,35.83,24.35,22.47,370.85,1.466,✔️,0.0,False,CPU +42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1183.0,+45/-43,0.022,36.73,30.5,21.153,2.4,0.218,✔️,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1159.0,+62/-69,0.043,38.24,26.46,23.47,370.85,0.161,✔️,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1152.0,+36/-44,0.027,38.64,33.15,21.722,2.9,0.133,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1132.0,+70/-68,0.041,39.91,25.86,23.974,527.42,1.39,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1092.0,+60/-65,0.01,42.2,27.78,24.812,527.42,0.123,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1079.0,+55/-62,0.016,42.9,30.59,25.033,10.38,0.188,✔️,0.0,False,CPU +48,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1070.0,+82/-130,0.089,43.44,25.27,27.954,2.43,0.495,➖,0.0,False,GPU +49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1057.0,+99/-105,0.054,44.12,28.99,30.261,436.81,0.874,✔️,0.0,False,CPU +50,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1033.0,+90/-101,0.02,45.32,36.7,27.651,4.4,0.974,✔️,0.0,False,GPU +51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1032.0,+100/-102,0.037,45.36,30.81,30.773,436.81,0.106,✔️,0.0,False,CPU +52,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1032.0,+72/-82,0.0,45.4,28.7,26.194,4.73,0.623,✔️,0.0,False,CPU +53,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1009.0,+76/-105,0.016,46.49,33.08,30.21,113.26,1.55,✔️,0.0,False,CPU +54,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+61/-84,0.003,46.9,40.2,29.346,0.45,0.066,✔️,0.0,False,CPU +55,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),970.0,+75/-110,0.009,48.23,40.88,30.707,0.4,0.07,✔️,0.0,False,CPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),958.0,+90/-124,0.023,48.74,33.8,33.52,2.16,0.139,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),910.0,+86/-140,0.001,50.55,34.68,36.12,113.26,0.12,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),599.0,+121/-261,0.0,57.31,52.26,49.949,0.26,0.023,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip index c7309b3a8849e785b5086e846aa08d1860522c78..020e51501fb8dbd546bc6b901b175fbd6aa5881f 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6540112dedaffdf05bc9503bfd629e0202ba29e3036de071a9d79292807a443b -size 2801926 +oid sha256:c249ceda419c8849900acd89085b5e3599b79af961e966d61ae40778feb7e149 +size 2820614 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index d4802503480f1965655786e083e3dac556db20c5..9af870f20303641b65f374141c62c021ebd0360c 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5f1b9f78b7791b06b684b5096d1fc9b28f6d7aa11c1fce9c77c7fa92c980273e -size 472876 +oid sha256:07e2c757d6b80b11fa30a61ca16720adcd3671fe1ffbed96fe9062a21a6c3981 +size 484472 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip index 6acb5521d7adef875b5a8fdb03c881dfab42dfba..b30fbc0f74c79c3bca443a15cdbf1913aabb8da6 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e5cbdc87d5255748326ffc8ae81b7d27245925787568802cd830dd1bb1dc34f4 -size 994907 +oid sha256:9ac1a2efff16ec132eae76c5538fa4ecd66ecd1b2f598f8fc53c6e7e7f17f93c +size 982384 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip index c084561a4283df1a086828f10b11249b881a2130..16242553b23c40f4cee6a96127dc18da593d5fcb 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bd738ebbfd20708f2c4cacf7dd853255d12d3b3a88bca7b82b9c598afb65cf58 -size 172927 +oid sha256:5a28562cd03cbe7650c1c214cac8b16d9bd9aa8fd5cd3ad4fbc83eea8bd08673 +size 172923 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/website_leaderboard.csv index b8bb6c4898eae2ea332fd830c592c63af9fc9814..262a4ebfd810b8339feca5c714a12a218ec891cb 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1724.0,+151/-70,9.367,0.833,7.12,4.6,1045.67,7.086,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1710.0,+222/-98,6.694,0.825,7.55,3.44,2710.27,1.347,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1706.0,+284/-137,5.887,0.825,7.67,2.39,2710.27,8.099,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1578.0,+229/-151,8.831,0.66,12.67,4.96,10.23,0.844,✔️,0.0,False,GPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1504.0,+192/-99,14.488,0.428,16.41,6.12,6219.24,3.78,✔️,0.0,False,CPU -5,🧠⚡,Foundation Model,[LimiX (default) [12.50% IMPUTED]](https://arxiv.org/abs/2509.03505),1497.0,+174/-104,12.101,0.471,16.84,6.1,5.85,0.91,➖,12.5,True,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1493.0,+163/-107,12.999,0.49,17.05,7.75,4150.25,19.131,✔️,0.0,False,GPU -7,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1482.0,+262/-213,16.891,0.593,17.66,5.63,2589.89,10.803,✔️,25.0,True,GPU -8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1467.0,+128/-74,14.123,0.432,18.53,10.07,4150.25,3.423,✔️,0.0,False,GPU -9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1452.0,+121/-82,16.878,0.395,19.44,11.32,5885.87,1.632,✔️,0.0,False,GPU -10,🧠⚡,Foundation Model,[Mitra (default) [25.00% IMPUTED]](https://arxiv.org/abs/2510.21204),1446.0,+200/-163,18.817,0.467,19.8,9.18,161.61,3.705,✔️,25.0,True,GPU -11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1437.0,+120/-84,16.499,0.318,20.35,14.42,1377.29,5.202,✔️,0.0,False,CPU -12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1434.0,+96/-58,14.953,0.303,20.56,17.07,3104.39,1.095,✔️,0.0,False,CPU -13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1428.0,+103/-77,16.719,0.291,20.92,15.57,1377.29,1.259,✔️,0.0,False,CPU -14,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1417.0,+164/-123,18.584,0.387,21.57,9.17,4635.14,212.285,✔️,0.0,False,GPU -15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1415.0,+113/-80,17.724,0.338,21.74,11.29,5885.87,0.299,✔️,0.0,False,GPU -16,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1409.0,+95/-59,15.572,0.248,22.06,17.73,3104.39,0.212,✔️,0.0,False,CPU -17,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1409.0,+198/-168,18.891,0.421,22.11,8.31,2589.89,0.54,✔️,25.0,True,GPU -18,🧠⚡,Foundation Model,[TabPFNv2 (default) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1398.0,+188/-143,17.461,0.357,22.75,11.93,6.47,0.363,✔️,25.0,True,GPU -19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1389.0,+92/-81,16.984,0.244,23.32,18.49,2210.02,0.974,✔️,0.0,False,CPU -20,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1386.0,+106/-80,16.17,0.209,23.52,15.97,7127.77,9.214,✔️,0.0,False,GPU -21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1383.0,+100/-84,17.116,0.235,23.69,18.39,2210.02,3.985,✔️,0.0,False,CPU -22,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1376.0,+72/-52,16.529,0.187,24.16,12.11,7127.77,0.776,✔️,0.0,False,GPU -23,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1364.0,+176/-118,20.408,0.33,24.87,10.15,4635.14,34.77,✔️,0.0,False,GPU -24,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1348.0,+147/-123,19.531,0.258,25.92,16.87,1980.6,3.605,✔️,0.0,False,GPU -25,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1346.0,+132/-110,20.079,0.27,26.08,18.97,507.73,4.428,➖,0.0,False,GPU -26,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1341.0,+109/-97,18.908,0.191,26.36,18.26,36484.23,0.827,✔️,0.0,False,CPU -27,🧠⚡,Foundation Model,[TabICL (default) [12.50% IMPUTED]](https://arxiv.org/abs/2502.05564),1319.0,+169/-167,22.325,0.269,27.84,12.57,10.98,1.898,✔️,12.5,True,GPU -28,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1317.0,+124/-118,20.804,0.238,27.97,17.42,17.83,0.197,✔️,0.0,False,GPU -29,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1309.0,+180/-174,20.881,0.287,28.5,11.75,737.91,1.864,✔️,0.0,False,CPU -30,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1307.0,+169/-141,21.669,0.238,28.59,14.2,717.36,1.612,✔️,0.0,False,CPU -31,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1304.0,+86/-71,19.049,0.147,28.79,24.75,50.54,0.227,✔️,0.0,False,CPU -32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1300.0,+136/-109,20.573,0.113,29.04,22.15,3901.28,2.028,✔️,0.0,False,CPU -33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1299.0,+107/-113,19.677,0.147,29.14,20.44,36484.23,0.127,✔️,0.0,False,CPU -34,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1267.0,+126/-136,21.594,0.148,31.18,23.71,19.31,0.108,✔️,0.0,False,CPU -35,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1262.0,+163/-157,23.154,0.196,31.48,13.13,717.36,0.326,✔️,0.0,False,CPU -36,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1259.0,+202/-138,23.472,0.219,31.68,9.38,43.39,42.575,✔️,0.0,False,GPU -37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1259.0,+199/-203,22.263,0.226,31.69,10.23,737.91,0.316,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1257.0,+176/-164,24.229,0.207,31.79,13.98,2644.99,12.294,✔️,0.0,False,CPU -39,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1253.0,+134/-120,21.886,0.161,32.06,18.15,1980.6,0.61,✔️,0.0,False,GPU -40,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1236.0,+119/-144,22.898,0.119,33.09,23.26,3901.28,0.223,✔️,0.0,False,CPU -41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1233.0,+118/-80,20.542,0.067,33.31,28.8,4.72,0.542,✔️,0.0,False,CPU -42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1212.0,+147/-122,22.667,0.134,34.53,26.5,5.04,0.429,✔️,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1206.0,+131/-146,25.342,0.1,34.95,19.21,2644.99,1.148,✔️,0.0,False,CPU -44,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1194.0,+79/-84,25.353,0.031,35.67,29.76,17.94,0.691,✔️,0.0,False,GPU -45,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1145.0,+58/-74,25.964,0.0,38.53,35.95,19.2,7.624,✔️,0.0,False,GPU -46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1081.0,+118/-200,29.352,0.05,41.88,36.27,9.42,1.675,✔️,0.0,False,GPU -47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1048.0,+72/-119,28.741,0.0,43.44,34.63,13.69,0.314,✔️,0.0,False,CPU -48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1001.0,+214/-366,39.389,0.043,45.5,24.18,151.69,2.346,✔️,0.0,False,CPU -49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+107/-182,35.605,0.0,45.54,42.17,0.74,0.137,✔️,0.0,False,CPU -50,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),979.0,+94/-231,40.046,0.0,46.4,44.39,892.95,0.818,✔️,0.0,False,CPU -51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),950.0,+122/-199,34.685,0.0,47.5,45.33,8.51,0.734,✔️,0.0,False,CPU -52,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),943.0,+84/-237,41.043,0.0,47.74,45.41,892.95,0.146,✔️,0.0,False,CPU -53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),899.0,+132/-177,40.735,0.0,49.25,46.82,0.69,0.137,✔️,0.0,False,CPU -54,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),850.0,+110/-155,39.73,0.0,50.73,50.4,5.05,1.013,➖,0.0,False,GPU -55,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),833.0,+268/-604,49.678,0.04,51.18,39.36,151.69,0.258,✔️,0.0,False,CPU -56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),731.0,+89/-254,47.224,0.0,53.53,53.21,4.53,0.164,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),188.0,+131/-663,70.56,0.0,57.77,57.76,0.52,0.076,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1713.0,+144/-65,0.825,7.37,4.75,10.208,1045.67,7.086,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1701.0,+204/-92,0.817,7.74,3.55,7.8,2710.27,1.347,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1697.0,+258/-132,0.818,7.87,2.49,6.994,2710.27,8.099,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1572.0,+216/-146,0.647,12.9,5.54,10.029,10.23,0.844,✔️,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1497.0,+188/-95,0.427,16.82,6.29,14.967,6219.24,3.78,✔️,0.0,False,CPU +5,🧠🔁,Neural Network,[LimiX (default) [12.50% IMPUTED]](https://arxiv.org/abs/2509.03505),1489.0,+163/-102,0.458,17.23,6.37,13.127,5.85,0.91,➖,12.5,True,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1485.0,+157/-105,0.479,17.47,8.36,13.946,4150.25,19.131,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1474.0,+256/-212,0.584,18.12,6.06,17.636,2589.89,10.803,✔️,25.0,True,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1459.0,+128/-75,0.429,19.01,10.85,14.966,4150.25,3.423,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1446.0,+118/-79,0.394,19.82,11.61,17.475,5885.87,1.632,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[Mitra (default) [25.00% IMPUTED]](https://arxiv.org/abs/2510.21204),1437.0,+195/-161,0.46,20.38,9.42,19.327,161.61,3.705,✔️,25.0,True,GPU +11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1432.0,+119/-83,0.321,20.73,14.66,16.943,1377.29,5.202,✔️,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1427.0,+87/-56,0.302,21.0,17.63,15.747,3104.39,1.095,✔️,0.0,False,CPU +13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1422.0,+102/-76,0.293,21.32,15.89,17.173,1377.29,1.259,✔️,0.0,False,CPU +14,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1410.0,+166/-123,0.383,22.06,9.3,19.189,4635.14,212.285,✔️,0.0,False,GPU +15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1409.0,+110/-80,0.338,22.17,11.52,18.27,5885.87,0.299,✔️,0.0,False,GPU +16,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1403.0,+92/-58,0.246,22.53,18.35,16.361,3104.39,0.212,✔️,0.0,False,CPU +17,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1401.0,+194/-165,0.413,22.66,8.6,19.584,2589.89,0.54,✔️,25.0,True,GPU +18,🧠⚡,Foundation Model,[TabPFNv2 (default) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1391.0,+172/-142,0.344,23.32,12.8,18.475,6.47,0.363,✔️,25.0,True,GPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1384.0,+90/-79,0.24,23.73,18.91,17.544,2210.02,0.974,✔️,0.0,False,CPU +20,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1381.0,+103/-76,0.208,23.96,16.37,16.752,7127.77,9.214,✔️,0.0,False,GPU +21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1378.0,+99/-79,0.231,24.1,18.79,17.654,2210.02,3.985,✔️,0.0,False,CPU +22,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1371.0,+70/-55,0.191,24.6,12.25,17.047,7127.77,0.776,✔️,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1358.0,+178/-117,0.328,25.4,10.25,20.911,4635.14,34.77,✔️,0.0,False,GPU +24,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1342.0,+145/-123,0.25,26.5,17.23,20.252,1980.6,3.605,✔️,0.0,False,GPU +25,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1340.0,+180/-227,0.396,26.64,6.01,17.775,22.52,1.682,➖,0.0,False,GPU +26,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1339.0,+127/-112,0.268,26.68,19.5,20.624,507.73,4.428,➖,0.0,False,GPU +27,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1336.0,+106/-96,0.195,26.86,18.65,19.36,36484.23,0.827,✔️,0.0,False,CPU +28,🧠⚡,Foundation Model,[TabICL (default) [12.50% IMPUTED]](https://arxiv.org/abs/2502.05564),1312.0,+170/-169,0.265,28.46,12.67,22.813,10.98,1.898,✔️,12.5,True,GPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1311.0,+122/-118,0.242,28.58,17.96,21.224,17.83,0.197,✔️,0.0,False,GPU +30,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1305.0,+178/-173,0.281,29.0,12.01,21.62,737.91,1.864,✔️,0.0,False,CPU +31,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1303.0,+168/-138,0.235,29.08,14.31,22.272,717.36,1.612,✔️,0.0,False,CPU +32,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1300.0,+84/-72,0.153,29.31,25.47,19.441,50.54,0.227,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1295.0,+132/-112,0.113,29.63,22.53,20.944,3901.28,2.028,✔️,0.0,False,CPU +34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1295.0,+107/-110,0.151,29.65,20.73,20.139,36484.23,0.127,✔️,0.0,False,CPU +35,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1263.0,+126/-132,0.151,31.73,23.98,21.963,19.31,0.108,✔️,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1259.0,+160/-155,0.194,31.98,13.22,23.704,717.36,0.326,✔️,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1256.0,+198/-203,0.223,32.2,10.58,22.999,737.91,0.316,✔️,0.0,False,CPU +38,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1255.0,+204/-140,0.219,32.24,9.42,23.735,43.39,42.575,✔️,0.0,False,GPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1252.0,+169/-163,0.206,32.42,14.26,24.508,2644.99,12.294,✔️,0.0,False,CPU +40,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1248.0,+136/-119,0.156,32.71,18.48,22.566,1980.6,0.61,✔️,0.0,False,GPU +41,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1233.0,+116/-144,0.126,33.68,23.52,23.181,3901.28,0.223,✔️,0.0,False,CPU +42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1231.0,+115/-79,0.068,33.8,29.15,20.935,4.72,0.542,✔️,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1209.0,+147/-120,0.136,35.17,26.88,22.964,5.04,0.429,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1201.0,+124/-146,0.098,35.68,19.52,25.599,2644.99,1.148,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1191.0,+81/-86,0.032,36.28,30.16,25.723,17.94,0.691,✔️,0.0,False,GPU +46,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1140.0,+58/-74,0.0,39.33,36.8,26.313,19.2,7.624,✔️,0.0,False,GPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1080.0,+112/-189,0.044,42.53,37.2,29.939,9.42,1.675,✔️,0.0,False,GPU +48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1047.0,+74/-124,0.0,44.18,35.35,29.095,13.69,0.314,✔️,0.0,False,CPU +49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1003.0,+211/-360,0.049,46.14,24.32,39.617,151.69,2.346,✔️,0.0,False,CPU +50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+106/-177,0.0,46.29,42.86,35.933,0.74,0.137,✔️,0.0,False,CPU +51,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),976.0,+90/-212,0.0,47.28,45.35,40.422,892.95,0.818,✔️,0.0,False,CPU +52,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),948.0,+119/-183,0.0,48.36,46.26,34.868,8.51,0.734,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),941.0,+81/-222,0.0,48.62,46.4,41.322,892.95,0.146,✔️,0.0,False,CPU +54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),901.0,+131/-175,0.0,50.03,47.48,40.975,0.69,0.137,✔️,0.0,False,CPU +55,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),854.0,+110/-162,0.0,51.5,51.12,39.903,5.05,1.013,➖,0.0,False,GPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),838.0,+263/-528,0.046,51.95,39.75,49.851,151.69,0.258,✔️,0.0,False,CPU +57,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),734.0,+80/-220,0.0,54.41,54.11,47.401,4.53,0.164,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),188.0,+127/-646,0.0,58.77,58.76,70.666,0.52,0.076,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/winrate_matrix.png.zip index 28d0e6d9e2aa5417ce16ccd9a6eaadef594129f6..fe5bc3b82ea533937a3a79eed436679e9b734907 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c481a2e8dbf92568e12b01670f04f10913e3e14d39b84dd50b7d19d512c3fa60 -size 2634680 +oid sha256:ac06b80957e0ba94905cd509b3ff5e45ace6847db810ed6664cf6d3148549881 +size 2613129 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index 7275e144f4c0fdaf48b873a4340a9a6b478cb2db..d1a2de2799ca95e1bce95af716e121eb87d06252 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7c7d7746e8be4d7d51f869a2d131235909cd5216e31cba92ee2fad01460c798b -size 444887 +oid sha256:bb2c6e1e5a3258cbc63896dbdeaf08bd47c4aa2dd1fb228e79b11f5bb3ac6e9d +size 459298 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip index 7edce77b3e40d6cb88d090d7dd9879ddfb47aaed..b851f283cf60bd74f7a99d59900aa2edc887218a 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4e0dd60a73c61b50382fc50fba7a66b269ce0d2ea60baae93afdbdf5d70c14d5 -size 951729 +oid sha256:7ba577b28dad032b0dcac670851398c2703995bb48cb1e73ceed67147215febf +size 981338 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip index 5c753136d7f344589c615e02a51e65d7e91c6d85..d5ac7839b4068e01e046d7cf6da478c7e21d1add 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:627adb339e35ae5905ec75da15ae52e93ff026a215063a1f917a984f7297d8c6 -size 106892 +oid sha256:b22fb3b11d7b1995eea06276ec5a73f49daa93fae6eb083f2860d109e2f9b437 +size 111706 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/website_leaderboard.csv index a419ca2395ccc34bf4d1abe9827fdbf5cbfafd8c..b7eaf658849154ebe3eb9e60d6f5b2f7a7c7ffa4 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/website_leaderboard.csv @@ -1,54 +1,55 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),9270.0,+0/-0,0.019,1.0,1.11,1.06,133.98,8.253,✔️,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",8803.0,+0/-0,1.7,0.934,2.56,2.2,276.87,4.662,✔️,0.0,False,CPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",8700.0,+0/-0,1.878,0.927,3.0,2.84,276.86,3.44,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),8621.0,+0/-0,2.424,0.905,3.33,3.0,133.98,2.816,✔️,0.0,False,GPU -4,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),6195.0,+0/-0,9.581,0.594,6.78,6.31,61.95,0.17,✔️,0.0,False,CPU -5,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),6182.0,+0/-0,9.545,0.595,6.89,6.45,7606.4,0.826,✔️,0.0,False,CPU -6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),6134.0,+0/-0,10.011,0.573,7.33,7.13,157.72,3.649,✔️,0.0,False,CPU -7,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),6038.0,+0/-0,9.898,0.578,8.22,7.72,7606.4,0.048,✔️,0.0,False,CPU -8,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),5997.0,+0/-0,9.921,0.578,8.67,7.91,61.95,0.016,✔️,0.0,False,CPU -9,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),5959.0,+0/-0,10.441,0.552,9.11,8.66,157.72,0.374,✔️,0.0,False,CPU -10,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),5682.0,+0/-0,11.347,0.508,11.89,10.98,2688.02,0.24,✔️,0.0,False,CPU -11,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),5671.0,+0/-0,11.168,0.518,12.0,11.9,466.47,1.036,✔️,0.0,False,CPU -12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),5573.0,+0/-0,11.536,0.499,13.0,12.51,2688.02,0.726,✔️,0.0,False,CPU -13,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),5519.0,+0/-0,11.505,0.501,13.56,13.39,0.39,0.051,✔️,0.0,False,CPU -14,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),5486.0,+0/-0,11.505,0.501,13.89,13.77,466.47,0.051,✔️,0.0,False,CPU -15,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),5145.0,+0/-0,13.544,0.399,17.0,16.0,1.48,0.53,✔️,0.0,False,GPU -16,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),5108.0,+0/-0,13.063,0.423,17.33,17.24,7.54,0.047,✔️,0.0,False,CPU -17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),5082.0,+0/-0,13.396,0.407,17.56,17.52,4423.24,29.2,✔️,0.0,False,GPU -18,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),5032.0,+0/-0,13.739,0.389,18.0,17.84,17.7,0.086,✔️,0.0,False,CPU -19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),4812.0,+0/-0,14.412,0.354,19.67,19.62,4423.24,2.237,✔️,0.0,False,GPU -20,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),4739.0,+0/-0,14.523,0.348,20.11,20.02,2.82,1.073,✔️,0.0,False,GPU -21,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),3202.0,+0/-0,16.353,0.25,22.11,22.11,20.98,0.329,✔️,0.0,False,CPU -22,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),2532.0,+0/-0,18.457,0.132,24.22,24.17,4638.91,343.215,✔️,0.0,False,GPU -23,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),2475.0,+0/-0,18.732,0.115,24.56,24.47,2389.31,0.267,✔️,0.0,False,CPU -24,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),2475.0,+0/-0,18.777,0.113,24.56,24.51,20.98,0.027,✔️,0.0,False,CPU -25,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),2456.0,+0/-0,18.841,0.109,24.67,24.61,2389.31,0.023,✔️,0.0,False,CPU -26,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1738.0,+0/-0,20.674,0.0,27.44,27.43,14.87,1.855,✔️,0.0,False,GPU -27,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1554.0,+0/-0,20.901,0.0,28.33,28.3,1993.14,0.452,✔️,0.0,False,GPU -28,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1404.0,+0/-0,21.65,0.0,29.22,29.16,1993.14,0.059,✔️,0.0,False,GPU -29,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1218.0,+0/-0,22.097,0.0,30.44,30.4,4638.91,43.781,✔️,0.0,False,GPU -30,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1218.0,+0/-0,22.475,0.0,30.44,30.35,40.54,38.741,✔️,0.0,False,GPU -31,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+0/-0,23.341,0.0,31.89,31.77,0.18,0.008,✔️,0.0,False,CPU -32,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),821.0,+0/-0,23.568,0.0,33.0,32.98,1294.93,0.417,✔️,0.0,False,GPU -33,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),665.0,+0/-0,24.01,0.0,33.89,33.86,1294.93,0.06,✔️,0.0,False,GPU -34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),599.0,+0/-0,24.919,0.0,34.22,34.18,0.18,0.04,✔️,0.0,False,CPU -35,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),-215.0,+0/-0,28.274,0.0,36.11,36.11,1863.58,6.302,✔️,0.0,False,GPU -36,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),-734.0,+0/-0,30.79,0.0,37.56,37.54,1863.58,1.131,✔️,0.0,False,GPU -37,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),-788.0,+0/-0,30.768,0.0,37.78,37.75,3.96,0.025,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),-1010.0,+0/-0,32.321,0.0,38.67,38.66,6.85,0.088,✔️,0.0,False,GPU -39,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),-1462.0,+0/-0,33.496,0.0,39.89,39.89,66.06,1.792,➖,0.0,False,GPU -40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),-2063.0,+0/-0,37.017,0.0,41.0,41.0,7.55,0.149,✔️,0.0,False,GPU -41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),-2631.0,+0/-0,41.044,0.0,42.11,42.11,569.0,0.17,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),-2955.0,+0/-0,41.644,0.0,42.89,42.89,569.0,1.658,✔️,0.0,False,CPU -43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),-3626.0,+0/-0,50.207,0.0,44.11,44.11,0.07,0.015,✔️,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),-4076.0,+0/-0,51.289,0.0,45.22,45.21,2.86,0.085,✔️,0.0,False,CPU -45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),-4255.0,+0/-0,51.687,0.0,45.78,45.77,30.41,0.089,✔️,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),-4823.0,+0/-0,54.976,0.0,47.44,47.44,30.41,0.015,✔️,0.0,False,CPU -47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),-5018.0,+0/-0,55.365,0.0,48.33,48.32,0.13,0.014,✔️,0.0,False,CPU -48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),-5106.0,+0/-0,55.242,0.0,48.78,48.72,3.09,0.997,✔️,0.0,False,GPU -49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),-5308.0,+0/-0,56.49,0.0,49.78,49.77,0.11,0.023,➖,0.0,False,GPU -50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),-5500.0,+0/-0,57.332,0.0,50.56,50.54,76.27,3.054,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),-6498.0,+0/-0,70.796,0.0,52.0,52.0,76.27,0.37,✔️,0.0,False,CPU -52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-7418.0,+0/-0,82.671,0.0,53.0,53.0,0.14,0.136,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),9356.0,+0/-0,1.0,1.11,1.06,0.019,133.98,8.253,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",8889.0,+0/-0,0.935,2.56,2.2,1.7,276.87,4.662,✔️,0.0,False,CPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",8788.0,+0/-0,0.928,3.0,2.84,1.878,276.86,3.44,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),8709.0,+0/-0,0.906,3.33,3.0,2.424,133.98,2.816,✔️,0.0,False,GPU +4,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),6299.0,+0/-0,0.597,6.78,6.31,9.581,61.95,0.17,✔️,0.0,False,CPU +5,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),6288.0,+0/-0,0.598,6.89,6.45,9.545,7606.4,0.826,✔️,0.0,False,CPU +6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),6240.0,+0/-0,0.576,7.33,7.13,10.011,157.72,3.649,✔️,0.0,False,CPU +7,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),6149.0,+0/-0,0.581,8.22,7.72,9.898,7606.4,0.048,✔️,0.0,False,CPU +8,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),6106.0,+0/-0,0.581,8.67,7.91,9.921,61.95,0.016,✔️,0.0,False,CPU +9,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),6064.0,+0/-0,0.555,9.11,8.66,10.441,157.72,0.374,✔️,0.0,False,CPU +10,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),5788.0,+0/-0,0.512,11.89,10.98,11.347,2688.02,0.24,✔️,0.0,False,CPU +11,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),5778.0,+0/-0,0.521,12.0,11.9,11.168,466.47,1.036,✔️,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),5680.0,+0/-0,0.503,13.0,12.51,11.536,2688.02,0.726,✔️,0.0,False,CPU +13,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),5624.0,+0/-0,0.505,13.56,13.39,11.505,0.39,0.051,✔️,0.0,False,CPU +14,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),5589.0,+0/-0,0.505,13.89,13.77,11.505,466.47,0.051,✔️,0.0,False,CPU +15,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),5256.0,+0/-0,0.403,17.0,16.0,13.544,1.48,0.53,✔️,0.0,False,GPU +16,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),5218.0,+0/-0,0.427,17.33,17.24,13.063,7.54,0.047,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),5193.0,+0/-0,0.411,17.56,17.52,13.396,4423.24,29.2,✔️,0.0,False,GPU +18,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),5142.0,+0/-0,0.393,18.0,17.84,13.739,17.7,0.086,✔️,0.0,False,CPU +19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),4927.0,+0/-0,0.359,19.67,19.62,14.412,4423.24,2.237,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),4852.0,+0/-0,0.353,20.11,20.02,14.523,2.82,1.073,✔️,0.0,False,GPU +21,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),3270.0,+0/-0,0.255,22.11,22.11,16.353,20.98,0.329,✔️,0.0,False,CPU +22,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),2605.0,+0/-0,0.138,24.22,24.17,18.457,4638.91,343.215,✔️,0.0,False,GPU +23,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),2547.0,+0/-0,0.119,24.56,24.51,18.777,20.98,0.027,✔️,0.0,False,CPU +24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),2547.0,+0/-0,0.121,24.56,24.47,18.732,2389.31,0.267,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),2528.0,+0/-0,0.115,24.67,24.61,18.841,2389.31,0.023,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1766.0,+0/-0,0.007,27.44,27.43,20.674,14.87,1.855,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1584.0,+0/-0,0.0,28.33,28.3,20.901,1993.14,0.452,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1400.0,+0/-0,0.0,29.44,29.37,21.65,1993.14,0.059,✔️,0.0,False,GPU +29,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1237.0,+0/-0,0.0,30.67,30.6,22.097,4638.91,43.781,✔️,0.0,False,GPU +30,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1195.0,+0/-0,0.0,31.0,30.87,22.475,40.54,38.741,✔️,0.0,False,GPU +31,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1126.0,+0/-0,0.0,31.56,31.48,22.424,9.87,9.237,➖,0.0,False,GPU +32,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+0/-0,0.0,32.56,32.38,23.341,0.18,0.008,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),828.0,+0/-0,0.0,33.78,33.75,23.568,1294.93,0.417,✔️,0.0,False,GPU +34,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),639.0,+0/-0,0.0,34.89,34.86,24.01,1294.93,0.06,✔️,0.0,False,GPU +35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),572.0,+0/-0,0.0,35.22,35.18,24.919,0.18,0.04,✔️,0.0,False,CPU +36,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),-247.0,+0/-0,0.0,37.11,37.11,28.274,1863.58,6.302,✔️,0.0,False,GPU +37,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),-746.0,+0/-0,0.0,38.56,38.54,30.79,1863.58,1.131,✔️,0.0,False,GPU +38,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),-800.0,+0/-0,0.0,38.78,38.75,30.768,3.96,0.025,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),-1014.0,+0/-0,0.0,39.67,39.66,32.321,6.85,0.088,✔️,0.0,False,GPU +40,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),-1449.0,+0/-0,0.0,40.89,40.89,33.496,66.06,1.792,➖,0.0,False,GPU +41,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),-2053.0,+0/-0,0.0,42.0,42.0,37.017,7.55,0.149,✔️,0.0,False,GPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),-2631.0,+0/-0,0.0,43.11,43.11,41.044,569.0,0.17,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),-2953.0,+0/-0,0.0,43.89,43.89,41.644,569.0,1.658,✔️,0.0,False,CPU +44,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),-3647.0,+0/-0,0.0,45.11,45.11,50.207,0.07,0.015,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),-4085.0,+0/-0,0.0,46.22,46.21,51.289,2.86,0.085,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),-4259.0,+0/-0,0.0,46.78,46.77,51.687,30.41,0.089,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),-4819.0,+0/-0,0.0,48.44,48.44,54.976,30.41,0.015,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),-5018.0,+0/-0,0.0,49.33,49.32,55.365,0.13,0.014,✔️,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),-5107.0,+0/-0,0.0,49.78,49.72,55.242,3.09,0.997,✔️,0.0,False,GPU +50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),-5312.0,+0/-0,0.0,50.78,50.77,56.49,0.11,0.023,➖,0.0,False,GPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),-5506.0,+0/-0,0.0,51.56,51.54,57.332,76.27,3.054,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),-6498.0,+0/-0,0.0,53.0,53.0,70.796,76.27,0.37,✔️,0.0,False,CPU +53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-7417.0,+0/-0,0.0,54.0,54.0,82.671,0.14,0.136,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/winrate_matrix.png.zip index 0a738ee205cf1ea6e56f22298c3c5ec5a92c453c..801b1402980b2a5861700f9fcd136aa07b97abec 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:35921b6a257591d7f0e463089aecf81634d72a2b7d1ed995ff0ed5610cf59a47 -size 1124587 +oid sha256:a3239e0593ae7cd933bdcb87062563f73db0a6111a34a9136d168141a1685435 +size 1208223 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 1a2b7fb7f82d3fce972f5d06cc78b82b1425372e..b35b994d3a48b18fd2d891fb85ee3e745bfaad95 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2b4ddaa7ccd24dbe565113585d909e64694185e275c47c6a27e1369b15581373 -size 466173 +oid sha256:018442390822eabe781e31d648aaabf71e49622c3490e7c8619d80a36935efb9 +size 482394 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip index 7ea570ae99276b3100d482ba9e3444f1a1918169..c7a526a23145e7516d3c7c5acca205a80593ef64 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e259b664f7856914c363fdffa3e3b4a194971125ca4551c1bf241c094ac7a4e4 -size 977440 +oid sha256:4dfec3ef2fd3a0a5f8c02e1ff76d49b63a32c87401382250902b313c24172d6c +size 964905 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip index 863e95395cf116c482726849783a9211543db174..292c31ea7640c81c5ce2dd0c3e79e4f9a25a1758 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7e21e39ca976633ff81babc99c5524189acb391cd2186298be234b759c6500ce -size 167986 +oid sha256:43fd79d83c0951fe1c3a33faebda3bc22e5b67a9d1cf7385efad5832a7329917 +size 176711 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/website_leaderboard.csv index 330b59da3ca4a96ed49105b0bb88eae2b8928c87..52f1df7f4772ca07933a06eae265ef8954f19c33 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1751.0,+173/-64,10.436,0.819,7.7,5.05,1154.67,7.879,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1737.0,+229/-89,7.304,0.813,8.15,3.51,2742.37,1.218,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1724.0,+264/-127,6.725,0.8,8.61,2.91,2742.37,7.945,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1637.0,+311/-165,8.157,0.691,12.05,4.51,11.21,0.845,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1589.0,+185/-115,11.489,0.56,14.33,6.97,4471.59,21.354,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1586.0,+164/-99,10.495,0.539,14.47,5.46,6.22,1.092,➖,0.0,False,GPU -6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1578.0,+333/-218,15.957,0.678,14.92,5.03,2986.29,15.005,✔️,14.29,True,GPU -7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1560.0,+120/-62,12.71,0.494,15.82,9.12,4471.59,3.76,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[Mitra (default) [14.29% IMPUTED]](https://arxiv.org/abs/2510.21204),1523.0,+229/-174,18.171,0.533,17.86,8.32,200.23,4.42,✔️,14.29,True,GPU -9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1519.0,+153/-74,16.304,0.443,18.09,10.41,5929.46,1.73,✔️,0.0,False,GPU -10,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1514.0,+213/-86,16.315,0.355,18.38,8.2,6548.58,2.898,✔️,0.0,False,CPU -11,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1487.0,+230/-196,18.242,0.482,20.01,7.48,2986.29,0.63,✔️,14.29,True,GPU -12,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1477.0,+144/-80,17.164,0.384,20.59,10.38,5929.46,0.337,✔️,0.0,False,GPU -13,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1467.0,+185/-157,18.602,0.415,21.19,8.42,4631.38,190.647,✔️,0.0,False,GPU -14,🧠⚡,Foundation Model,[TabPFNv2 (default) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1466.0,+203/-166,16.621,0.408,21.23,10.93,7.26,0.403,✔️,14.29,True,GPU -15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1460.0,+92/-51,15.441,0.27,21.64,18.01,3272.91,1.185,✔️,0.0,False,CPU -16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1451.0,+161/-83,17.426,0.278,22.2,16.88,1502.45,6.33,✔️,0.0,False,CPU -17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1444.0,+136/-72,17.616,0.249,22.61,17.58,1502.45,1.367,✔️,0.0,False,CPU -18,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1431.0,+130/-108,18.163,0.309,23.4,17.52,684.78,4.816,➖,0.0,False,GPU -19,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1429.0,+82/-46,16.176,0.206,23.52,19.44,3272.91,0.21,✔️,0.0,False,CPU -20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1426.0,+107/-136,18.282,0.295,23.75,15.56,2097.62,3.142,✔️,0.0,False,GPU -21,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1425.0,+196/-162,20.167,0.377,23.79,9.25,4631.38,30.351,✔️,0.0,False,GPU -22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1416.0,+127/-95,16.566,0.174,24.37,15.77,7359.32,9.026,✔️,0.0,False,GPU -23,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1411.0,+108/-73,17.767,0.202,24.67,19.44,2551.65,1.021,✔️,0.0,False,CPU -24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1409.0,+78/-58,16.831,0.157,24.81,11.48,7359.32,0.596,✔️,0.0,False,GPU -25,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1400.0,+124/-73,17.966,0.189,25.36,19.94,2551.65,4.527,✔️,0.0,False,CPU -26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1394.0,+158/-112,19.159,0.272,25.73,16.04,19.65,0.236,✔️,0.0,False,GPU -27,🧠⚡,Foundation Model,[TabICL (default) [14.29% IMPUTED]](https://arxiv.org/abs/2502.05564),1344.0,+200/-161,23.44,0.252,28.94,11.94,11.18,2.132,✔️,14.29,True,GPU -28,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1341.0,+80/-84,20.246,0.129,29.14,24.71,63121.14,0.827,✔️,0.0,False,CPU -29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1336.0,+217/-166,21.741,0.237,29.49,12.7,2775.63,12.592,✔️,0.0,False,CPU -30,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1336.0,+190/-189,22.429,0.229,29.51,13.51,717.53,1.628,✔️,0.0,False,CPU -31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1333.0,+164/-116,20.836,0.103,29.68,21.86,4142.25,2.105,✔️,0.0,False,CPU -32,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1321.0,+93/-70,19.905,0.102,30.43,26.4,64.18,0.255,✔️,0.0,False,CPU -33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1319.0,+102/-118,20.614,0.184,30.56,16.78,2097.62,0.515,✔️,0.0,False,GPU -34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1302.0,+185/-212,22.496,0.239,31.6,13.41,755.05,2.155,✔️,0.0,False,CPU -35,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1301.0,+229/-201,23.615,0.25,31.69,8.53,46.25,45.74,✔️,0.0,False,GPU -36,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1294.0,+86/-94,21.074,0.081,32.12,26.74,63121.14,0.173,✔️,0.0,False,CPU -37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1288.0,+190/-222,23.78,0.199,32.47,12.32,717.53,0.339,✔️,0.0,False,CPU -38,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1279.0,+146/-130,22.716,0.108,33.06,24.87,20.91,0.126,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1276.0,+199/-153,23.099,0.115,33.22,17.71,2775.63,1.431,✔️,0.0,False,CPU -40,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1263.0,+171/-174,22.345,0.153,34.05,25.43,5.32,0.431,✔️,0.0,False,CPU -41,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1259.0,+129/-141,23.478,0.112,34.29,23.07,4142.25,0.287,✔️,0.0,False,CPU -42,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1247.0,+184/-230,24.026,0.171,34.98,10.68,755.05,0.317,✔️,0.0,False,CPU -43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1228.0,+38/-42,21.833,0.0,36.13,34.47,5.48,0.599,✔️,0.0,False,CPU -44,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1216.0,+98/-86,26.021,0.025,36.84,30.12,20.96,0.482,✔️,0.0,False,GPU -45,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1205.0,+52/-73,24.386,0.0,37.46,34.86,19.63,7.686,✔️,0.0,False,GPU -46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1156.0,+98/-168,25.653,0.057,40.18,34.66,10.84,2.189,✔️,0.0,False,GPU -47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1088.0,+73/-139,28.451,0.0,43.54,33.72,14.75,0.337,✔️,0.0,False,CPU -48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1077.0,+206/-394,36.825,0.049,44.06,22.38,173.37,1.638,✔️,0.0,False,CPU -49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1038.0,+99/-212,38.383,0.0,45.77,43.6,905.14,0.875,✔️,0.0,False,CPU -50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1005.0,+97/-238,39.053,0.0,47.07,44.56,905.14,0.175,✔️,0.0,False,CPU -51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1004.0,+153/-222,32.313,0.0,47.11,44.7,8.97,0.743,✔️,0.0,False,CPU -52,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+121/-211,37.357,0.0,47.28,43.84,0.85,0.148,✔️,0.0,False,CPU -53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),943.0,+142/-240,39.382,0.0,49.27,46.51,0.78,0.147,✔️,0.0,False,CPU -54,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),915.0,+88/-170,37.336,0.0,50.15,49.83,5.83,1.167,➖,0.0,False,GPU -55,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),908.0,+270/-615,46.661,0.046,50.35,37.69,173.37,0.147,✔️,0.0,False,CPU -56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),776.0,+96/-280,46.061,0.0,53.56,53.19,4.72,0.188,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),261.0,+123/-665,68.83,0.0,57.74,57.72,0.57,0.067,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1737.0,+163/-62,0.81,8.0,5.25,11.398,1154.67,7.879,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1726.0,+208/-81,0.803,8.37,3.64,8.568,2742.37,1.218,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1713.0,+247/-118,0.792,8.84,3.08,7.991,2742.37,7.945,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1628.0,+301/-160,0.676,12.32,5.06,9.527,11.21,0.845,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1580.0,+182/-110,0.547,14.69,7.53,12.571,4471.59,21.354,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1577.0,+155/-93,0.523,14.83,5.71,11.668,6.22,1.092,➖,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1567.0,+313/-214,0.667,15.35,5.41,16.809,2986.29,15.005,✔️,14.29,True,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1551.0,+114/-60,0.49,16.21,9.84,13.674,4471.59,3.76,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[Mitra (default) [14.29% IMPUTED]](https://arxiv.org/abs/2510.21204),1512.0,+227/-171,0.526,18.43,8.54,18.754,200.23,4.42,✔️,14.29,True,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1510.0,+151/-72,0.44,18.52,10.7,16.986,5929.46,1.73,✔️,0.0,False,GPU +10,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1505.0,+203/-85,0.354,18.86,8.55,16.862,6548.58,2.898,✔️,0.0,False,CPU +11,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1477.0,+211/-194,0.472,20.54,7.74,19.035,2986.29,0.63,✔️,14.29,True,GPU +12,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1469.0,+140/-77,0.382,21.06,10.59,17.787,5929.46,0.337,✔️,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1458.0,+185/-155,0.409,21.75,8.55,19.293,4631.38,190.647,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabPFNv2 (default) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1457.0,+200/-164,0.393,21.78,11.76,17.78,7.26,0.403,✔️,14.29,True,GPU +15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1451.0,+83/-47,0.268,22.14,18.73,16.348,3272.91,1.185,✔️,0.0,False,CPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1443.0,+160/-86,0.28,22.64,17.26,17.933,1502.45,6.33,✔️,0.0,False,CPU +17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1437.0,+134/-73,0.251,23.07,18.05,18.134,1502.45,1.367,✔️,0.0,False,CPU +18,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1423.0,+130/-108,0.307,23.93,18.02,18.785,684.78,4.816,➖,0.0,False,GPU +19,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1421.0,+69/-40,0.203,24.05,20.3,17.077,3272.91,0.21,✔️,0.0,False,CPU +20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1418.0,+108/-136,0.286,24.27,15.89,19.106,2097.62,3.142,✔️,0.0,False,GPU +21,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1417.0,+197/-160,0.375,24.36,9.34,20.741,4631.38,30.351,✔️,0.0,False,GPU +22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1409.0,+125/-91,0.173,24.87,16.21,17.232,7359.32,9.026,✔️,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1405.0,+104/-68,0.197,25.13,19.98,18.407,2551.65,1.021,✔️,0.0,False,CPU +24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1402.0,+80/-59,0.16,25.31,11.63,17.423,7359.32,0.596,✔️,0.0,False,GPU +25,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1396.0,+257/-217,0.452,25.7,5.38,17.111,22.56,1.484,➖,0.0,False,GPU +26,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1394.0,+113/-66,0.184,25.83,20.48,18.58,2551.65,4.527,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1387.0,+150/-109,0.277,26.29,16.54,19.638,19.65,0.236,✔️,0.0,False,GPU +28,🧠⚡,Foundation Model,[TabICL (default) [14.29% IMPUTED]](https://arxiv.org/abs/2502.05564),1336.0,+195/-162,0.245,29.66,12.04,23.997,11.18,2.132,✔️,14.29,True,GPU +29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1335.0,+79/-82,0.133,29.71,25.54,20.763,63121.14,0.827,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1330.0,+210/-163,0.235,30.07,12.95,22.06,2775.63,12.592,✔️,0.0,False,CPU +31,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1330.0,+190/-189,0.224,30.07,13.62,23.117,717.53,1.628,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1325.0,+151/-117,0.102,30.35,22.27,21.26,4142.25,2.105,✔️,0.0,False,CPU +33,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1315.0,+89/-70,0.107,31.02,27.34,20.353,64.18,0.255,✔️,0.0,False,CPU +34,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1313.0,+104/-119,0.178,31.16,17.08,21.391,2097.62,0.515,✔️,0.0,False,GPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1297.0,+187/-213,0.231,32.17,13.78,23.34,755.05,2.155,✔️,0.0,False,CPU +36,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1296.0,+224/-200,0.25,32.26,8.56,23.915,46.25,45.74,✔️,0.0,False,GPU +37,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1289.0,+87/-91,0.085,32.71,27.31,21.602,63121.14,0.173,✔️,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1284.0,+191/-216,0.195,33.04,12.4,24.408,717.53,0.339,✔️,0.0,False,CPU +39,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1274.0,+146/-124,0.11,33.69,25.22,23.138,20.91,0.126,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1270.0,+197/-152,0.112,33.9,18.0,23.392,2775.63,1.431,✔️,0.0,False,CPU +41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1258.0,+166/-171,0.155,34.64,25.77,22.684,5.32,0.431,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1253.0,+120/-143,0.118,34.97,23.37,23.801,4142.25,0.287,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1244.0,+183/-228,0.167,35.57,11.12,24.867,755.05,0.317,✔️,0.0,False,CPU +44,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1225.0,+38/-46,0.0,36.69,35.04,22.283,5.48,0.599,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1211.0,+99/-88,0.025,37.54,30.59,26.444,20.96,0.482,✔️,0.0,False,GPU +46,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1200.0,+50/-72,0.0,38.23,35.7,24.783,19.63,7.686,✔️,0.0,False,GPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1154.0,+94/-159,0.05,40.79,35.57,26.325,10.84,2.189,✔️,0.0,False,GPU +48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1087.0,+73/-144,0.0,44.24,34.4,28.857,14.75,0.337,✔️,0.0,False,CPU +49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1078.0,+204/-356,0.056,44.65,22.49,37.086,173.37,1.638,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1034.0,+96/-197,0.0,46.63,44.56,38.813,905.14,0.875,✔️,0.0,False,CPU +51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1003.0,+94/-216,0.0,47.93,45.54,39.372,905.14,0.175,✔️,0.0,False,CPU +52,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1002.0,+147/-195,0.0,47.95,45.63,32.523,8.97,0.743,✔️,0.0,False,CPU +53,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+121/-207,0.0,48.03,44.54,37.732,0.85,0.148,✔️,0.0,False,CPU +54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),945.0,+142/-220,0.0,50.02,47.13,39.656,0.78,0.147,✔️,0.0,False,CPU +55,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),918.0,+88/-168,0.0,50.89,50.52,37.533,5.83,1.167,➖,0.0,False,GPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),912.0,+266/-546,0.052,51.09,38.04,46.859,173.37,0.147,✔️,0.0,False,CPU +57,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),779.0,+87/-242,0.0,54.42,54.08,46.264,4.72,0.188,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),261.0,+117/-619,0.0,58.74,58.72,68.951,0.57,0.067,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/winrate_matrix.png.zip index ff288e173c5328d8029a3f594f7867cff5038bc9..00fa8af5ba52c70b2b5d069c2573c7137fb1ebd2 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2f55ba77437cdf32cc6097928bc3b4ec3998b63fb850b15f511fd7aadc6cf4a6 -size 2647495 +oid sha256:8115ca06ff695eb98487df1b9eaac7d331a67e4208dff6afb2f753e167bf439d +size 2643400 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 99ae29aafd20ef6e3a6efce576f564bfb8c2bdaf..8333d7e08dd1c7e55e177c54ba85df87c4233336 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:01058b28cdf66bea9ac44de109845a5d68623282edbae027f301b710860fbbf4 -size 470825 +oid sha256:96dc3b09af410bd2b2d0f1f11213f6d5dfae2daf59c0ee2994e1c3bfd5c9abda +size 492686 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip index e93bbf13b6edcace10d5541e91c86bc4e9eac54b..3f07e835cc99d3e1fcb5f864e5910ee0ed7a0e5b 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:52d3f25db480d8807d0731b89399343a0d0a2b337fd843fbfa6f16140e6a5a95 -size 993086 +oid sha256:8be0060c77e64c4eb7507d78a86265c500993c08e5a72fa382cbfac8e989566d +size 973613 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip index 55f9afbe3f2734b4bbd475e2d971f90c32fa27c7..c042290f75842ff31a28d06dd27d0ed42dabf9e9 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d4793da334dd47f8dabcc8c3d71845b4bf61d19513b32a3bfb5b5502c9d40f1f -size 122677 +oid sha256:ebf7dd6af7ca0d8813045e0e27121a6f0b710f00de5f9280f3c0e4a7777bafcc +size 127420 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv index 76f7eaabc270279279db328ea3355f3fc07d6416..cb1dd5bd2c4eb37897ba9984e8e2bff79c0831e5 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1876.0,+270/-100,7.591,0.87,6.14,2.55,2710.27,7.889,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1852.0,+240/-92,8.338,0.851,6.75,3.29,2710.27,1.076,✔️,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1838.0,+207/-77,12.046,0.833,7.12,4.7,1374.26,12.843,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1799.0,+306/-137,8.785,0.807,8.24,3.94,10.23,0.844,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1788.0,+296/-112,12.979,0.791,8.58,4.37,3585.96,32.815,✔️,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1708.0,+168/-44,12.954,0.653,11.4,6.17,4150.25,19.131,✔️,0.0,False,GPU -6,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1688.0,+166/-61,11.831,0.628,12.2,5.26,5.85,0.91,➖,0.0,False,GPU -7,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1680.0,+197/-107,15.607,0.622,12.51,7.31,205.23,4.488,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1633.0,+220/-107,15.646,0.562,14.52,6.55,3585.96,1.167,✔️,0.0,False,GPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1630.0,+135/-43,14.49,0.553,14.68,8.59,4150.25,3.423,✔️,0.0,False,GPU -10,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1592.0,+203/-102,13.798,0.476,16.45,9.67,6.47,0.427,✔️,0.0,False,GPU -11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1575.0,+284/-135,18.07,0.414,17.28,7.5,7307.18,4.205,✔️,0.0,False,CPU -12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1549.0,+239/-110,18.854,0.412,18.59,10.08,5885.87,1.632,✔️,0.0,False,GPU -13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1513.0,+108/-46,17.651,0.303,20.47,17.01,3104.39,1.095,✔️,0.0,False,CPU -14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1497.0,+197/-114,19.852,0.346,21.34,10.08,5885.87,0.299,✔️,0.0,False,GPU -15,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1481.0,+240/-202,21.539,0.377,22.22,8.34,4513.63,201.505,✔️,0.0,False,GPU -16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1474.0,+122/-64,18.788,0.203,22.61,14.46,7367.31,7.816,✔️,0.0,False,GPU -17,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1470.0,+95/-49,18.528,0.218,22.86,18.63,3104.39,0.18,✔️,0.0,False,CPU -18,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1468.0,+142/-63,20.186,0.236,22.95,18.11,2210.02,1.236,✔️,0.0,False,CPU -19,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1460.0,+224/-87,20.164,0.219,23.37,17.51,1377.29,6.652,✔️,0.0,False,CPU -20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1455.0,+147/-63,20.414,0.221,23.69,18.61,2210.02,7.723,✔️,0.0,False,CPU -21,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1450.0,+196/-70,20.379,0.188,23.97,18.81,1377.29,1.395,✔️,0.0,False,CPU -22,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1433.0,+202/-128,21.045,0.245,24.92,19.16,696.38,4.428,➖,0.0,False,GPU -23,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1431.0,+237/-220,23.355,0.338,25.04,8.66,4513.63,26.967,✔️,0.0,False,GPU -24,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1424.0,+226/-151,21.754,0.294,25.44,10.59,12.57,2.243,✔️,0.0,False,GPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1420.0,+212/-145,22.067,0.267,25.68,15.1,17.83,0.197,✔️,0.0,False,GPU -26,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1412.0,+144/-153,21.235,0.205,26.15,21.17,1880.81,2.744,✔️,0.0,False,GPU -27,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1410.0,+86/-72,19.439,0.092,26.26,14.18,7367.31,0.587,✔️,0.0,False,GPU -28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1381.0,+315/-274,24.53,0.276,27.89,11.42,2644.99,12.716,✔️,0.0,False,CPU -29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1347.0,+200/-164,23.884,0.121,29.89,21.48,4654.35,2.028,✔️,0.0,False,CPU -30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1340.0,+132/-106,23.387,0.076,30.27,25.3,36484.23,0.712,✔️,0.0,False,CPU -31,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1320.0,+134/-78,22.957,0.06,31.45,27.13,264.83,0.227,✔️,0.0,False,CPU -32,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1318.0,+331/-279,27.104,0.291,31.55,7.6,40.51,46.546,✔️,0.0,False,GPU -33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1289.0,+146/-150,23.905,0.099,33.19,27.21,1880.81,0.353,✔️,0.0,False,GPU -34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1289.0,+262/-248,26.079,0.134,33.2,18.09,2644.99,1.47,✔️,0.0,False,CPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1282.0,+107/-99,24.339,0.027,33.59,27.61,36484.23,0.127,✔️,0.0,False,CPU -36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1278.0,+176/-198,26.121,0.106,33.8,27.13,717.36,1.838,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1255.0,+115/-83,26.62,0.029,35.05,28.39,17.94,0.418,✔️,0.0,False,GPU -38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1247.0,+64/-58,24.705,0.0,35.52,33.72,4.72,0.714,✔️,0.0,False,CPU -39,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1239.0,+180/-243,26.195,0.119,35.94,22.24,737.91,2.289,✔️,0.0,False,CPU -40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1232.0,+70/-89,26.605,0.0,36.29,33.66,20.73,8.179,✔️,0.0,False,GPU -41,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1230.0,+181/-140,27.149,0.06,36.45,24.6,4654.35,0.223,✔️,0.0,False,CPU -42,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1223.0,+101/-89,26.38,0.0,36.81,34.51,14.75,0.108,✔️,0.0,False,CPU -43,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1212.0,+159/-217,27.709,0.065,37.39,29.45,717.36,0.326,✔️,0.0,False,CPU -44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1204.0,+196/-142,25.943,0.055,37.78,34.29,5.04,0.449,✔️,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1160.0,+160/-231,27.997,0.033,40.02,25.17,737.91,0.316,✔️,0.0,False,CPU -46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1156.0,+183/-247,29.106,0.067,40.22,33.93,9.81,2.345,✔️,0.0,False,GPU -47,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1069.0,+131/-286,36.78,0.0,44.15,42.1,892.95,0.895,✔️,0.0,False,CPU -48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1066.0,+132/-195,32.333,0.0,44.29,33.03,13.69,0.314,✔️,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1031.0,+119/-328,37.562,0.0,45.67,43.14,892.95,0.146,✔️,0.0,False,CPU -50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+170/-313,37.99,0.0,46.83,42.97,0.74,0.137,✔️,0.0,False,CPU -51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),985.0,+174/-259,34.643,0.0,47.35,44.55,8.51,0.828,✔️,0.0,False,CPU -52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),954.0,+212/-444,42.698,0.0,48.38,41.73,151.69,1.252,✔️,0.0,False,CPU -53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),932.0,+195/-360,40.173,0.0,49.06,45.92,0.69,0.137,✔️,0.0,False,CPU -54,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),851.0,+29/-142,42.312,0.0,51.27,51.09,5.05,1.013,➖,0.0,False,GPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),775.0,+103/-312,43.542,0.0,52.93,52.56,4.53,0.164,✔️,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),580.0,+156/-543,54.165,0.0,55.58,55.16,151.69,0.099,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),178.0,+157/-1027,69.817,0.0,57.75,57.73,0.52,0.056,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1854.0,+258/-94,0.853,6.4,2.7,9.067,2710.27,7.889,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1831.0,+239/-86,0.834,7.0,3.42,9.813,2710.27,1.076,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1815.0,+176/-79,0.82,7.46,4.91,13.168,1374.26,12.843,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1780.0,+274/-128,0.788,8.54,4.44,10.383,10.23,0.844,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1768.0,+293/-109,0.778,8.92,4.71,13.973,3585.96,32.815,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1689.0,+165/-40,0.638,11.83,6.68,14.217,4150.25,19.131,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1670.0,+167/-65,0.61,12.61,5.54,13.199,5.85,0.91,➖,0.0,False,GPU +7,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),1660.0,+188/-104,0.613,13.01,7.5,16.287,205.23,4.488,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1617.0,+208/-103,0.551,14.97,6.78,16.57,3585.96,1.167,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1613.0,+126/-44,0.539,15.14,9.34,15.614,4150.25,3.423,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1577.0,+200/-98,0.459,16.92,10.42,15.15,6.47,0.427,✔️,0.0,False,GPU +11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1560.0,+264/-128,0.413,17.8,7.84,18.707,7307.18,4.205,✔️,0.0,False,CPU +12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1535.0,+236/-106,0.404,19.09,10.39,19.65,5885.87,1.632,✔️,0.0,False,GPU +13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1499.0,+99/-40,0.29,21.05,17.75,18.709,3104.39,1.095,✔️,0.0,False,CPU +14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1484.0,+191/-110,0.339,21.88,10.31,20.579,5885.87,0.299,✔️,0.0,False,GPU +15,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1472.0,+432/-287,0.528,22.56,4.69,18.42,22.52,1.352,➖,0.0,False,GPU +16,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1467.0,+218/-200,0.367,22.87,8.48,22.345,4513.63,201.505,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1461.0,+122/-67,0.201,23.2,14.89,19.565,7367.31,7.816,✔️,0.0,False,GPU +18,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1456.0,+90/-49,0.205,23.49,19.54,19.58,3104.39,0.18,✔️,0.0,False,CPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1456.0,+134/-60,0.23,23.49,18.66,20.933,2210.02,1.236,✔️,0.0,False,CPU +20,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1450.0,+210/-85,0.217,23.88,17.99,20.755,1377.29,6.652,✔️,0.0,False,CPU +21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1443.0,+142/-61,0.215,24.25,19.16,21.131,2210.02,7.723,✔️,0.0,False,CPU +22,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1439.0,+184/-68,0.186,24.5,19.44,20.984,1377.29,1.395,✔️,0.0,False,CPU +23,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1421.0,+190/-125,0.239,25.55,19.86,21.77,696.38,4.428,➖,0.0,False,GPU +24,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1419.0,+226/-222,0.33,25.7,8.76,24.025,4513.63,26.967,✔️,0.0,False,GPU +25,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1412.0,+227/-154,0.286,26.11,10.68,22.404,12.57,2.243,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1408.0,+200/-143,0.265,26.33,15.63,22.627,17.83,0.197,✔️,0.0,False,GPU +27,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1401.0,+141/-150,0.192,26.76,21.89,22.196,1880.81,2.744,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1399.0,+85/-74,0.091,26.84,14.45,20.13,7367.31,0.587,✔️,0.0,False,GPU +29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1370.0,+296/-274,0.274,28.56,11.66,24.902,2644.99,12.716,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1335.0,+188/-163,0.119,30.67,21.95,24.38,4654.35,2.028,✔️,0.0,False,CPU +31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1331.0,+126/-102,0.074,30.94,26.31,23.99,36484.23,0.712,✔️,0.0,False,CPU +32,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1310.0,+123/-79,0.058,32.13,28.3,23.48,264.83,0.227,✔️,0.0,False,CPU +33,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1309.0,+329/-286,0.292,32.21,7.62,27.455,40.51,46.546,✔️,0.0,False,GPU +34,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1280.0,+137/-143,0.088,33.89,28.16,24.812,1880.81,0.353,✔️,0.0,False,GPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1279.0,+255/-249,0.131,33.96,18.43,26.421,2644.99,1.47,✔️,0.0,False,CPU +36,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1274.0,+101/-95,0.025,34.28,28.32,24.955,36484.23,0.127,✔️,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1271.0,+173/-199,0.1,34.45,27.64,26.924,717.36,1.838,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1249.0,+115/-84,0.03,35.71,28.82,27.113,17.94,0.418,✔️,0.0,False,GPU +39,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1240.0,+62/-58,0.0,36.18,34.36,25.23,4.72,0.714,✔️,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1233.0,+177/-242,0.11,36.61,23.48,27.18,737.91,2.289,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1224.0,+66/-87,0.0,37.07,34.49,27.069,20.73,8.179,✔️,0.0,False,GPU +42,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1221.0,+172/-142,0.061,37.25,24.99,27.526,4654.35,0.223,✔️,0.0,False,CPU +43,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1216.0,+97/-83,0.0,37.55,35.3,26.871,14.75,0.108,✔️,0.0,False,CPU +44,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1206.0,+156/-215,0.061,38.05,30.01,28.442,717.36,0.326,✔️,0.0,False,CPU +45,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1198.0,+182/-146,0.054,38.47,35.02,26.339,5.04,0.449,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1156.0,+156/-228,0.028,40.7,28.24,28.978,737.91,0.316,✔️,0.0,False,CPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1151.0,+171/-232,0.059,40.94,34.96,29.89,9.81,2.345,✔️,0.0,False,GPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1064.0,+121/-249,0.0,44.99,43.05,37.281,892.95,0.895,✔️,0.0,False,CPU +49,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1061.0,+120/-198,0.0,45.11,33.8,32.806,13.69,0.314,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1027.0,+107/-286,0.0,46.51,44.12,37.933,892.95,0.146,✔️,0.0,False,CPU +51,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+166/-304,0.0,47.55,43.63,38.428,0.74,0.137,✔️,0.0,False,CPU +52,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),983.0,+169/-237,0.0,48.16,45.47,34.887,8.51,0.828,✔️,0.0,False,CPU +53,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),957.0,+207/-446,0.0,49.08,42.18,43.003,151.69,1.252,✔️,0.0,False,CPU +54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),935.0,+190/-378,0.0,49.76,46.49,40.494,0.69,0.137,✔️,0.0,False,CPU +55,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),852.0,+29/-148,0.0,52.11,51.92,42.542,5.05,1.013,➖,0.0,False,GPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),779.0,+98/-284,0.0,53.77,53.43,43.779,4.53,0.164,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),595.0,+131/-452,0.0,56.43,56.03,54.395,151.69,0.099,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),180.0,+144/-928,0.0,58.75,58.73,69.958,0.52,0.056,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip index 0283a9a362ffb8fe9c812fb95f595f1e0065650b..61cc51ea97b9f7420c581341b69140dc445a3eb5 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4b515780fe16db2c2a6a719375aab2e3ff732c2395831088070bcc6b5d0cc3fd -size 2656649 +oid sha256:c6e337178c0ef31f755a8a6919353286a61dd8a37851c8ed1c35c8df41032641 +size 2665550 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index 18eb87b58a1276c0e199a118e22cb0d72b0d8cbc..f62e5685345688b9fbde8d121814db3d9c179918 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d0b9fd084e27e289410b1d17c6cb797cecffb8f820d2b875f34753ef58e0b65f -size 450891 +oid sha256:57983c52b04603afaa61bf664539e0c64b3d30626df3dce4e0cf169a36c47d49 +size 463649 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip index 4ea67acb76c7c3342da5ec53791f072e93a1928e..3e9786b2a84abe74cee5fdfbff7199804d92589a 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f15cf0c6dce6d0688816075ec45b3651093bebe496aeb725ca68c9c6182cdd31 -size 1012565 +oid sha256:09e35448a72f7ff9be96f59cb7e8716c2aa2ba68eaae3301b8b9c8b426f0c234 +size 1021574 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_all/tuning-impact-elo.png.zip index 50b4fd66a30f280d47bd4d6e46d6e83fce0c2dc2..0819b82a381ddf122c9ae25cb540118dbff24e7c 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a8b54bcefa8b112e1bbf33cb098a29205bff08d9c739ef4537157ca4ed46b8b8 -size 146665 +oid sha256:3233549ca857c4b507148f668b7d41c8f86c38a19a74b88a96431dc50fb6c163 +size 151350 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_regression/datasets_all/website_leaderboard.csv index a13830b2ec3c234295495c10706f3db5345ddd0a..36bf2bc2b864b3ce02425e79dbf5c201655c41d0 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_all/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_all/website_leaderboard.csv @@ -1,56 +1,57 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1779.0,+172/-123,1.541,0.805,6.65,2.66,1709.05,8.122,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1741.0,+134/-74,2.451,0.789,7.69,2.75,4786.55,239.537,✔️,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1723.0,+109/-80,2.962,0.689,8.24,4.37,649.34,5.869,✔️,0.0,False,GPU -3,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1691.0,+101/-62,3.249,0.676,9.28,5.76,3995.01,10.051,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1690.0,+109/-63,2.902,0.727,9.3,4.54,4786.55,38.501,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1654.0,+164/-114,2.299,0.691,10.59,4.46,1709.05,0.812,✔️,0.0,False,GPU -6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1632.0,+109/-89,4.693,0.553,11.42,6.63,1866.07,6.07,✔️,0.0,False,CPU -7,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1598.0,+123/-63,3.844,0.603,12.77,4.99,46.62,39.212,✔️,0.0,False,GPU -8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1572.0,+107/-73,4.229,0.506,13.85,8.41,3995.01,0.844,✔️,0.0,False,GPU -9,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1567.0,+141/-106,3.812,0.56,14.06,7.91,7.04,0.508,✔️,0.0,False,GPU -10,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1558.0,+132/-115,5.58,0.499,14.44,5.51,3779.52,7.69,✔️,0.0,False,GPU -11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1506.0,+102/-62,6.274,0.361,16.85,10.35,3552.96,0.966,✔️,0.0,False,CPU -12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1498.0,+92/-84,6.736,0.356,17.22,13.25,700.15,9.322,✔️,0.0,False,CPU -13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1488.0,+107/-82,5.764,0.376,17.72,10.78,714.5,1.376,✔️,0.0,False,GPU -14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1483.0,+105/-68,6.394,0.335,17.95,10.1,3552.96,0.105,✔️,0.0,False,CPU -15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1469.0,+125/-91,5.166,0.348,18.65,13.69,4158.29,1.41,✔️,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1434.0,+89/-74,7.328,0.274,20.42,15.98,700.15,0.968,✔️,0.0,False,CPU -17,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1426.0,+183/-165,13.181,0.426,20.84,5.7,5.2,0.755,➖,0.0,False,GPU -18,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1419.0,+60/-46,7.27,0.248,21.16,18.57,834.93,2.614,✔️,0.0,False,CPU -19,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1404.0,+94/-77,6.439,0.265,21.9,17.61,714.5,0.099,✔️,0.0,False,GPU -20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1398.0,+63/-51,7.418,0.218,22.21,19.08,834.93,0.388,✔️,0.0,False,CPU -21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1393.0,+108/-87,7.578,0.277,22.51,13.43,3779.52,0.396,✔️,0.0,False,GPU -22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1387.0,+120/-99,6.122,0.247,22.8,15.86,4158.29,0.167,✔️,0.0,False,GPU -23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1383.0,+91/-93,7.975,0.254,22.99,16.39,10.89,0.092,✔️,0.0,False,CPU -24,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1335.0,+170/-110,6.948,0.323,25.52,8.92,4223.87,27.543,✔️,46.15,True,GPU -25,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1292.0,+87/-74,9.117,0.138,27.84,21.63,15.5,0.299,✔️,0.0,False,GPU -26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1292.0,+127/-111,7.84,0.157,27.84,22.13,13.32,0.131,✔️,0.0,False,GPU -27,🧠⚡,Foundation Model,[Mitra (default) [46.15% IMPUTED]](https://arxiv.org/abs/2510.21204),1281.0,+136/-104,8.622,0.194,28.41,10.03,71.01,1.848,✔️,46.15,True,GPU -28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1260.0,+117/-101,9.33,0.111,29.53,20.46,4608.59,1.233,✔️,0.0,False,CPU -29,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1253.0,+144/-139,8.01,0.21,29.91,15.14,4223.87,0.452,✔️,46.15,True,GPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1250.0,+117/-90,8.854,0.087,30.07,24.85,8.9,1.645,✔️,0.0,False,GPU -31,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1228.0,+106/-100,11.668,0.101,31.22,23.16,158.22,0.844,✔️,0.0,False,CPU -32,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1222.0,+44/-43,9.759,0.006,31.49,30.02,2.11,0.275,✔️,0.0,False,CPU -33,🧠⚡,Foundation Model,[TabPFNv2 (default) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1205.0,+137/-137,9.409,0.153,32.37,19.91,2.8,0.313,✔️,46.15,True,GPU -34,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1200.0,+82/-82,10.418,0.071,32.61,28.64,2.24,0.242,✔️,0.0,False,CPU -35,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1199.0,+95/-95,11.88,0.076,32.65,22.52,158.22,0.151,✔️,0.0,False,CPU -36,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1198.0,+111/-102,10.122,0.076,32.69,24.55,4608.59,0.097,✔️,0.0,False,CPU -37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1172.0,+64/-71,12.437,0.033,33.99,31.5,515.73,0.771,✔️,0.0,False,CPU -38,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1165.0,+117/-116,12.184,0.038,34.34,26.93,2.45,0.743,✔️,0.0,False,GPU -39,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1159.0,+132/-162,12.916,0.108,34.62,25.51,2929.85,0.294,✔️,0.0,False,CPU -40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1128.0,+68/-83,12.925,0.023,36.1,33.54,515.73,0.124,✔️,0.0,False,CPU -41,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1112.0,+129/-163,13.453,0.09,36.89,27.31,2929.85,0.03,✔️,0.0,False,CPU -42,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1078.0,+103/-121,13.818,0.004,38.39,34.16,0.47,0.055,✔️,0.0,False,CPU -43,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1048.0,+120/-182,14.389,0.046,39.7,33.61,8.47,0.035,✔️,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1042.0,+100/-116,13.796,0.022,39.93,36.96,540.06,2.672,✔️,0.0,False,CPU -45,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1040.0,+89/-111,13.552,0.001,40.01,37.65,20.48,0.08,✔️,0.0,False,CPU -46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+49/-76,14.44,0.0,41.58,40.25,0.53,0.062,✔️,0.0,False,CPU -47,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),986.0,+100/-110,14.283,0.012,42.11,39.99,540.06,0.325,✔️,0.0,False,CPU -48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),896.0,+144/-178,19.61,0.0,45.08,39.28,92.55,0.897,✔️,0.0,False,CPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),860.0,+113/-170,18.544,0.0,46.1,44.94,2.6,0.392,✔️,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),792.0,+148/-192,22.032,0.0,47.8,43.61,92.55,0.052,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),664.0,+192/-310,29.211,0.0,50.22,42.38,0.19,0.04,✔️,0.0,False,CPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),490.0,+135/-304,36.423,0.0,52.38,52.21,193.95,0.173,✔️,0.0,False,CPU -53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),442.0,+158/-371,36.597,0.0,52.82,52.63,193.95,0.074,✔️,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),223.0,+141/-372,39.031,0.0,54.31,54.29,0.95,0.097,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1769.0,+173/-123,0.803,6.77,2.67,1.552,1709.05,8.122,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1733.0,+134/-75,0.787,7.8,2.77,2.463,4786.55,239.537,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1714.0,+112/-81,0.687,8.41,4.41,2.973,649.34,5.869,✔️,0.0,False,GPU +3,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1684.0,+102/-64,0.674,9.4,5.83,3.26,3995.01,10.051,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1682.0,+107/-62,0.725,9.44,4.62,2.914,4786.55,38.501,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1645.0,+165/-115,0.688,10.8,4.48,2.311,1709.05,0.812,✔️,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1623.0,+111/-90,0.548,11.64,6.68,4.705,1866.07,6.07,✔️,0.0,False,CPU +7,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1590.0,+126/-60,0.602,12.97,5.07,3.856,46.62,39.212,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1566.0,+105/-74,0.505,14.02,8.49,4.241,3995.01,0.844,✔️,0.0,False,GPU +9,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1560.0,+140/-107,0.559,14.28,7.98,3.824,7.04,0.508,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1550.0,+133/-115,0.497,14.73,5.57,5.592,3779.52,7.69,✔️,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1499.0,+100/-62,0.357,17.13,10.44,6.286,3552.96,0.966,✔️,0.0,False,CPU +12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1490.0,+91/-81,0.353,17.58,13.43,6.748,700.15,9.322,✔️,0.0,False,CPU +13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1481.0,+107/-83,0.375,18.02,11.07,5.775,714.5,1.376,✔️,0.0,False,GPU +14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1477.0,+103/-66,0.33,18.24,10.18,6.406,3552.96,0.105,✔️,0.0,False,CPU +15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1463.0,+126/-91,0.347,18.94,13.85,5.178,4158.29,1.41,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1427.0,+90/-72,0.269,20.79,16.15,7.339,700.15,0.968,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1422.0,+182/-161,0.427,21.08,5.75,13.193,5.2,0.755,➖,0.0,False,GPU +18,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1413.0,+62/-44,0.243,21.53,18.87,7.281,834.93,2.614,✔️,0.0,False,CPU +19,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1400.0,+92/-78,0.267,22.23,17.93,6.451,714.5,0.099,✔️,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1393.0,+63/-52,0.213,22.57,19.35,7.429,834.93,0.388,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1387.0,+102/-87,0.279,22.9,13.56,7.59,3779.52,0.396,✔️,0.0,False,GPU +22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1381.0,+119/-99,0.246,23.17,16.02,6.134,4158.29,0.167,✔️,0.0,False,GPU +23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1376.0,+89/-94,0.247,23.48,16.6,7.987,10.89,0.092,✔️,0.0,False,CPU +24,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1332.0,+166/-107,0.316,25.86,9.17,6.96,4223.87,27.543,✔️,46.15,True,GPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1289.0,+125/-113,0.154,28.23,22.38,7.851,13.32,0.131,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1287.0,+86/-75,0.143,28.33,22.27,9.129,15.5,0.299,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[Mitra (default) [46.15% IMPUTED]](https://arxiv.org/abs/2510.21204),1279.0,+134/-102,0.193,28.8,10.32,8.634,71.01,1.848,✔️,46.15,True,GPU +28,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1276.0,+140/-133,0.22,28.95,12.98,9.16,20.22,6.621,➖,0.0,False,GPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1257.0,+114/-103,0.112,30.0,20.66,9.342,4608.59,1.233,✔️,0.0,False,CPU +30,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1250.0,+140/-135,0.207,30.38,15.35,8.021,4223.87,0.452,✔️,46.15,True,GPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1246.0,+115/-88,0.09,30.57,25.45,8.866,8.9,1.645,✔️,0.0,False,GPU +32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1225.0,+102/-100,0.098,31.72,23.65,11.679,158.22,0.844,✔️,0.0,False,CPU +33,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1218.0,+46/-46,0.01,32.05,30.5,9.77,2.11,0.275,✔️,0.0,False,CPU +34,🧠⚡,Foundation Model,[TabPFNv2 (default) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1202.0,+132/-133,0.151,32.87,20.46,9.421,2.8,0.313,✔️,46.15,True,GPU +35,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1197.0,+109/-104,0.075,33.16,24.77,10.133,4608.59,0.097,✔️,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1197.0,+91/-94,0.073,33.16,23.2,11.892,158.22,0.151,✔️,0.0,False,CPU +37,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1195.0,+81/-85,0.069,33.24,29.07,10.43,2.24,0.242,✔️,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1170.0,+64/-71,0.029,34.55,32.11,12.449,515.73,0.771,✔️,0.0,False,CPU +39,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1163.0,+116/-114,0.042,34.87,27.84,12.195,2.45,0.743,✔️,0.0,False,GPU +40,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1155.0,+130/-158,0.106,35.31,26.02,12.927,2929.85,0.294,✔️,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1127.0,+68/-85,0.019,36.67,34.09,12.937,515.73,0.124,✔️,0.0,False,CPU +42,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1107.0,+127/-162,0.089,37.63,27.75,13.464,2929.85,0.03,✔️,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1076.0,+101/-121,0.004,39.11,34.96,13.83,0.47,0.055,✔️,0.0,False,CPU +44,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1043.0,+120/-182,0.048,40.52,34.24,14.4,8.47,0.035,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1039.0,+90/-113,0.003,40.71,38.23,13.563,20.48,0.08,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1038.0,+99/-116,0.023,40.74,37.61,13.808,540.06,2.672,✔️,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+50/-79,0.0,42.27,40.99,14.452,0.53,0.062,✔️,0.0,False,CPU +48,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),982.0,+100/-110,0.013,42.95,40.71,14.294,540.06,0.325,✔️,0.0,False,CPU +49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),893.0,+141/-174,0.0,45.96,40.1,19.621,92.55,0.897,✔️,0.0,False,CPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),858.0,+114/-172,0.0,46.98,45.75,18.555,2.6,0.392,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),786.0,+146/-190,0.0,48.79,44.65,22.043,92.55,0.052,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),660.0,+191/-308,0.0,51.21,43.22,29.222,0.19,0.04,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),484.0,+136/-305,0.0,53.38,53.21,36.434,193.95,0.173,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),436.0,+157/-367,0.0,53.82,53.63,36.608,193.95,0.074,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),218.0,+139/-368,0.0,55.31,55.29,39.041,0.95,0.097,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_all/winrate_matrix.png.zip index 0f6f06aab2f8ed4768a0c9370357148a111b5636..675b4d8cc7dff4ab02551f0da2ecf45f91a142b8 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:606159a521a7d039fd66432384bdd9d435724fcd34d513dc4993e88436d88a97 -size 2131657 +oid sha256:62968cb9d91f3e338f725dccf57169e72385801442c4d221dace8f634476bfd5 +size 2328008 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index dd56a8e951d78af2dc92084490677654d022ba72..4e45b3c81633f9ac02662a9d2aac58ca8509858e 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:10a1fb80b2917bb99671af7473075c26a3c071ae4c30bc6f8e5297cb05fd1690 -size 429005 +oid sha256:8fa3747a6469b351929940951006101e78db142f0d0a3de0946e16b667e99447 +size 442026 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip index c3c6ffc7f6c80ca425880c7deb9eac51cf340bca..fbd38800517b0e531dedb5c5d1e884f56f93d47a 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:422dfdc35bcd212f01a7e528ff65f7d427c5e32c1d35f7c2f4ee59a2827a7399 -size 928150 +oid sha256:cefbee39096ab67f85b0ecb47f5edd3ee1e2e9aec1cb01f480733dd3950e7e25 +size 922963 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/tuning-impact-elo.png.zip index 34716789f8d53fbc5a2e1e68aebe15438166d14b..3d8ef06bdeab34a052ee0c538f5167ec8faa1706 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f3b29f0820d7bfc06bfea5a97fbff3701d1842dbb104382a4608c4b9ef057abc -size 119720 +oid sha256:7c61083d351da1586cb23cc57abe188208d327f900a305e8254a49bad90d4155 +size 122391 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/website_leaderboard.csv index e3cd8978a55c78ce453557b9d24fccd2ef76e30c..e1c802cedd223c5af92024526bfa387c2d5925ef 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/website_leaderboard.csv @@ -1,52 +1,53 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),2069.0,+1045/-120,1.209,0.862,4.4,2.67,735.58,9.557,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1971.0,+783/-126,1.696,0.802,6.31,4.21,735.58,0.714,✔️,0.0,False,GPU -2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1922.0,+681/-41,2.304,0.713,7.47,5.68,2526.28,3.717,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1917.0,+1210/-123,1.293,0.782,7.6,2.16,4786.55,361.217,✔️,0.0,False,GPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1909.0,+515/-11,3.331,0.661,7.8,4.92,473.01,8.73,✔️,0.0,False,CPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1900.0,+797/-114,2.7,0.688,8.04,5.61,2028.29,1.675,✔️,0.0,False,GPU -6,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1884.0,+532/-21,3.488,0.644,8.47,4.35,430.4,7.174,✔️,0.0,False,GPU -7,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1850.0,+1175/-128,1.707,0.717,9.42,4.22,4786.55,41.599,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1802.0,+856/-218,2.636,0.564,10.87,7.04,1.6,0.508,✔️,0.0,False,GPU -9,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1753.0,+1086/-143,2.355,0.591,12.42,4.95,45.77,39.212,✔️,0.0,False,GPU -10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1721.0,+794/-134,3.741,0.535,13.47,8.53,2028.29,0.081,✔️,0.0,False,GPU -11,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1684.0,+503/-95,4.288,0.474,14.71,9.45,425.61,0.615,✔️,0.0,False,GPU -12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1671.0,+376/-133,5.936,0.388,15.18,11.21,222.05,11.236,✔️,0.0,False,CPU -13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1650.0,+347/-105,6.099,0.356,15.93,8.83,1658.41,0.742,✔️,0.0,False,CPU -14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1623.0,+616/-142,4.108,0.443,16.89,13.4,2526.28,0.166,✔️,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1619.0,+361/-178,6.197,0.35,17.04,7.26,1658.41,0.059,✔️,0.0,False,CPU -16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1618.0,+331/-121,6.24,0.326,17.09,13.63,222.05,0.968,✔️,0.0,False,CPU -17,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1602.0,+522/-214,5.117,0.341,17.64,14.24,3440.87,0.619,✔️,0.0,False,GPU -18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1555.0,+327/-104,5.243,0.296,19.4,17.23,425.61,0.04,✔️,0.0,False,GPU -19,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1545.0,+667/-210,4.564,0.305,19.76,5.04,2.06,0.755,➖,0.0,False,GPU -20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1533.0,+425/-345,6.882,0.326,20.22,14.49,5.49,0.037,✔️,0.0,False,CPU -21,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1526.0,+421/-182,5.96,0.24,20.44,18.38,3440.87,0.052,✔️,0.0,False,GPU -22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1509.0,+268/-136,6.83,0.217,21.09,19.43,282.13,1.74,✔️,0.0,False,CPU -23,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1506.0,+647/-156,5.272,0.285,21.18,18.43,6.88,0.094,✔️,0.0,False,GPU -24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1497.0,+267/-142,6.953,0.197,21.51,19.3,282.13,0.388,✔️,0.0,False,CPU -25,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1360.0,+317/-308,8.067,0.159,26.42,21.95,0.6,0.08,✔️,0.0,False,CPU -26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1330.0,+330/-211,7.827,0.091,27.49,25.06,5.03,0.025,✔️,0.0,False,GPU -27,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1305.0,+240/-126,8.79,0.011,28.33,27.32,2.28,0.13,✔️,0.0,False,GPU -28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1301.0,+209/-114,8.242,0.061,28.49,26.91,4348.61,0.528,✔️,0.0,False,CPU -29,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1292.0,+283/-160,8.309,0.013,28.78,27.91,0.55,0.233,✔️,0.0,False,CPU -30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1219.0,+204/-297,9.231,0.038,31.24,29.53,4348.61,0.034,✔️,0.0,False,CPU -31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1199.0,+231/-316,9.7,0.0,31.89,30.73,6.75,0.235,✔️,0.0,False,GPU -32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1176.0,+359/-419,10.264,0.022,32.62,30.66,35.67,0.529,✔️,0.0,False,CPU -33,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1156.0,+87/-156,9.993,0.0,33.27,33.0,79.08,0.546,✔️,0.0,False,CPU -34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1144.0,+260/-394,10.361,0.006,33.64,32.41,35.67,0.05,✔️,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1127.0,+112/-248,10.297,0.0,34.16,33.79,79.08,0.057,✔️,0.0,False,CPU -36,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1102.0,+387/-661,13.832,0.143,34.91,29.95,2044.56,0.128,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1037.0,+193/-452,13.252,0.001,36.8,35.28,15.08,0.028,✔️,0.0,False,CPU -38,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1029.0,+364/-709,14.402,0.114,37.02,33.51,2044.56,0.01,✔️,0.0,False,CPU -39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1026.0,+358/-676,12.06,0.006,37.11,35.07,0.17,0.049,✔️,0.0,False,CPU -40,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+37/-248,11.446,0.0,37.82,37.67,0.43,0.053,✔️,0.0,False,CPU -41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),976.0,+370/-780,15.083,0.102,38.47,35.09,3.66,0.011,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),848.0,+60/-540,14.323,0.0,41.4,41.16,318.98,0.827,✔️,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),794.0,+54/-529,14.781,0.0,42.44,42.26,318.98,0.114,✔️,0.0,False,CPU -44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),789.0,+112/-640,18.087,0.0,42.53,42.33,142.58,1.184,✔️,0.0,False,CPU -45,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),664.0,+83/-704,20.643,0.0,44.53,44.43,142.58,0.064,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),607.0,+116/-707,18.384,0.0,45.27,45.07,1.36,0.111,✔️,0.0,False,CPU -47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),349.0,+100/-1185,28.1,0.0,47.51,47.41,0.06,0.034,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),81.0,+131/-1638,39.828,0.0,48.92,48.92,44.3,0.048,✔️,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),-114.0,+182/-2167,40.21,0.0,49.77,49.76,44.3,0.011,✔️,0.0,False,CPU -50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),-414.0,+224/-2121,43.378,0.0,50.8,50.8,0.13,0.018,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),2047.0,+764/-106,0.863,4.4,2.67,1.209,735.58,9.557,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1950.0,+666/-113,0.804,6.31,4.21,1.696,735.58,0.714,✔️,0.0,False,GPU +2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1901.0,+626/-36,0.714,7.47,5.68,2.304,2526.28,3.717,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1896.0,+870/-123,0.784,7.6,2.16,1.293,4786.55,361.217,✔️,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1884.0,+405/-19,0.661,7.91,4.94,3.331,473.01,8.73,✔️,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1878.0,+626/-101,0.69,8.04,5.61,2.7,2028.29,1.675,✔️,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1856.0,+413/-22,0.644,8.64,4.37,3.488,430.4,7.174,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1827.0,+835/-112,0.72,9.49,4.23,1.707,4786.55,41.599,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1781.0,+682/-204,0.563,10.87,7.04,2.636,1.6,0.508,✔️,0.0,False,GPU +9,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1728.0,+756/-126,0.596,12.6,4.97,2.355,45.77,39.212,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1702.0,+631/-121,0.537,13.47,8.53,3.741,2028.29,0.081,✔️,0.0,False,GPU +11,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1660.0,+414/-86,0.477,14.91,9.51,4.288,425.61,0.615,✔️,0.0,False,GPU +12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1647.0,+417/-129,0.388,15.38,11.27,5.936,222.05,11.236,✔️,0.0,False,CPU +13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1626.0,+343/-86,0.356,16.13,8.87,6.099,1658.41,0.742,✔️,0.0,False,CPU +14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1602.0,+520/-127,0.446,17.0,13.53,4.108,2526.28,0.166,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1595.0,+380/-166,0.35,17.24,7.29,6.197,1658.41,0.059,✔️,0.0,False,CPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1594.0,+350/-124,0.327,17.29,13.7,6.24,222.05,0.968,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1580.0,+488/-198,0.341,17.82,14.46,5.117,3440.87,0.619,✔️,0.0,False,GPU +18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1533.0,+333/-88,0.3,19.6,17.39,5.243,425.61,0.04,✔️,0.0,False,GPU +19,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1529.0,+582/-198,0.312,19.76,5.04,4.564,2.06,0.755,➖,0.0,False,GPU +20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1507.0,+389/-310,0.326,20.6,14.61,6.882,5.49,0.037,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1506.0,+419/-165,0.24,20.64,18.61,5.96,3440.87,0.052,✔️,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1488.0,+296/-123,0.218,21.29,19.54,6.83,282.13,1.74,✔️,0.0,False,CPU +23,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1488.0,+498/-141,0.29,21.31,18.73,5.272,6.88,0.094,✔️,0.0,False,GPU +24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1477.0,+282/-127,0.2,21.71,19.41,6.953,282.13,0.388,✔️,0.0,False,CPU +25,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1340.0,+331/-237,0.161,26.8,22.15,8.067,0.6,0.08,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1316.0,+308/-192,0.09,27.69,25.31,7.827,5.03,0.025,✔️,0.0,False,GPU +27,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1294.0,+235/-101,0.011,28.53,27.45,8.79,2.28,0.13,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1289.0,+216/-86,0.061,28.69,27.06,8.242,4348.61,0.528,✔️,0.0,False,CPU +29,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1280.0,+286/-140,0.013,29.02,28.13,8.309,0.55,0.233,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1212.0,+206/-194,0.038,31.44,29.68,9.231,4348.61,0.034,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1189.0,+234/-185,0.001,32.24,31.03,9.7,6.75,0.235,✔️,0.0,False,GPU +32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1172.0,+372/-356,0.022,32.84,30.78,10.264,35.67,0.529,✔️,0.0,False,CPU +33,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1153.0,+93/-122,0.0,33.49,33.19,9.993,79.08,0.546,✔️,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1141.0,+280/-307,0.007,33.87,32.55,10.361,35.67,0.05,✔️,0.0,False,CPU +35,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1140.0,+414/-404,0.117,33.91,25.88,9.879,8.79,7.637,➖,0.0,False,GPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1126.0,+116/-182,0.0,34.38,33.98,10.297,79.08,0.057,✔️,0.0,False,CPU +37,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1086.0,+378/-683,0.145,35.67,30.41,13.832,2044.56,0.128,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1032.0,+193/-356,0.001,37.31,35.66,13.252,15.08,0.028,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1018.0,+370/-475,0.007,37.71,35.47,12.06,0.17,0.049,✔️,0.0,False,CPU +40,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1015.0,+356/-736,0.117,37.82,34.06,14.402,2044.56,0.01,✔️,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+53/-251,0.0,38.24,38.08,11.446,0.43,0.053,✔️,0.0,False,CPU +42,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),963.0,+363/-777,0.106,39.27,35.64,15.083,3.66,0.011,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),841.0,+67/-394,0.0,42.22,41.93,14.323,318.98,0.827,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),786.0,+61/-473,0.0,43.31,43.1,14.781,318.98,0.114,✔️,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),785.0,+97/-596,0.0,43.33,43.12,18.087,142.58,1.184,✔️,0.0,False,CPU +46,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),651.0,+66/-607,0.0,45.53,45.43,20.643,142.58,0.064,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),602.0,+108/-646,0.0,46.18,45.96,18.384,1.36,0.111,✔️,0.0,False,CPU +48,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),338.0,+94/-1186,0.0,48.51,48.42,28.1,0.06,0.034,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),70.0,+111/-1680,0.0,49.92,49.92,39.828,44.3,0.048,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),-125.0,+167/-2054,0.0,50.77,50.76,40.21,44.3,0.011,✔️,0.0,False,CPU +51,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),-424.0,+210/-2019,0.0,51.8,51.8,43.378,0.13,0.018,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/winrate_matrix.png.zip index d89d739d2a6bcb5a1ec4e70aade6b02883e9e1c6..0d5ddc124ced4bda95e1b4030cd635d913464a03 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9d3c74b837da47402b288b5d1b16560ff38851ecc10edfdc0ac1295c66466286 -size 1489576 +oid sha256:4689b1ce3c64163ca076d064c1314d1fef037392e057262df7a5a8ff3934bba0 +size 1623604 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 0145f7036b9213b0f283b3f10143bc8ad85aa92f..a70248f57ddbddad02e055afd81039c214bd3234 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:dca244a7249bc27fd9ccef2a0ec6aec3e9bff0b929743767fa7891cc015f4b97 -size 449740 +oid sha256:8a548cf336dced33e8a0ed1377599056fedbbb7edd97aea54cd80e9fd115b75c +size 463338 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip index 1af65caa1c21f07a8448f1029291ab99a1fa3f86..025d2fb86aecf76e43f9f90e6e46c735baa36e4f 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6d0d1fdb578bc3575be12ae1c031831a0e2e345fd640074e1954dc588c5621f5 -size 1055036 +oid sha256:7138192d9c9e070336a6ee6edae9479f6aa8811bc7aa3a4c22710a7561d877eb +size 1039702 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_small/tuning-impact-elo.png.zip index 3d895dae4fe3589d606cf2e24a9cda961c6e7321..6b33cc578de457e1f2c93ca336e09af6e54f9565 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:add6494fc27962748116fba959a0895e4776331e2ba44037e77963f479a874ec -size 157524 +oid sha256:fbf45b4896a7a2d1e01b22d6d63ea37572fc157f16391e06b465babef397b9c0 +size 162242 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_regression/datasets_small/website_leaderboard.csv index a69c6b779ace8aa97c32b11679e3cb3369f4afd1..c33e6d7b6c29295ab9e432450f2b9b85962594d2 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_small/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_small/website_leaderboard.csv @@ -1,56 +1,57 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1732.0,+139/-53,3.175,0.779,7.74,3.31,4822.0,140.81,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1722.0,+271/-134,1.748,0.765,8.05,2.65,2306.87,7.231,✔️,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1721.0,+208/-83,2.633,0.707,8.09,4.39,709.58,5.864,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1687.0,+107/-42,3.649,0.716,9.22,4.77,4822.0,22.324,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1664.0,+125/-50,3.591,0.658,10.05,5.86,4558.61,33.853,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1591.0,+114/-48,4.775,0.584,12.98,5.02,48.13,42.001,✔️,0.0,False,GPU -6,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1585.0,+224/-129,2.677,0.615,13.26,4.62,2306.87,0.902,✔️,0.0,False,GPU -7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1576.0,+131/-73,5.544,0.476,13.68,8.49,5805.42,4.31,✔️,0.0,False,CPU -8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1566.0,+119/-72,4.535,0.476,14.1,8.34,4558.61,2.234,✔️,0.0,False,GPU -9,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1525.0,+209/-141,4.547,0.537,16.05,8.57,7.67,0.547,✔️,0.0,False,GPU -10,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1514.0,+258/-178,4.137,0.525,16.57,6.01,6559.81,56.314,✔️,12.5,True,GPU -11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1498.0,+128/-64,6.384,0.346,17.42,11.6,4430.73,1.103,✔️,0.0,False,CPU -12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1476.0,+120/-73,7.235,0.319,18.5,14.95,1203.24,7.106,✔️,0.0,False,CPU -13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1476.0,+125/-64,6.518,0.31,18.52,13.37,4430.73,0.179,✔️,0.0,False,CPU -14,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1470.0,+177/-92,7.628,0.358,18.8,5.41,8977.36,8.017,✔️,0.0,False,GPU -15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1461.0,+187/-96,5.197,0.342,19.27,13.36,4438.37,1.981,✔️,0.0,False,GPU -16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1455.0,+118/-86,6.686,0.294,19.59,11.82,2039.04,2.144,✔️,0.0,False,GPU -17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1425.0,+95/-43,7.544,0.241,21.2,18.07,1265.82,3.15,✔️,0.0,False,CPU -18,🧠⚡,Foundation Model,[Mitra (default) [12.50% IMPUTED]](https://arxiv.org/abs/2510.21204),1423.0,+172/-124,6.857,0.315,21.3,6.84,93.39,2.396,✔️,12.5,True,GPU -19,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1419.0,+279/-232,18.567,0.478,21.52,6.2,6.44,0.709,➖,0.0,False,GPU -20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1401.0,+106/-79,8.008,0.221,22.5,17.92,1203.24,0.956,✔️,0.0,False,CPU -21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1399.0,+88/-44,7.708,0.204,22.65,18.94,1265.82,0.476,✔️,0.0,False,CPU -22,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1384.0,+125/-75,7.186,0.215,23.47,17.87,2039.04,0.109,✔️,0.0,False,GPU -23,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1380.0,+255/-186,5.862,0.342,23.69,10.92,6559.81,0.485,✔️,12.5,True,GPU -24,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1369.0,+174/-114,6.224,0.237,24.27,14.61,4438.37,0.231,✔️,0.0,False,GPU -25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1361.0,+141/-86,8.658,0.192,24.72,17.85,12.31,0.204,✔️,0.0,False,CPU -26,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1339.0,+167/-97,9.748,0.155,25.96,13.44,8977.36,0.421,✔️,0.0,False,GPU -27,🧠⚡,Foundation Model,[TabPFNv2 (default) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1308.0,+225/-226,8.136,0.248,27.73,15.18,8.74,0.773,✔️,12.5,True,GPU -28,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1304.0,+171/-124,7.848,0.183,27.94,20.59,22.14,0.179,✔️,0.0,False,GPU -29,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1298.0,+135/-104,8.326,0.133,28.32,22.03,21.51,8.356,✔️,0.0,False,GPU -30,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1271.0,+153/-168,10.01,0.141,29.79,17.71,5150.31,1.592,✔️,0.0,False,CPU -31,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1270.0,+110/-149,12.545,0.149,29.84,19.99,664.2,1.363,✔️,0.0,False,CPU -32,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1240.0,+109/-141,12.829,0.118,31.48,18.81,664.2,0.274,✔️,0.0,False,CPU -33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1233.0,+75/-77,11.52,0.018,31.88,24.2,26.36,0.302,✔️,0.0,False,GPU -34,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1214.0,+152/-154,10.679,0.099,32.86,21.99,5150.31,0.136,✔️,0.0,False,CPU -35,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1213.0,+175/-164,12.343,0.079,32.94,22.94,4333.58,0.422,✔️,0.0,False,CPU -36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1208.0,+59/-44,10.665,0.0,33.18,31.5,4.3,0.318,✔️,0.0,False,CPU -37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1185.0,+95/-138,13.964,0.053,34.38,30.6,675.17,1.541,✔️,0.0,False,CPU -38,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1170.0,+178/-174,12.86,0.065,35.15,23.99,4333.58,0.037,✔️,0.0,False,CPU -39,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1143.0,+50/-59,11.888,0.0,36.48,35.38,4.06,0.299,✔️,0.0,False,CPU -40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1141.0,+88/-140,14.568,0.037,36.59,32.81,675.17,0.206,✔️,0.0,False,CPU -41,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1136.0,+107/-134,13.467,0.036,36.85,33.44,2214.9,7.795,✔️,0.0,False,CPU -42,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1112.0,+152/-226,14.305,0.048,37.98,26.64,10.2,2.718,✔️,0.0,False,GPU -43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1110.0,+96/-178,14.916,0.001,38.08,33.03,0.5,0.116,✔️,0.0,False,CPU -44,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1094.0,+148/-172,13.954,0.001,38.8,31.92,16.01,0.041,✔️,0.0,False,CPU -45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1077.0,+103/-140,13.971,0.02,39.57,37.02,2214.9,0.838,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1049.0,+116/-145,13.739,0.0,40.74,38.28,22.39,0.134,✔️,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+87/-150,16.312,0.0,42.69,40.63,0.63,0.12,✔️,0.0,False,CPU -48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),959.0,+106/-170,18.644,0.0,44.17,42.75,6.97,0.818,✔️,0.0,False,CPU -49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),954.0,+157/-376,20.561,0.0,44.34,35.99,88.43,0.512,✔️,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),853.0,+158/-384,22.9,0.0,47.34,41.05,88.43,0.051,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),762.0,+178/-559,29.906,0.0,49.41,38.18,0.42,0.047,✔️,0.0,False,CPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),594.0,+118/-414,34.296,0.0,52.04,51.78,777.57,0.312,✔️,0.0,False,CPU -53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),579.0,+128/-434,34.339,0.0,52.23,51.95,777.57,0.108,✔️,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),396.0,+82/-395,36.314,0.0,54.0,53.97,3.91,0.126,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1728.0,+140/-56,0.775,7.93,3.37,3.194,4822.0,140.81,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1717.0,+270/-132,0.761,8.26,2.67,1.767,2306.87,7.231,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1717.0,+210/-85,0.703,8.26,4.45,2.652,709.58,5.864,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1684.0,+107/-44,0.711,9.41,4.91,3.668,4822.0,22.324,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1661.0,+122/-49,0.652,10.24,5.98,3.611,4558.61,33.853,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1590.0,+114/-50,0.578,13.2,5.14,4.794,48.13,42.001,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1581.0,+222/-130,0.608,13.6,4.66,2.696,2306.87,0.902,✔️,0.0,False,GPU +7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1573.0,+133/-73,0.467,13.96,8.58,5.563,5805.42,4.31,✔️,0.0,False,CPU +8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1564.0,+117/-72,0.469,14.37,8.46,4.554,4558.61,2.234,✔️,0.0,False,GPU +9,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1522.0,+208/-141,0.53,16.41,8.71,4.566,7.67,0.547,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1513.0,+250/-173,0.514,16.85,6.19,4.156,6559.81,56.314,✔️,12.5,True,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1496.0,+125/-63,0.336,17.75,11.75,6.404,4430.73,1.103,✔️,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1475.0,+124/-64,0.298,18.86,13.54,6.537,4430.73,0.179,✔️,0.0,False,CPU +13,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1473.0,+123/-74,0.311,18.96,15.26,7.255,1203.24,7.106,✔️,0.0,False,CPU +14,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1467.0,+179/-93,0.353,19.27,5.5,7.648,8977.36,8.017,✔️,0.0,False,GPU +15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1460.0,+186/-97,0.336,19.64,13.49,5.216,4438.37,1.981,✔️,0.0,False,GPU +16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1454.0,+116/-87,0.288,19.96,12.32,6.705,2039.04,2.144,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[Mitra (default) [12.50% IMPUTED]](https://arxiv.org/abs/2510.21204),1423.0,+169/-123,0.313,21.67,7.05,6.876,93.39,2.396,✔️,12.5,True,GPU +18,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1423.0,+93/-44,0.228,21.68,18.47,7.563,1265.82,3.15,✔️,0.0,False,CPU +19,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1419.0,+276/-229,0.472,21.9,6.29,18.586,6.44,0.709,➖,0.0,False,GPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1400.0,+110/-79,0.209,22.98,18.17,8.027,1203.24,0.956,✔️,0.0,False,CPU +21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1397.0,+87/-45,0.191,23.11,19.31,7.727,1265.82,0.476,✔️,0.0,False,CPU +22,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1384.0,+125/-76,0.213,23.87,18.28,7.205,2039.04,0.109,✔️,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1378.0,+255/-182,0.337,24.19,11.08,5.881,6559.81,0.485,✔️,12.5,True,GPU +24,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1375.0,+146/-152,0.281,24.4,9.83,8.711,22.46,1.039,➖,0.0,False,GPU +25,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1368.0,+174/-116,0.232,24.76,14.74,6.243,4438.37,0.231,✔️,0.0,False,GPU +26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1359.0,+139/-85,0.18,25.29,18.14,8.677,12.31,0.204,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1338.0,+166/-99,0.154,26.53,13.56,9.766,8977.36,0.421,✔️,0.0,False,GPU +28,🧠⚡,Foundation Model,[TabPFNv2 (default) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1308.0,+226/-218,0.245,28.28,15.67,8.156,8.74,0.773,✔️,12.5,True,GPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1305.0,+169/-124,0.179,28.45,20.83,7.866,22.14,0.179,✔️,0.0,False,GPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1297.0,+136/-106,0.133,28.92,22.71,8.344,21.51,8.356,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1270.0,+151/-171,0.135,30.43,17.91,10.029,5150.31,1.592,✔️,0.0,False,CPU +32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1269.0,+110/-148,0.143,30.51,20.55,12.564,664.2,1.363,✔️,0.0,False,CPU +33,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1239.0,+109/-139,0.112,32.17,19.55,12.848,664.2,0.274,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1231.0,+77/-79,0.017,32.6,25.2,11.539,26.36,0.302,✔️,0.0,False,GPU +35,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1214.0,+148/-158,0.094,33.51,22.23,10.697,5150.31,0.136,✔️,0.0,False,CPU +36,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1213.0,+173/-163,0.075,33.58,23.46,12.362,4333.58,0.422,✔️,0.0,False,CPU +37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1206.0,+60/-46,0.001,33.94,32.19,10.683,4.3,0.318,✔️,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1184.0,+94/-138,0.047,35.16,31.43,13.984,675.17,1.541,✔️,0.0,False,CPU +39,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1170.0,+176/-170,0.061,35.85,24.38,12.878,4333.58,0.037,✔️,0.0,False,CPU +40,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1142.0,+52/-62,0.0,37.27,36.13,11.907,4.06,0.299,✔️,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1140.0,+84/-138,0.032,37.39,33.58,14.587,675.17,0.206,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1135.0,+108/-138,0.037,37.65,34.03,13.486,2214.9,7.795,✔️,0.0,False,CPU +43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1112.0,+152/-222,0.046,38.72,28.03,14.324,10.2,2.718,✔️,0.0,False,GPU +44,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1109.0,+93/-172,0.001,38.87,34.05,14.936,0.5,0.116,✔️,0.0,False,CPU +45,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1093.0,+146/-167,0.0,39.63,32.6,13.973,16.01,0.041,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1076.0,+106/-144,0.021,40.39,37.67,13.99,2214.9,0.838,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1049.0,+115/-148,0.0,41.56,39.03,13.757,22.39,0.134,✔️,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+84/-154,0.0,43.55,41.58,16.331,0.63,0.12,✔️,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),959.0,+111/-175,0.0,45.04,43.52,18.662,6.97,0.818,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),953.0,+157/-379,0.0,45.27,36.81,20.58,88.43,0.512,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),851.0,+156/-381,0.0,48.33,42.1,22.918,88.43,0.051,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),760.0,+178/-556,0.0,50.4,38.95,29.924,0.42,0.047,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),592.0,+116/-411,0.0,53.04,52.78,34.313,777.57,0.312,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),577.0,+127/-431,0.0,53.22,52.94,34.357,777.57,0.108,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),394.0,+81/-396,0.0,55.0,54.97,36.331,3.91,0.126,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_small/winrate_matrix.png.zip index c1c495f1598d606e171e60deaa945acc87b0838b..7c8146bc788a8fc7bb8495565040361c4739a4ab 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5e931d4fe75c863b57de71fc1f359d7180afd7cdf73dae4c32e09cfcce06a3ca -size 2146599 +oid sha256:f30048be393be3cb9bc3a9cdcf953c102ab2cfd2390be68426c8d3689930954c +size 2336382 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index e67b660ce9cf22f099ce685707d1676289a92411..c9012656a8ff4b9b402002f2200a5e81392e11d8 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:056f72436b3a73b9395b9b2829f949ad0cd9b6deef88bcd3986cc39a2af6fc8b -size 453334 +oid sha256:75bacac0a9fff9888369d04d07a1f03b30860a86abc9ea567f89ad8d5e0c1aa8 +size 467971 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip index 7a72c2f4ba8d52b97bfe90c17badb6036b752414..b6b1c510e1ed8fae293fdae27f974762bef73c66 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0ca42b7b7f8ec97378c7127070aaff875caa06928d35dd0ab869b99eac501266 -size 1019357 +oid sha256:fa2539a16c0fd51e98693c16759b6c7accfdf7f7b504d9a32df6b7df22253f6d +size 1032206 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip index 7c25e1cd2b77e54422cfa6905c3df2e2abc8502c..bb64b5caf13f289c7624adf483cb4a2c886ded77 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bae81847e8d874f7724ee14d97aea388bece2409711ab183af81de997c4385f7 -size 119094 +oid sha256:59ece146cb205dd25b7d9af3c3290427fbebec3e5f0a0026df4f863ebb6155f5 +size 123745 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/website_leaderboard.csv index 0e833c45528f5d3f913975bf8a03140d92d16394..b9ed4ea397cbb4f9d924fb29633ff04017be57a4 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/website_leaderboard.csv @@ -1,56 +1,57 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1738.0,+227/-92,2.676,0.724,8.07,4.14,669.92,5.859,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1729.0,+168/-38,3.492,0.767,8.36,3.52,4663.16,132.911,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1723.0,+332/-141,1.797,0.764,8.56,2.6,2166.18,6.341,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1699.0,+128/-34,3.903,0.721,9.38,4.73,4663.16,18.295,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1682.0,+131/-57,3.736,0.676,10.01,5.56,4823.88,33.539,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1626.0,+176/-56,4.903,0.627,12.17,4.57,47.35,39.185,✔️,0.0,False,GPU -6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1607.0,+195/-116,3.411,0.601,13.01,5.36,6577.99,56.795,✔️,0.0,False,GPU -7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1592.0,+287/-164,2.725,0.619,13.66,4.39,2166.18,0.812,✔️,0.0,False,GPU -8,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1568.0,+116/-54,6.052,0.451,14.73,9.53,6530.9,4.363,✔️,0.0,False,CPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1565.0,+155/-64,4.84,0.462,14.89,8.5,4823.88,1.952,✔️,0.0,False,GPU -10,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1557.0,+236/-169,4.606,0.583,15.25,7.94,7.15,0.501,✔️,0.0,False,GPU -11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1540.0,+116/-42,6.578,0.392,16.08,10.73,3558.14,0.966,✔️,0.0,False,CPU -12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1520.0,+129/-52,6.697,0.354,17.06,12.43,3558.14,0.105,✔️,0.0,False,CPU -13,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1503.0,+250/-242,19.511,0.546,17.88,5.52,5.94,0.652,➖,0.0,False,GPU -14,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1492.0,+191/-91,6.52,0.36,18.42,6.11,99.14,2.48,✔️,0.0,False,GPU -15,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1476.0,+131/-107,7.027,0.309,19.23,11.16,2045.39,1.388,✔️,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1475.0,+112/-68,7.822,0.303,19.3,15.45,1003.11,4.891,✔️,0.0,False,CPU -17,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1459.0,+217/-133,5.499,0.328,20.15,13.53,4158.29,1.946,✔️,0.0,False,GPU -18,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1440.0,+231/-181,5.382,0.391,21.16,9.88,6577.99,0.498,✔️,0.0,False,GPU -19,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1437.0,+138/-67,8.679,0.266,21.28,11.17,9308.22,7.885,✔️,0.0,False,GPU -20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1428.0,+88/-37,8.1,0.229,21.77,18.45,933.42,2.759,✔️,0.0,False,CPU -21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1412.0,+127/-98,7.406,0.246,22.61,16.94,2045.39,0.104,✔️,0.0,False,GPU -22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1401.0,+91/-46,8.265,0.192,23.23,19.21,933.42,0.388,✔️,0.0,False,CPU -23,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1399.0,+114/-70,8.659,0.201,23.35,18.34,1003.11,0.894,✔️,0.0,False,CPU -24,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1385.0,+140/-87,9.129,0.219,24.07,16.92,10.91,0.197,✔️,0.0,False,CPU -25,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1362.0,+218/-156,6.622,0.219,25.36,14.47,4158.29,0.208,✔️,0.0,False,GPU -26,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1354.0,+232/-206,7.982,0.284,25.77,13.92,9.1,0.85,✔️,0.0,False,GPU -27,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1299.0,+96/-60,10.902,0.073,28.81,20.91,9308.22,0.416,✔️,0.0,False,GPU -28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1293.0,+194/-163,13.262,0.171,29.15,18.85,623.85,1.144,✔️,0.0,False,CPU -29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1292.0,+215/-168,8.427,0.168,29.2,20.9,15.01,0.154,✔️,0.0,False,GPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1287.0,+170/-106,8.943,0.118,29.44,22.47,23.19,8.523,✔️,0.0,False,GPU -31,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1272.0,+170/-162,13.412,0.135,30.3,17.5,623.85,0.291,✔️,0.0,False,CPU -32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1250.0,+149/-168,12.707,0.091,31.47,21.51,2929.85,0.41,✔️,0.0,False,CPU -33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1228.0,+90/-90,12.445,0.015,32.61,24.17,27.82,0.299,✔️,0.0,False,GPU -34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1219.0,+163/-152,11.15,0.07,33.04,27.68,5692.02,1.354,✔️,0.0,False,CPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1204.0,+166/-180,13.286,0.074,33.82,22.52,2929.85,0.037,✔️,0.0,False,CPU -36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1202.0,+47/-44,11.45,0.0,33.94,32.18,4.57,0.275,✔️,0.0,False,CPU -37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1198.0,+126/-178,14.807,0.06,34.14,29.95,633.57,1.512,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1187.0,+188/-175,11.641,0.077,34.7,22.5,5692.02,0.131,✔️,0.0,False,CPU -39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1161.0,+120/-186,15.352,0.042,35.98,31.92,633.57,0.248,✔️,0.0,False,CPU -40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1142.0,+154/-178,14.894,0.055,36.87,25.15,10.27,3.409,✔️,0.0,False,GPU -41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1142.0,+46/-60,12.598,0.0,36.87,35.63,3.79,0.311,✔️,0.0,False,CPU -42,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1131.0,+130/-155,14.18,0.001,37.4,30.44,11.19,0.038,✔️,0.0,False,CPU -43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1128.0,+134/-196,15.73,0.002,37.53,32.07,0.47,0.117,✔️,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1085.0,+62/-94,14.849,0.0,39.46,38.31,2248.07,7.53,✔️,0.0,False,CPU -45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1025.0,+68/-114,15.336,0.0,41.94,41.02,2248.07,0.89,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1013.0,+116/-188,14.924,0.0,42.38,40.15,24.31,0.134,✔️,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+109/-207,17.325,0.0,42.87,40.52,0.53,0.118,✔️,0.0,False,CPU -48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),982.0,+155/-419,21.384,0.0,43.54,34.61,84.3,0.438,✔️,0.0,False,CPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),902.0,+53/-155,20.488,0.0,46.1,45.46,7.09,0.843,✔️,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),877.0,+150/-428,23.664,0.0,46.82,39.94,84.3,0.05,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),801.0,+167/-577,30.315,0.0,48.7,36.63,0.39,0.044,✔️,0.0,False,CPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),607.0,+111/-430,36.055,0.0,52.0,51.7,779.18,0.247,✔️,0.0,False,CPU -53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),594.0,+126/-470,36.078,0.0,52.17,51.85,779.18,0.089,✔️,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),421.0,+89/-422,37.463,0.0,53.9,53.88,3.97,0.122,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1739.0,+227/-91,0.719,8.27,4.19,2.698,669.92,5.859,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1729.0,+168/-39,0.762,8.57,3.59,3.514,4663.16,132.911,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1723.0,+330/-141,0.759,8.8,2.61,1.819,2166.18,6.341,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1700.0,+126/-35,0.714,9.59,4.88,3.925,4663.16,18.295,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1683.0,+128/-58,0.668,10.23,5.68,3.758,4823.88,33.539,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1629.0,+173/-55,0.621,12.42,4.68,4.925,47.35,39.185,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1608.0,+192/-116,0.588,13.34,5.52,3.433,6577.99,56.795,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1592.0,+290/-166,0.611,14.05,4.42,2.746,2166.18,0.812,✔️,0.0,False,GPU +8,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1570.0,+118/-56,0.44,15.05,9.66,6.073,6530.9,4.363,✔️,0.0,False,CPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1567.0,+155/-64,0.453,15.2,8.65,4.861,4823.88,1.952,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1558.0,+240/-170,0.574,15.66,8.09,4.628,7.15,0.501,✔️,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1541.0,+117/-41,0.378,16.46,10.88,6.6,3558.14,0.966,✔️,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1522.0,+130/-54,0.34,17.44,12.59,6.719,3558.14,0.105,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1506.0,+244/-239,0.539,18.25,5.6,19.533,5.94,0.652,➖,0.0,False,GPU +14,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),1495.0,+192/-92,0.358,18.84,6.31,6.542,99.14,2.48,✔️,0.0,False,GPU +15,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1479.0,+131/-107,0.301,19.65,11.67,7.049,2045.39,1.388,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1476.0,+114/-68,0.293,19.83,15.82,7.844,1003.11,4.891,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1462.0,+214/-134,0.321,20.58,13.69,5.521,4158.29,1.946,✔️,0.0,False,GPU +18,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1456.0,+100/-90,0.321,20.87,8.82,8.181,22.35,1.037,➖,0.0,False,GPU +19,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1440.0,+229/-182,0.385,21.72,10.03,5.404,6577.99,0.498,✔️,0.0,False,GPU +20,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1439.0,+137/-69,0.26,21.82,11.61,8.701,9308.22,7.885,✔️,0.0,False,GPU +21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1430.0,+85/-37,0.214,22.31,18.92,8.121,933.42,2.759,✔️,0.0,False,CPU +22,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1416.0,+125/-102,0.243,23.08,17.37,7.428,2045.39,0.104,✔️,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1403.0,+91/-46,0.176,23.76,19.64,8.287,933.42,0.388,✔️,0.0,False,CPU +24,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1401.0,+111/-70,0.187,23.9,18.64,8.681,1003.11,0.894,✔️,0.0,False,CPU +25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1386.0,+140/-88,0.206,24.73,17.21,9.151,10.91,0.197,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1365.0,+215/-157,0.213,25.91,14.61,6.643,4158.29,0.208,✔️,0.0,False,GPU +27,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1356.0,+232/-206,0.28,26.4,14.39,8.004,9.1,0.85,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1302.0,+96/-64,0.072,29.46,21.24,10.923,9308.22,0.416,✔️,0.0,False,GPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1296.0,+216/-171,0.162,29.79,21.18,8.448,15.01,0.154,✔️,0.0,False,GPU +30,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1294.0,+195/-165,0.164,29.92,19.42,13.284,623.85,1.144,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1290.0,+172/-110,0.117,30.13,23.29,8.964,23.19,8.523,✔️,0.0,False,GPU +32,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1272.0,+171/-164,0.128,31.09,18.24,13.434,623.85,0.291,✔️,0.0,False,CPU +33,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1252.0,+148/-168,0.086,32.21,22.04,12.728,2929.85,0.41,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1229.0,+91/-91,0.014,33.43,25.31,12.467,27.82,0.299,✔️,0.0,False,GPU +35,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1222.0,+164/-151,0.062,33.78,28.25,11.172,5692.02,1.354,✔️,0.0,False,CPU +36,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1206.0,+166/-180,0.07,34.62,22.92,13.308,2929.85,0.037,✔️,0.0,False,CPU +37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1202.0,+45/-45,0.0,34.8,33.0,11.471,4.57,0.275,✔️,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1198.0,+126/-179,0.053,35.02,30.86,14.829,633.57,1.512,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1190.0,+188/-176,0.07,35.44,22.79,11.662,5692.02,0.131,✔️,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1161.0,+121/-186,0.036,36.89,32.75,15.374,633.57,0.248,✔️,0.0,False,CPU +41,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1144.0,+154/-182,0.053,37.71,26.57,14.916,10.27,3.409,✔️,0.0,False,GPU +42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1143.0,+46/-61,0.0,37.77,36.5,12.62,3.79,0.311,✔️,0.0,False,CPU +43,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1132.0,+130/-156,0.0,38.27,31.11,14.201,11.19,0.038,✔️,0.0,False,CPU +44,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1129.0,+133/-199,0.001,38.43,33.18,15.752,0.47,0.117,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1086.0,+62/-96,0.0,40.38,39.21,14.87,2248.07,7.53,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1026.0,+70/-116,0.0,42.87,41.95,15.356,2248.07,0.89,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1014.0,+116/-188,0.0,43.33,41.08,14.945,24.31,0.134,✔️,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+109/-209,0.0,43.85,41.61,17.347,0.53,0.118,✔️,0.0,False,CPU +49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),983.0,+154/-421,0.0,44.47,35.41,21.405,84.3,0.438,✔️,0.0,False,CPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),903.0,+53/-156,0.0,47.09,46.46,20.509,7.09,0.843,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),878.0,+150/-428,0.0,47.8,40.99,23.685,84.3,0.05,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),802.0,+167/-576,0.0,49.69,37.37,30.336,0.39,0.044,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),608.0,+112/-431,0.0,53.0,52.7,36.075,779.18,0.247,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),594.0,+126/-471,0.0,53.16,52.84,36.098,779.18,0.089,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),422.0,+88/-423,0.0,54.9,54.88,37.483,3.97,0.122,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip index f7a171a32be4692d56904c1ed060e1cacf197bb5..38c3cc8dfd4ae56e3a7ac065a3eb4d1c6fca9099 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:81bebdd1a18ec60bb8b20509e0cead272022d5378d9b9e0939df7bae4104e9d2 -size 2122484 +oid sha256:6e45f069ac50d0eeda7d3db00a68e25eaa5fe9722778b8889c3f1a508b78159b +size 2285400 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index 3bcb78758a246a4eea9f9e134d9907232877699d..74d4abaf1506cb5d34702401077cbe3930ee5177 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6ae087668d285948e91705c8f30f2d1f99db8a486ee593da20b787f26be1b8aa -size 476815 +oid sha256:486bf62d10b82d0560858d4d87b2a77bb50c7f37f3e319eeddebbcedb8adf76c +size 490549 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_all/pareto_n_configs_imp.png.zip index fd0f9973e548a7dea8e1b91455614589bc9ef600..7db0b29da59935fd83eae90e48feb133a7285d0d 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:214b2abeaa8f6279a86d4b2088d37cbd84b376fe8555c0c0921a6806acd78e3e -size 1070652 +oid sha256:37177d3e85f516a28031b955a94b792a66b7c9182f76de2f66b68c1b7a1f9b58 +size 1037777 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_all/tuning-impact-elo.png.zip index c0a9ccfd00d322d632c269a4109cd50feae45984..acee58b51eddee93d330c58f7f6df1828f5a00ac 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:229a8f768f80a5b65881b401e85db03d6af5afd1229eccc8430009d0c042cc92 -size 165567 +oid sha256:bce51e5ff583347b6c9a93dcd57c6a6416a60b795cda330dc287e311fab0fbfe +size 168252 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_all/datasets_all/website_leaderboard.csv index d2e2c4a37ed3786c6e8ca2b43bb8b3214136ae63..0fc8d3543923db337dff91386e2d00fcf394d95d 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_all/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_all/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1628.0,+99/-65,3.917,0.679,8.55,2.79,2059.94,9.785,✔️,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1604.0,+80/-74,6.026,0.651,9.45,4.05,556.15,6.31,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1576.0,+84/-62,6.034,0.605,10.57,4.51,2059.94,1.03,✔️,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1543.0,+67/-62,7.8,0.556,12.0,4.98,1754.94,1.767,✔️,0.0,False,CPU -4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1539.0,+70/-51,6.68,0.559,12.17,6.36,5.71,0.611,✔️,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1507.0,+58/-50,8.918,0.496,13.71,8.01,2791.97,13.886,✔️,0.0,False,GPU -6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1436.0,+68/-58,9.317,0.449,17.55,5.1,6154.73,386.167,✔️,0.0,False,GPU -7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1432.0,+54/-55,10.444,0.401,17.77,9.48,2791.97,0.373,✔️,0.0,False,GPU -8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1411.0,+66/-44,10.905,0.365,19.02,9.32,3133.91,1.273,✔️,0.0,False,GPU -9,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1410.0,+39/-40,11.851,0.317,19.03,14.04,416.56,2.236,✔️,0.0,False,CPU -10,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1391.0,+75/-59,10.959,0.39,20.23,6.99,6154.73,39.452,✔️,0.0,False,GPU -11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1390.0,+58/-44,11.444,0.32,20.26,12.91,1665.53,0.559,✔️,0.0,False,CPU -12,🧠⚡,Foundation Model,[LimiX (default) [11.76% IMPUTED]](https://arxiv.org/abs/2509.03505),1389.0,+73/-61,12.972,0.396,20.35,6.09,3.88,0.594,➖,11.76,True,GPU -13,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1383.0,+82/-60,11.741,0.378,20.73,8.09,4618.5,7.737,✔️,0.0,False,GPU -14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1366.0,+52/-46,11.867,0.296,21.79,13.57,1665.53,0.065,✔️,0.0,False,CPU -15,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1363.0,+39/-50,12.481,0.272,21.93,14.17,700.96,1.439,✔️,0.0,False,CPU -16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1350.0,+51/-47,12.755,0.258,22.76,17.23,416.56,0.381,✔️,0.0,False,CPU -17,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1347.0,+60/-58,12.351,0.298,22.96,11.99,4618.5,0.47,✔️,0.0,False,GPU -18,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1346.0,+62/-53,11.839,0.289,23.03,13.65,3133.91,0.13,✔️,0.0,False,GPU -19,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1344.0,+43/-45,12.455,0.256,23.18,13.07,6.7,0.088,✔️,0.0,False,CPU -20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1342.0,+45/-46,12.662,0.251,23.29,13.19,700.96,0.213,✔️,0.0,False,CPU -21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1340.0,+60/-46,12.633,0.279,23.39,14.52,866.11,2.007,✔️,0.0,False,GPU -22,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1334.0,+75/-72,12.812,0.334,23.77,7.85,2942.08,17.372,✔️,35.29,True,GPU -23,🧠⚡,Foundation Model,[Mitra (default) [35.29% IMPUTED]](https://arxiv.org/abs/2510.21204),1310.0,+56/-67,13.616,0.281,25.35,10.45,87.34,2.433,✔️,35.29,True,GPU -24,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1291.0,+56/-45,14.011,0.202,26.64,14.41,866.11,0.097,✔️,0.0,False,GPU -25,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1288.0,+71/-72,13.65,0.284,26.81,9.54,45.42,39.406,✔️,0.0,False,GPU -26,🧠⚡,Foundation Model,[TabICL (default) [29.41% IMPUTED]](https://arxiv.org/abs/2502.05564),1284.0,+60/-55,13.464,0.248,27.1,9.5,6.86,1.52,✔️,29.41,True,GPU -27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1283.0,+51/-50,14.241,0.221,27.14,18.82,11.56,0.127,✔️,0.0,False,GPU -28,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1268.0,+70/-58,14.474,0.231,28.09,14.28,2942.08,0.262,✔️,35.29,True,GPU -29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1266.0,+51/-55,15.201,0.186,28.25,17.26,2961.52,0.482,✔️,0.0,False,CPU -30,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1257.0,+46/-56,13.981,0.151,28.84,22.85,2832.8,1.801,✔️,0.0,False,CPU -31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1256.0,+43/-48,14.615,0.136,28.9,20.14,10.44,1.714,✔️,0.0,False,GPU -32,🧠⚡,Foundation Model,[BetaTabPFN (default) [25.49% IMPUTED]](https://arxiv.org/abs/2502.02527),1243.0,+68/-56,15.841,0.191,29.75,17.81,202.99,1.155,➖,25.49,True,GPU -33,🧠⚡,Foundation Model,[TabPFNv2 (default) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1232.0,+67/-79,15.328,0.21,30.48,12.04,3.27,0.315,✔️,35.29,True,GPU -34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1218.0,+62/-50,16.847,0.132,31.4,16.55,13.74,0.316,✔️,0.0,False,GPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1217.0,+60/-59,16.032,0.141,31.47,16.17,2961.52,0.048,✔️,0.0,False,CPU -36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1205.0,+56/-60,17.265,0.116,32.22,23.68,191.44,0.76,✔️,0.0,False,CPU -37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1193.0,+55/-58,16.984,0.124,33.02,17.8,7.66,0.046,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1189.0,+48/-60,15.841,0.117,33.25,25.96,2832.8,0.112,✔️,0.0,False,CPU -39,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1186.0,+56/-52,15.887,0.114,33.47,22.61,2.06,0.122,✔️,0.0,False,CPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1176.0,+63/-66,17.624,0.11,34.11,24.29,594.95,4.651,✔️,0.0,False,CPU -41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1172.0,+64/-62,18.313,0.117,34.32,20.6,191.44,0.101,✔️,0.0,False,CPU -42,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1166.0,+56/-62,18.204,0.085,34.75,26.14,377.08,0.747,✔️,0.0,False,CPU -43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1154.0,+51/-47,16.682,0.081,35.44,30.91,2.2,0.171,✔️,0.0,False,CPU -44,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1122.0,+47/-47,19.017,0.049,37.4,31.35,377.08,0.091,✔️,0.0,False,CPU -45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1103.0,+59/-77,19.244,0.07,38.51,27.59,594.95,0.337,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1032.0,+55/-68,21.088,0.022,42.41,37.99,8.96,0.129,✔️,0.0,False,CPU -47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1032.0,+71/-72,23.862,0.057,42.43,27.72,3.14,0.741,✔️,0.0,False,GPU -48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+61/-56,23.881,0.014,44.01,36.11,0.43,0.053,✔️,0.0,False,CPU -49,🧠⚡,Foundation Model,[TabFlex (default) [25.49% IMPUTED]](https://arxiv.org/abs/2506.05584),979.0,+77/-88,26.116,0.035,45.0,37.8,0.8,0.119,➖,25.49,True,GPU -50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),970.0,+73/-77,25.472,0.019,45.39,41.48,0.26,0.054,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),969.0,+63/-82,25.723,0.024,45.47,40.12,129.1,1.627,✔️,0.0,False,CPU -52,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),966.0,+67/-78,23.185,0.022,45.57,42.03,3.12,0.312,✔️,0.0,False,CPU -53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),898.0,+81/-109,32.354,0.024,48.35,24.73,240.73,0.308,✔️,0.0,False,CPU -54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),868.0,+82/-109,32.94,0.017,49.46,33.4,240.73,0.068,✔️,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),805.0,+91/-127,35.498,0.008,51.41,48.35,1.23,0.115,✔️,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),804.0,+69/-92,31.855,0.011,51.43,49.38,129.1,0.103,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),589.0,+81/-131,44.446,0.0,55.84,55.51,0.19,0.037,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1626.0,+97/-66,0.68,8.67,2.79,3.917,2059.94,9.785,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1600.0,+80/-74,0.65,9.65,4.07,6.026,556.15,6.31,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1573.0,+82/-61,0.607,10.73,4.55,6.034,2059.94,1.03,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1539.0,+67/-62,0.556,12.24,5.02,7.8,1754.94,1.767,✔️,0.0,False,CPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1537.0,+69/-50,0.563,12.32,6.53,6.68,5.71,0.611,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1505.0,+58/-50,0.497,13.9,8.13,8.918,2791.97,13.886,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1434.0,+66/-58,0.452,17.82,5.12,9.317,6154.73,386.167,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1429.0,+52/-55,0.401,18.09,9.84,10.444,2791.97,0.373,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1408.0,+67/-42,0.364,19.33,9.42,10.905,3133.91,1.273,✔️,0.0,False,GPU +9,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1408.0,+39/-40,0.318,19.36,14.15,11.851,416.56,2.236,✔️,0.0,False,CPU +10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1389.0,+58/-44,0.32,20.54,13.3,11.444,1665.53,0.559,✔️,0.0,False,CPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1388.0,+74/-58,0.391,20.58,7.05,10.959,6154.73,39.452,✔️,0.0,False,GPU +12,🧠🔁,Neural Network,[LimiX (default) [11.76% IMPUTED]](https://arxiv.org/abs/2509.03505),1388.0,+72/-60,0.397,20.63,6.16,12.972,3.88,0.594,➖,11.76,True,GPU +13,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1380.0,+82/-61,0.378,21.1,8.11,11.741,4618.5,7.737,✔️,0.0,False,GPU +14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1364.0,+52/-46,0.296,22.13,13.67,11.867,1665.53,0.065,✔️,0.0,False,CPU +15,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1361.0,+38/-50,0.274,22.32,14.32,12.481,700.96,1.439,✔️,0.0,False,CPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1348.0,+50/-47,0.261,23.14,17.37,12.755,416.56,0.381,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1346.0,+61/-57,0.299,23.29,12.06,12.351,4618.5,0.47,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1344.0,+60/-54,0.291,23.42,13.8,11.839,3133.91,0.13,✔️,0.0,False,GPU +19,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1342.0,+44/-46,0.26,23.57,13.44,12.455,6.7,0.088,✔️,0.0,False,CPU +20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1340.0,+43/-46,0.252,23.69,13.35,12.662,700.96,0.213,✔️,0.0,False,CPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1338.0,+60/-45,0.28,23.8,14.67,12.633,866.11,2.007,✔️,0.0,False,GPU +22,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1332.0,+74/-71,0.333,24.19,7.95,12.812,2942.08,17.372,✔️,35.29,True,GPU +23,🧠🔁,Neural Network,[Mitra (default) [35.29% IMPUTED]](https://arxiv.org/abs/2510.21204),1308.0,+55/-66,0.278,25.82,10.58,13.616,87.34,2.433,✔️,35.29,True,GPU +24,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1290.0,+56/-45,0.203,27.03,14.56,14.011,866.11,0.097,✔️,0.0,False,GPU +25,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1286.0,+71/-72,0.286,27.28,9.59,13.65,45.42,39.406,✔️,0.0,False,GPU +26,🧠⚡,Foundation Model,[TabICL (default) [29.41% IMPUTED]](https://arxiv.org/abs/2502.05564),1282.0,+59/-55,0.249,27.53,9.56,13.464,6.86,1.52,✔️,29.41,True,GPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1281.0,+50/-51,0.224,27.59,19.02,14.241,11.56,0.127,✔️,0.0,False,GPU +28,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1267.0,+69/-57,0.23,28.57,14.59,14.474,2942.08,0.262,✔️,35.29,True,GPU +29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1265.0,+51/-54,0.188,28.67,17.42,15.201,2961.52,0.482,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1256.0,+46/-55,0.155,29.33,23.12,13.981,2832.8,1.801,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1254.0,+44/-49,0.137,29.41,20.58,14.615,10.44,1.714,✔️,0.0,False,GPU +32,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1247.0,+69/-62,0.206,29.89,13.64,15.618,13.95,2.081,➖,0.0,False,GPU +33,🧠⚡,Foundation Model,[BetaTabPFN (default) [25.49% IMPUTED]](https://arxiv.org/abs/2502.02527),1242.0,+68/-56,0.194,30.22,18.05,15.841,202.99,1.155,➖,25.49,True,GPU +34,🧠⚡,Foundation Model,[TabPFNv2 (default) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1230.0,+65/-80,0.21,31.01,12.41,15.328,3.27,0.315,✔️,35.29,True,GPU +35,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1216.0,+62/-49,0.134,31.95,16.7,16.847,13.74,0.316,✔️,0.0,False,GPU +36,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1216.0,+60/-57,0.142,31.98,16.32,16.032,2961.52,0.048,✔️,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1205.0,+57/-60,0.116,32.73,23.94,17.265,191.44,0.76,✔️,0.0,False,CPU +38,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1192.0,+54/-59,0.126,33.59,18.17,16.984,7.66,0.046,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1188.0,+48/-60,0.121,33.8,26.36,15.841,2832.8,0.112,✔️,0.0,False,CPU +40,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1185.0,+57/-53,0.117,34.0,22.79,15.887,2.06,0.122,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1174.0,+62/-67,0.111,34.75,25.02,17.624,594.95,4.651,✔️,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1172.0,+63/-62,0.117,34.87,20.84,18.313,191.44,0.101,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1165.0,+56/-60,0.084,35.31,26.84,18.204,377.08,0.747,✔️,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1153.0,+52/-47,0.083,36.05,31.32,16.682,2.2,0.171,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1122.0,+47/-47,0.048,37.99,32.14,19.017,377.08,0.091,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1101.0,+59/-76,0.07,39.24,28.21,19.244,594.95,0.337,✔️,0.0,False,CPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1031.0,+70/-72,0.055,43.16,28.11,23.862,3.14,0.741,✔️,0.0,False,GPU +48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1031.0,+54/-68,0.022,43.16,38.61,21.088,8.96,0.129,✔️,0.0,False,CPU +49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+61/-56,0.014,44.75,37.94,23.881,0.43,0.053,✔️,0.0,False,CPU +50,🧠⚡,Foundation Model,[TabFlex (default) [25.49% IMPUTED]](https://arxiv.org/abs/2506.05584),978.0,+75/-87,0.035,45.82,38.62,26.116,0.8,0.119,➖,25.49,True,GPU +51,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),970.0,+73/-75,0.017,46.2,42.25,25.472,0.26,0.054,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),968.0,+63/-81,0.024,46.27,40.78,25.723,129.1,1.627,✔️,0.0,False,CPU +53,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),965.0,+66/-76,0.022,46.41,42.88,23.185,3.12,0.312,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),897.0,+80/-107,0.024,49.25,24.96,32.354,240.73,0.308,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),867.0,+81/-108,0.018,50.34,33.79,32.94,240.73,0.068,✔️,0.0,False,CPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),805.0,+92/-125,0.008,52.31,49.17,35.498,1.23,0.115,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),803.0,+67/-90,0.012,52.35,50.24,31.855,129.1,0.103,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),586.0,+80/-129,0.0,56.84,56.52,44.446,0.19,0.037,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_all/winrate_matrix.png.zip index cbb8f48b7d273cd9dd59249613a26ebe4fb187f8..f64597b49633fe22bd2db05f18a431e54cdcfe5c 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d63e8561929c0de79a37f81010df44932784b7cf6cd2a8680e8e27020ca461cf -size 2541132 +oid sha256:1d80d367e189cda9f9e30875b93d79a1051ad0b2e15a8c3e298e55dca06243a6 +size 2512231 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index 9d714972b8fd91a086f46990162723390e35d3c1..c23bffa947d829f95494c7bb7700107f315be416 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0866d69b3f69ef9c110afbf865021cac69fa77dcc9f120dc1abcfa6ab0abddcd -size 459061 +oid sha256:136619c712190497b9d48edea43be00a55ed434c54450c71f182f50897a71d16 +size 473675 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip index 959a07bc153cf66faac04b59ad8545427a9e7226..3b63d271d1534a8ccfeecba996f462b27e6dbe95 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:643a4b716f5ed89c6a4c50e2390695bef099d7560ad3ceb612ce72a643cc2e65 -size 991803 +oid sha256:1925e968d8e3054fa664496988b75349803a1bcb7cf4c5a8cfc84775e8720421 +size 978006 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/tuning-impact-elo.png.zip index dc01388bbdc17e1b83378ca81525b74405d5c321..9e60c3de3a0414599d8133954a355497fbe0ba82 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4f66de16edb07fa6e75a6a3d8af7d0dc7d2c9eedcfea349fdb818cbfe14fd8bd -size 138318 +oid sha256:eb3c9277fc26afe0f2582e7299cbfc17e025ac7ef23f76c7a4e6a48975779943 +size 141742 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/website_leaderboard.csv index 24a9dc4877f7820154e4b6de516f9844bfec4707..bd90c9fb3f023338490761a4906674f99f85dd8e 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/website_leaderboard.csv @@ -1,55 +1,56 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1884.0,+146/-62,2.12,0.804,4.67,2.45,432.14,4.214,✔️,0.0,False,CPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1841.0,+140/-68,2.589,0.774,5.53,2.88,302.05,2.868,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1732.0,+175/-116,3.925,0.666,8.4,3.18,739.1,11.65,✔️,0.0,False,GPU -3,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1689.0,+167/-130,4.338,0.642,9.8,4.58,2522.58,6.234,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1681.0,+158/-108,5.359,0.613,10.07,4.61,739.1,1.134,✔️,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1621.0,+99/-84,5.94,0.532,12.33,8.51,1714.29,1.743,✔️,0.0,False,GPU -6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1582.0,+118/-80,6.784,0.461,13.93,8.65,763.16,0.246,✔️,0.0,False,CPU -7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1582.0,+86/-60,7.733,0.436,13.93,10.4,129.25,2.621,✔️,0.0,False,CPU -8,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1555.0,+139/-115,7.879,0.441,15.13,11.23,2522.58,0.318,✔️,0.0,False,GPU -9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1554.0,+109/-69,6.913,0.436,15.17,8.23,763.16,0.04,✔️,0.0,False,CPU -10,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1549.0,+95/-57,7.467,0.393,15.4,10.77,289.35,0.706,✔️,0.0,False,CPU -11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1540.0,+150/-93,7.635,0.408,15.8,6.85,1993.57,0.581,✔️,0.0,False,GPU -12,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1529.0,+100/-84,8.359,0.364,16.33,12.67,129.25,0.117,✔️,0.0,False,CPU -13,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1522.0,+152/-115,6.533,0.415,16.67,8.21,1.91,0.668,✔️,0.0,False,GPU -14,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1514.0,+132/-117,7.967,0.404,17.0,10.11,1714.29,0.079,✔️,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1512.0,+109/-84,7.185,0.41,17.1,7.62,2.93,0.045,✔️,0.0,False,CPU -16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1495.0,+104/-81,7.701,0.355,17.93,13.2,289.35,0.08,✔️,0.0,False,CPU -17,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1458.0,+133/-120,8.104,0.353,19.73,5.46,4532.14,387.927,✔️,0.0,False,GPU -18,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1458.0,+144/-86,8.363,0.313,19.73,11.0,1993.57,0.06,✔️,0.0,False,GPU -19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1414.0,+112/-107,9.218,0.304,21.93,15.24,559.64,0.88,✔️,0.0,False,GPU -20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1394.0,+152/-142,10.694,0.303,22.97,8.77,4532.14,39.452,✔️,0.0,False,GPU -21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1373.0,+104/-94,11.567,0.204,24.07,18.29,559.64,0.051,✔️,0.0,False,GPU -22,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1372.0,+124/-124,9.557,0.241,24.13,13.15,0.48,0.047,✔️,0.0,False,CPU -23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1368.0,+120/-107,10.953,0.205,24.33,20.34,5.42,0.078,✔️,0.0,False,GPU -24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1335.0,+83/-66,10.076,0.133,26.07,21.96,1803.23,1.199,✔️,0.0,False,CPU -25,🧠⚡,Foundation Model,[TabICL (default) [33.33% IMPUTED]](https://arxiv.org/abs/2502.05564),1326.0,+149/-121,10.541,0.195,26.53,15.87,3.98,1.593,✔️,33.33,True,GPU -26,🧠⚡,Foundation Model,[LimiX (default) [40.00% IMPUTED]](https://arxiv.org/abs/2509.03505),1323.0,+159/-146,14.403,0.24,26.7,5.9,1.09,0.318,➖,40.0,True,GPU -27,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1319.0,+134/-147,13.157,0.219,26.93,18.11,1535.55,0.182,✔️,0.0,False,CPU -28,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1282.0,+80/-74,12.919,0.064,28.87,26.1,6.54,0.294,✔️,0.0,False,GPU -29,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1281.0,+124/-138,13.622,0.197,28.9,13.45,41.5,39.095,✔️,0.0,False,GPU -30,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1273.0,+94/-95,11.636,0.117,29.33,25.0,9.07,0.224,✔️,0.0,False,GPU -31,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1234.0,+97/-92,12.885,0.049,31.33,25.01,65.73,0.23,✔️,0.0,False,CPU -32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1231.0,+82/-95,12.561,0.065,31.47,28.31,1803.23,0.051,✔️,0.0,False,CPU -33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1224.0,+95/-118,12.624,0.055,31.8,28.99,36.73,0.359,✔️,0.0,False,CPU -34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1218.0,+78/-94,12.051,0.044,32.13,30.42,0.34,0.07,✔️,0.0,False,CPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1210.0,+134/-142,14.45,0.131,32.53,25.81,1535.55,0.013,✔️,0.0,False,CPU -36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1201.0,+89/-83,13.221,0.038,32.97,28.86,65.73,0.025,✔️,0.0,False,CPU -37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1200.0,+111/-142,15.546,0.116,33.0,28.81,2.99,0.016,✔️,0.0,False,CPU -38,🧠⚡,Foundation Model,[BetaTabPFN (default) [33.33% IMPUTED]](https://arxiv.org/abs/2502.02527),1181.0,+124/-115,14.165,0.078,33.93,24.31,66.06,0.61,➖,33.33,True,GPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1175.0,+114/-140,15.053,0.067,34.2,27.87,354.91,1.636,✔️,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1154.0,+89/-111,15.004,0.031,35.2,32.74,36.73,0.038,✔️,0.0,False,CPU -41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1078.0,+85/-91,16.371,0.008,38.67,37.11,2.58,0.039,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1042.0,+91/-111,17.618,0.012,40.13,38.37,354.91,0.094,✔️,0.0,False,CPU -43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1014.0,+159/-215,25.313,0.022,41.27,35.06,2.55,0.157,✔️,0.0,False,GPU -44,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+65/-106,18.863,0.0,41.8,40.8,0.24,0.022,✔️,0.0,False,CPU -45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),969.0,+92/-124,22.593,0.012,42.93,40.74,150.58,4.952,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),932.0,+109/-151,21.122,0.016,44.2,41.74,1.49,0.113,✔️,0.0,False,CPU -47,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),918.0,+111/-145,22.439,0.01,44.67,42.84,0.13,0.022,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),915.0,+167/-225,32.033,0.011,44.77,41.01,43.96,0.081,✔️,0.0,False,CPU -49,🧠⚡,Foundation Model,[TabFlex (default) [33.33% IMPUTED]](https://arxiv.org/abs/2506.05584),874.0,+95/-141,27.553,0.0,46.0,45.16,0.27,0.043,➖,33.33,True,GPU -50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),864.0,+152/-239,32.796,0.001,46.3,43.62,43.96,0.017,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),785.0,+59/-123,29.049,0.0,48.33,48.08,150.58,0.546,✔️,0.0,False,CPU -52,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),725.0,+107/-185,34.849,0.0,49.6,49.2,0.2,0.02,✔️,0.0,False,CPU -53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),536.0,+93/-196,42.696,0.0,52.33,52.25,0.1,0.068,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1878.0,+150/-64,0.805,4.73,2.46,2.12,432.14,4.214,✔️,0.0,False,CPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1836.0,+144/-71,0.776,5.6,2.89,2.589,302.05,2.868,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1728.0,+174/-119,0.668,8.47,3.18,3.925,739.1,11.65,✔️,0.0,False,GPU +3,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1686.0,+169/-132,0.644,9.87,4.58,4.338,2522.58,6.234,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1678.0,+160/-110,0.616,10.13,4.61,5.359,739.1,1.134,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1620.0,+96/-82,0.537,12.33,8.51,5.94,1714.29,1.743,✔️,0.0,False,GPU +6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1580.0,+116/-79,0.466,14.0,8.67,6.784,763.16,0.246,✔️,0.0,False,CPU +7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1578.0,+89/-63,0.44,14.07,10.43,7.733,129.25,2.621,✔️,0.0,False,CPU +8,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1552.0,+108/-70,0.441,15.23,8.24,6.913,763.16,0.04,✔️,0.0,False,CPU +9,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1551.0,+145/-113,0.445,15.27,11.31,7.879,2522.58,0.318,✔️,0.0,False,GPU +10,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1547.0,+96/-59,0.398,15.47,10.78,7.467,289.35,0.706,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1537.0,+147/-91,0.411,15.93,6.88,7.635,1993.57,0.581,✔️,0.0,False,GPU +12,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1526.0,+101/-85,0.368,16.47,12.7,8.359,129.25,0.117,✔️,0.0,False,CPU +13,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1520.0,+150/-114,0.421,16.73,8.21,6.533,1.91,0.668,✔️,0.0,False,GPU +14,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1511.0,+131/-117,0.409,17.13,10.12,7.967,1714.29,0.079,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1509.0,+111/-85,0.415,17.23,7.63,7.185,2.93,0.045,✔️,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1492.0,+105/-82,0.359,18.07,13.26,7.701,289.35,0.08,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1456.0,+145/-85,0.316,19.87,11.03,8.363,1993.57,0.06,✔️,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1455.0,+137/-118,0.359,19.93,5.47,8.104,4532.14,387.927,✔️,0.0,False,GPU +19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1410.0,+113/-104,0.305,22.2,15.32,9.218,559.64,0.88,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1391.0,+155/-142,0.306,23.23,8.78,10.694,4532.14,39.452,✔️,0.0,False,GPU +21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1372.0,+105/-93,0.206,24.2,18.36,11.567,559.64,0.051,✔️,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1370.0,+121/-124,0.242,24.33,13.17,9.557,0.48,0.047,✔️,0.0,False,CPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1365.0,+122/-107,0.208,24.6,20.47,10.953,5.42,0.078,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1335.0,+83/-64,0.138,26.2,22.09,10.076,1803.23,1.199,✔️,0.0,False,CPU +25,🧠⚡,Foundation Model,[TabICL (default) [33.33% IMPUTED]](https://arxiv.org/abs/2502.05564),1324.0,+147/-118,0.199,26.8,15.97,10.541,3.98,1.593,✔️,33.33,True,GPU +26,🧠🔁,Neural Network,[LimiX (default) [40.00% IMPUTED]](https://arxiv.org/abs/2509.03505),1321.0,+158/-147,0.242,26.97,5.9,14.403,1.09,0.318,➖,40.0,True,GPU +27,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1316.0,+134/-145,0.222,27.2,18.17,13.157,1535.55,0.182,✔️,0.0,False,CPU +28,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1279.0,+84/-75,0.071,29.2,26.28,12.919,6.54,0.294,✔️,0.0,False,GPU +29,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1277.0,+124/-139,0.197,29.3,13.53,13.622,41.5,39.095,✔️,0.0,False,GPU +30,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1269.0,+93/-95,0.123,29.73,25.4,11.636,9.07,0.224,✔️,0.0,False,GPU +31,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1234.0,+96/-90,0.05,31.6,25.18,12.885,65.73,0.23,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1231.0,+82/-94,0.069,31.73,28.47,12.561,1803.23,0.051,✔️,0.0,False,CPU +33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1222.0,+96/-117,0.058,32.2,29.23,12.624,36.73,0.359,✔️,0.0,False,CPU +34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1215.0,+81/-97,0.05,32.53,30.68,12.051,0.34,0.07,✔️,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1209.0,+133/-142,0.135,32.87,25.93,14.45,1535.55,0.013,✔️,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1202.0,+90/-83,0.039,33.23,29.01,13.221,65.73,0.025,✔️,0.0,False,CPU +37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1198.0,+111/-141,0.121,33.4,28.97,15.546,2.99,0.016,✔️,0.0,False,CPU +38,🧠⚡,Foundation Model,[BetaTabPFN (default) [33.33% IMPUTED]](https://arxiv.org/abs/2502.02527),1181.0,+122/-106,0.078,34.27,24.43,14.165,66.06,0.61,➖,33.33,True,GPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1173.0,+115/-141,0.068,34.67,28.06,15.053,354.91,1.636,✔️,0.0,False,CPU +40,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1154.0,+115/-114,0.059,35.6,29.57,15.308,11.42,10.384,➖,0.0,False,GPU +41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1151.0,+90/-113,0.033,35.73,33.1,15.004,36.73,0.038,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1075.0,+87/-90,0.009,39.27,37.57,16.371,2.58,0.039,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1038.0,+94/-110,0.013,40.87,38.99,17.618,354.91,0.094,✔️,0.0,False,CPU +44,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1012.0,+160/-213,0.024,41.93,35.77,25.313,2.55,0.157,✔️,0.0,False,GPU +45,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+64/-107,0.0,42.4,41.31,18.863,0.24,0.022,✔️,0.0,False,CPU +46,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),963.0,+92/-120,0.012,43.8,41.71,22.593,150.58,4.952,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),927.0,+109/-153,0.017,45.07,42.46,21.122,1.49,0.113,✔️,0.0,False,CPU +48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),915.0,+112/-144,0.01,45.47,43.5,22.439,0.13,0.022,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),910.0,+168/-224,0.013,45.63,41.73,32.033,43.96,0.081,✔️,0.0,False,CPU +50,🧠⚡,Foundation Model,[TabFlex (default) [33.33% IMPUTED]](https://arxiv.org/abs/2506.05584),872.0,+96/-147,0.0,46.8,45.9,27.553,0.27,0.043,➖,33.33,True,GPU +51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),860.0,+153/-235,0.002,47.17,44.36,32.796,43.96,0.017,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),777.0,+57/-122,0.0,49.33,49.09,29.049,150.58,0.546,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),718.0,+105/-186,0.0,50.6,50.21,34.849,0.2,0.02,✔️,0.0,False,CPU +54,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),528.0,+94/-197,0.0,53.33,53.25,42.696,0.1,0.068,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/winrate_matrix.png.zip index 28bf5b65594cfe68382863b29121d6364dbb7dcb..84ce6a17a7587f14ca344423823269ecb2ae10bc 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b98f79e24ae6a96b2cd4a1f13ced754a7876d720081264c4d403c1270aa2b66c -size 1860124 +oid sha256:0bd6a576c52893cedbfba55716ec435cc685bb7f8a0a32f6b25275e31e58f56c +size 1996866 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index cf0c659d962313d797ccb3049bdd0a908817eddb..9ca63a1993a548ae85a0056ca8e08f56f434edcf 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6c0fd6c09905f93da237c4aa3428f177ea2161db3bc42cee13bf7cfe09b80ab5 -size 471941 +oid sha256:1a6c33e86dfd947b548efd345269423e581f7c00b7ddeebc6435b1c42fa12423 +size 486299 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_small/pareto_n_configs_imp.png.zip index d117c16c8105a7f494c16c201a4de383df9c5b40..bae3aea5176c2c9e0b9b45be1a222c8921390409 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:49d4cff54a4cd572129d5a414784d45f474d93974f5b5ff7abded3a4fa4ccf02 -size 1060558 +oid sha256:4ff60ae856ae26280a9b39d343b6dec151ddf3e19993505645b23dfa96b5f7b2 +size 1035484 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_small/tuning-impact-elo.png.zip index 4bc3bf16b98e4e22d11955d48b9c23fa9ca92ea3..9393f9022fb8f550b1719f864385782971508c17 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e587becbfb7263378c73856bf63979e90018b5c56b4010d37fe62ead9339a3e4 -size 174670 +oid sha256:563f55e46c71803af466f73d4b4d128e814d9c2ad1851451451f99bc37b4a32e +size 174250 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_all/datasets_small/website_leaderboard.csv index 7eda4e21afb47688c8532fac603a5bc92e90d54c..b2eb13bdcad585fb57c1479046642b7686098501 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_small/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_small/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1628.0,+128/-83,3.914,0.678,8.61,2.65,2319.48,8.957,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1587.0,+101/-69,6.742,0.598,10.18,5.81,6.58,0.586,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1572.0,+108/-71,6.315,0.593,10.78,4.47,2319.48,0.942,✔️,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1565.0,+80/-67,7.459,0.591,11.08,4.88,657.21,7.989,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1500.0,+69/-57,10.159,0.466,14.17,7.81,3855.22,24.935,✔️,0.0,False,GPU -5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1482.0,+57/-52,10.167,0.444,15.06,8.72,3351.09,1.56,✔️,0.0,False,CPU -6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1475.0,+111/-81,10.288,0.474,15.44,5.86,3560.23,49.21,✔️,8.33,True,GPU -7,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1453.0,+94/-72,9.823,0.469,16.64,4.96,8059.77,364.288,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1440.0,+93/-72,12.376,0.454,17.38,6.16,5.11,0.624,➖,0.0,False,GPU -9,🧠⚡,Foundation Model,[Mitra (default) [8.33% IMPUTED]](https://arxiv.org/abs/2510.21204),1435.0,+89/-50,11.429,0.398,17.67,7.95,128.8,2.999,✔️,8.33,True,GPU -10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1432.0,+71/-72,11.477,0.384,17.88,9.22,3855.22,0.954,✔️,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1414.0,+89/-80,11.069,0.419,18.92,6.44,8059.77,39.456,✔️,0.0,False,GPU -12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1390.0,+70/-56,12.268,0.336,20.36,10.97,3629.74,1.48,✔️,0.0,False,GPU -13,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1377.0,+46/-43,13.566,0.251,21.15,16.43,833.03,2.182,✔️,0.0,False,CPU -14,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1371.0,+88/-72,12.643,0.327,21.56,11.17,3560.23,0.519,✔️,8.33,True,GPU -15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1350.0,+61/-58,13.386,0.244,22.9,16.25,2448.79,0.814,✔️,0.0,False,CPU -16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1334.0,+72/-63,14.055,0.259,23.89,14.22,1346.37,2.149,✔️,0.0,False,GPU -17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1328.0,+73/-72,13.287,0.265,24.29,15.16,3629.74,0.189,✔️,0.0,False,GPU -18,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1324.0,+63/-55,13.932,0.218,24.56,18.59,2448.79,0.078,✔️,0.0,False,CPU -19,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1322.0,+57/-45,14.57,0.202,24.65,16.31,886.51,1.982,✔️,0.0,False,CPU -20,🧠⚡,Foundation Model,[TabPFNv2 (default) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1318.0,+79/-65,13.855,0.298,24.93,9.27,4.98,0.511,✔️,8.33,True,GPU -21,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1313.0,+67/-71,14.826,0.26,25.28,11.87,6341.76,8.556,✔️,0.0,False,GPU -22,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1310.0,+52/-46,14.586,0.196,25.44,20.28,833.03,0.435,✔️,0.0,False,CPU -23,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1310.0,+85/-82,13.661,0.312,25.5,8.49,51.78,39.927,✔️,0.0,False,GPU -24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1309.0,+60/-52,14.729,0.188,25.53,13.18,886.51,0.318,✔️,0.0,False,CPU -25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1306.0,+50/-44,14.65,0.172,25.71,18.61,8.39,0.139,✔️,0.0,False,CPU -26,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1298.0,+64/-64,14.214,0.224,26.22,12.34,6341.76,0.501,✔️,0.0,False,GPU -27,🧠⚡,Foundation Model,[TabICL (default) [27.78% IMPUTED]](https://arxiv.org/abs/2502.05564),1286.0,+92/-100,14.682,0.256,27.03,8.13,7.56,1.494,✔️,27.78,True,GPU -28,🧠⚡,Foundation Model,[BetaTabPFN (default) [22.22% IMPUTED]](https://arxiv.org/abs/2502.02527),1281.0,+77/-78,16.539,0.225,27.36,15.93,269.12,1.881,➖,22.22,True,GPU -29,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1279.0,+65/-63,15.03,0.191,27.49,13.22,1346.37,0.103,✔️,0.0,False,GPU -30,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1270.0,+66/-61,15.611,0.211,28.08,18.21,13.71,0.149,✔️,0.0,False,GPU -31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1268.0,+68/-58,16.053,0.162,28.25,16.85,3249.24,0.548,✔️,0.0,False,CPU -32,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1262.0,+57/-60,15.321,0.149,28.58,18.33,16.6,4.299,✔️,0.0,False,GPU -33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1243.0,+51/-62,15.608,0.149,29.89,23.21,3693.32,2.14,✔️,0.0,False,CPU -34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1234.0,+65/-61,16.691,0.136,30.47,13.95,3249.24,0.059,✔️,0.0,False,CPU -35,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1208.0,+72/-71,19.018,0.119,32.15,14.49,14.9,0.323,✔️,0.0,False,GPU -36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1206.0,+79/-77,19.199,0.135,32.28,21.98,497.58,1.147,✔️,0.0,False,CPU -37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1203.0,+71/-60,17.583,0.113,32.47,15.3,9.93,0.052,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1188.0,+76/-91,18.695,0.125,33.4,22.89,1666.78,7.827,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1186.0,+59/-70,17.208,0.13,33.56,24.95,3693.32,0.131,✔️,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1185.0,+82/-88,19.691,0.149,33.62,17.79,497.58,0.163,✔️,0.0,False,CPU -41,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1148.0,+77/-82,20.42,0.098,35.94,26.54,575.62,1.155,✔️,0.0,False,CPU -42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1137.0,+57/-66,18.612,0.078,36.6,30.99,3.63,0.218,✔️,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1129.0,+69/-91,19.921,0.092,37.06,24.49,1666.78,0.561,✔️,0.0,False,CPU -44,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1126.0,+53/-63,18.524,0.047,37.25,32.25,3.1,0.207,✔️,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1097.0,+63/-68,21.432,0.052,38.97,32.32,575.62,0.147,✔️,0.0,False,CPU -46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1043.0,+90/-101,23.257,0.067,41.92,25.27,6.09,0.951,✔️,0.0,False,GPU -47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1014.0,+64/-97,23.054,0.026,43.42,38.0,12.68,0.155,✔️,0.0,False,CPU -48,🧠⚡,Foundation Model,[TabFlex (default) [22.22% IMPUTED]](https://arxiv.org/abs/2506.05584),1010.0,+75/-83,25.516,0.05,43.61,34.89,1.59,0.149,➖,22.22,True,GPU -49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+69/-89,25.971,0.02,44.1,33.9,0.51,0.085,✔️,0.0,False,CPU -50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),992.0,+76/-81,26.736,0.021,44.47,40.13,0.43,0.081,✔️,0.0,False,CPU -51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),980.0,+58/-83,24.045,0.023,45.03,41.34,5.83,0.571,✔️,0.0,False,CPU -52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),967.0,+70/-121,27.027,0.029,45.64,39.24,116.3,1.243,✔️,0.0,False,CPU -53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),890.0,+85/-106,32.488,0.028,48.74,21.03,520.62,0.491,✔️,0.0,False,CPU -54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),864.0,+93/-116,33.0,0.023,49.67,30.07,520.62,0.097,✔️,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),824.0,+102/-147,35.769,0.011,50.94,46.99,2.59,0.129,✔️,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),801.0,+79/-133,33.024,0.016,51.61,48.91,116.3,0.075,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),608.0,+97/-193,45.176,0.0,55.64,55.2,0.37,0.035,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1626.0,+127/-86,0.677,8.75,2.66,3.914,2319.48,8.957,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1584.0,+102/-67,0.599,10.38,6.01,6.742,6.58,0.586,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1570.0,+105/-71,0.595,10.97,4.53,6.315,2319.48,0.942,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1562.0,+81/-66,0.588,11.33,4.91,7.459,657.21,7.989,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1497.0,+66/-57,0.465,14.44,7.97,10.159,3855.22,24.935,✔️,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1480.0,+58/-53,0.443,15.36,8.88,10.167,3351.09,1.56,✔️,0.0,False,CPU +6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1472.0,+109/-80,0.471,15.78,5.93,10.288,3560.23,49.21,✔️,8.33,True,GPU +7,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1452.0,+94/-72,0.469,16.94,4.99,9.823,8059.77,364.288,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1439.0,+91/-71,0.454,17.65,6.26,12.376,5.11,0.624,➖,0.0,False,GPU +9,🧠🔁,Neural Network,[Mitra (default) [8.33% IMPUTED]](https://arxiv.org/abs/2510.21204),1432.0,+86/-50,0.394,18.08,8.05,11.429,128.8,2.999,✔️,8.33,True,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1429.0,+71/-72,0.381,18.26,9.71,11.477,3855.22,0.954,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1412.0,+87/-81,0.418,19.31,6.51,11.069,8059.77,39.456,✔️,0.0,False,GPU +12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1388.0,+68/-56,0.332,20.75,11.13,12.268,3629.74,1.48,✔️,0.0,False,GPU +13,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1376.0,+45/-43,0.25,21.57,16.63,13.566,833.03,2.182,✔️,0.0,False,CPU +14,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1369.0,+88/-71,0.325,21.99,11.42,12.643,3560.23,0.519,✔️,8.33,True,GPU +15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1349.0,+60/-57,0.239,23.26,17.1,13.386,2448.79,0.814,✔️,0.0,False,CPU +16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1332.0,+70/-63,0.258,24.36,14.4,14.055,1346.37,2.149,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1326.0,+74/-71,0.265,24.79,15.4,13.287,3629.74,0.189,✔️,0.0,False,GPU +18,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1323.0,+64/-53,0.215,25.0,18.83,13.932,2448.79,0.078,✔️,0.0,False,CPU +19,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1320.0,+55/-43,0.199,25.18,16.6,14.57,886.51,1.982,✔️,0.0,False,CPU +20,🧠⚡,Foundation Model,[TabPFNv2 (default) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1316.0,+78/-64,0.297,25.43,9.56,13.855,4.98,0.511,✔️,8.33,True,GPU +21,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1311.0,+67/-71,0.258,25.78,11.95,14.826,6341.76,8.556,✔️,0.0,False,GPU +22,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1309.0,+52/-46,0.196,25.92,20.51,14.586,833.03,0.435,✔️,0.0,False,CPU +23,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1308.0,+85/-80,0.311,26.0,8.54,13.661,51.78,39.927,✔️,0.0,False,GPU +24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1307.0,+58/-51,0.185,26.03,13.38,14.729,886.51,0.318,✔️,0.0,False,CPU +25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1305.0,+48/-45,0.173,26.21,19.69,14.65,8.39,0.139,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1298.0,+63/-64,0.223,26.64,12.4,14.214,6341.76,0.501,✔️,0.0,False,GPU +27,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1295.0,+76/-79,0.265,26.85,11.09,15.747,16.54,1.255,➖,0.0,False,GPU +28,🧠⚡,Foundation Model,[TabICL (default) [27.78% IMPUTED]](https://arxiv.org/abs/2502.05564),1285.0,+90/-99,0.253,27.53,8.17,14.682,7.56,1.494,✔️,27.78,True,GPU +29,🧠⚡,Foundation Model,[BetaTabPFN (default) [22.22% IMPUTED]](https://arxiv.org/abs/2502.02527),1280.0,+76/-77,0.226,27.89,16.19,16.539,269.12,1.881,➖,22.22,True,GPU +30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1278.0,+65/-62,0.191,27.99,13.38,15.03,1346.37,0.103,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1269.0,+65/-61,0.213,28.61,18.45,15.611,13.71,0.149,✔️,0.0,False,GPU +32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1267.0,+67/-57,0.162,28.72,17.04,16.053,3249.24,0.548,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1261.0,+57/-61,0.147,29.17,18.82,15.321,16.6,4.299,✔️,0.0,False,GPU +34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1240.0,+51/-62,0.151,30.53,23.54,15.608,3693.32,2.14,✔️,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1232.0,+64/-60,0.136,31.06,14.09,16.691,3249.24,0.059,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1207.0,+72/-72,0.117,32.76,14.6,19.018,14.9,0.323,✔️,0.0,False,GPU +37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1206.0,+78/-77,0.133,32.83,22.24,19.199,497.58,1.147,✔️,0.0,False,CPU +38,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1202.0,+71/-61,0.114,33.11,15.67,17.583,9.93,0.052,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1186.0,+76/-91,0.126,34.12,23.77,18.695,1666.78,7.827,✔️,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1185.0,+79/-88,0.148,34.18,18.0,19.691,497.58,0.163,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1184.0,+58/-71,0.131,34.22,25.42,17.208,3693.32,0.131,✔️,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1147.0,+75/-81,0.096,36.64,27.49,20.42,575.62,1.155,✔️,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1136.0,+57/-67,0.079,37.29,31.48,18.612,3.63,0.218,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1128.0,+67/-91,0.091,37.78,25.08,19.921,1666.78,0.561,✔️,0.0,False,CPU +45,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1126.0,+53/-65,0.048,37.92,32.72,18.524,3.1,0.207,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1096.0,+60/-69,0.05,39.69,33.43,21.432,575.62,0.147,✔️,0.0,False,CPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1044.0,+92/-100,0.064,42.67,25.58,23.257,6.09,0.951,✔️,0.0,False,GPU +48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1014.0,+64/-97,0.026,44.22,38.69,23.054,12.68,0.155,✔️,0.0,False,CPU +49,🧠⚡,Foundation Model,[TabFlex (default) [22.22% IMPUTED]](https://arxiv.org/abs/2506.05584),1009.0,+76/-86,0.049,44.44,35.71,25.516,1.59,0.149,➖,22.22,True,GPU +50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+69/-87,0.02,44.9,36.07,25.971,0.51,0.085,✔️,0.0,False,CPU +51,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),992.0,+75/-79,0.019,45.28,40.93,26.736,0.43,0.081,✔️,0.0,False,CPU +52,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),980.0,+56/-82,0.022,45.86,42.25,24.045,5.83,0.571,✔️,0.0,False,CPU +53,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),968.0,+69/-118,0.029,46.42,39.78,27.027,116.3,1.243,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),890.0,+84/-106,0.028,49.65,21.2,32.488,520.62,0.491,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),864.0,+94/-114,0.024,50.56,30.39,33.0,520.62,0.097,✔️,0.0,False,CPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),826.0,+102/-147,0.011,51.81,47.74,35.769,2.59,0.129,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),802.0,+80/-132,0.017,52.5,49.71,33.024,116.3,0.075,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),606.0,+95/-190,0.0,56.64,56.21,45.176,0.37,0.035,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_small/winrate_matrix.png.zip index 652c1c7a59d32a4c2a8d0f2904cb5b65739b8497..a0739c90faec261ca9817f3ca9d4aa0e95564352 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:faf8469be491dbfcf7401953d736c2ed1f5c39409c827b4a0c0acb6ce5cea3c2 -size 2471898 +oid sha256:ea7ba95773cc8bbd204623bda1aaba07c3cec29818a565c5a1a5501a623be18c +size 2451338 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 806651db9e2856d756a75753d6105442019ef39d..0e86aaf85c04c82c8d32028fbe013cdc35bec172 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:07f2ec8dd72f1bc6ae1c305cd03443745fa47a41e5f90085bc3219dbf5e65302 -size 470545 +oid sha256:1576c31242283d34d20cb608eef2588dde644cf326aba436b116c2e58aa822fc +size 485390 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip index 4ba4631c7d153c4ef61ed8815868a0ec97102e5e..ac759c178330d169ed0ce5544f43b59377f4371c 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c4e27df66c0e772c355e2cde58b8fff8cacd141378fa055e706b6ab6dc57a12a -size 1068077 +oid sha256:27e39aa5ff65cae28390dc15421096df75f9cdac5f62078ac0ca452ad9999c84 +size 1028754 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip index 4fd48b754dd3a016457f8b2831143b20dcc42d60..c09d609ed0ad7eec42cce0dae27bdd378e7304f8 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:410bba1c7c70862639a6df85afbf506e0b97ec1dd9339cf823c136e5ece3272a -size 143442 +oid sha256:48981924794915047f2796cf3aaa2c4e7f8171efd3308b67bcc72175b8859e31 +size 149794 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/website_leaderboard.csv index 8d61b31c9252293053cf10b3133c17b4e3bcc9f6..46de48c0f01dd001040af4d784da6a932769405e 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1650.0,+124/-84,4.175,0.683,8.42,2.71,2200.35,8.314,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1630.0,+84/-70,7.065,0.622,9.14,5.48,6.21,0.561,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1598.0,+113/-87,6.692,0.6,10.36,4.25,2200.35,0.882,✔️,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1570.0,+96/-80,7.982,0.581,11.52,4.86,625.63,7.662,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1536.0,+102/-76,9.632,0.517,13.08,5.44,3650.56,52.333,✔️,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1533.0,+74/-57,10.676,0.488,13.21,7.36,3565.03,23.942,✔️,0.0,False,GPU -6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1488.0,+62/-51,10.911,0.426,15.52,8.68,3035.75,1.498,✔️,0.0,False,CPU -7,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1486.0,+86/-68,10.885,0.434,15.59,7.42,137.76,3.046,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1469.0,+92/-79,13.013,0.461,16.5,5.8,4.66,0.611,➖,0.0,False,GPU -9,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1462.0,+96/-75,10.452,0.458,16.94,4.96,7821.64,382.96,✔️,0.0,False,GPU -10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1456.0,+69/-67,12.039,0.4,17.23,8.75,3565.03,0.706,✔️,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1422.0,+108/-73,11.708,0.415,19.21,6.24,7821.64,39.414,✔️,0.0,False,GPU -12,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1413.0,+88/-73,12.201,0.357,19.74,10.49,3650.56,0.543,✔️,0.0,False,GPU -13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1401.0,+81/-72,12.887,0.33,20.48,10.84,3407.57,1.449,✔️,0.0,False,GPU -14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1370.0,+60/-45,14.631,0.213,22.35,18.6,780.12,2.129,✔️,0.0,False,CPU -15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1361.0,+70/-63,14.299,0.239,22.95,16.22,2234.83,0.653,✔️,0.0,False,CPU -16,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1352.0,+86/-61,13.532,0.325,23.52,8.67,4.97,0.545,✔️,0.0,False,GPU -17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1334.0,+82/-72,13.965,0.253,24.65,15.76,3407.57,0.173,✔️,0.0,False,GPU -18,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1329.0,+74/-58,15.058,0.227,24.94,17.24,1216.65,2.007,✔️,0.0,False,GPU -19,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1328.0,+56/-52,14.923,0.202,25.03,19.5,2234.83,0.071,✔️,0.0,False,CPU -20,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1327.0,+103/-84,14.25,0.33,25.09,7.99,51.52,40.033,✔️,0.0,False,GPU -21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1323.0,+51/-55,15.678,0.178,25.35,20.59,848.99,1.866,✔️,0.0,False,CPU -22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1323.0,+91/-73,15.733,0.253,25.36,16.7,6308.39,7.737,✔️,0.0,False,GPU -23,🧠⚡,Foundation Model,[TabICL (default) [21.21% IMPUTED]](https://arxiv.org/abs/2502.05564),1316.0,+94/-98,14.435,0.279,25.8,7.59,7.68,1.615,✔️,21.21,True,GPU -24,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1313.0,+58/-46,15.66,0.161,25.95,18.68,8.13,0.126,✔️,0.0,False,CPU -25,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1312.0,+77/-83,14.889,0.223,26.06,12.6,6308.39,0.477,✔️,0.0,False,GPU -26,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1309.0,+59/-50,15.82,0.165,26.24,13.96,848.99,0.298,✔️,0.0,False,CPU -27,🧠⚡,Foundation Model,[BetaTabPFN (default) [21.21% IMPUTED]](https://arxiv.org/abs/2502.02527),1306.0,+84/-85,16.996,0.235,26.42,15.22,277.28,1.727,➖,21.21,True,GPU -28,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1302.0,+72/-63,15.671,0.16,26.67,21.7,780.12,0.381,✔️,0.0,False,CPU -29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1290.0,+73/-68,16.838,0.172,27.48,16.1,2961.52,0.521,✔️,0.0,False,CPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1287.0,+62/-54,15.815,0.163,27.67,17.51,16.29,4.082,✔️,0.0,False,GPU -31,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1281.0,+77/-76,16.232,0.211,28.03,17.74,13.3,0.146,✔️,0.0,False,GPU -32,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1272.0,+68/-66,16.037,0.16,28.65,20.28,1216.65,0.097,✔️,0.0,False,GPU -33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1248.0,+64/-62,16.297,0.144,30.15,23.3,3460.09,2.057,✔️,0.0,False,CPU -34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1246.0,+74/-66,17.567,0.138,30.3,13.32,2961.52,0.054,✔️,0.0,False,CPU -35,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1221.0,+66/-59,19.358,0.117,31.89,14.0,14.5,0.3,✔️,0.0,False,GPU -36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1220.0,+78/-78,18.183,0.11,31.94,14.62,9.77,0.046,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1204.0,+86/-83,19.428,0.134,32.98,22.16,1453.07,6.678,✔️,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1195.0,+72/-79,20.536,0.107,33.56,26.02,418.39,0.972,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1180.0,+71/-61,17.9,0.106,34.45,27.58,3460.09,0.121,✔️,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1176.0,+75/-82,20.959,0.13,34.76,19.79,418.39,0.147,✔️,0.0,False,CPU -41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1141.0,+77/-85,20.713,0.1,36.85,23.73,1453.07,0.637,✔️,0.0,False,CPU -42,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1134.0,+74/-98,21.857,0.073,37.24,27.69,563.93,0.997,✔️,0.0,False,CPU -43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1120.0,+51/-60,19.796,0.026,38.06,35.14,2.86,0.19,✔️,0.0,False,CPU -44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1112.0,+51/-65,20.033,0.037,38.53,35.23,3.28,0.179,✔️,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1092.0,+70/-84,22.761,0.046,39.67,32.65,563.93,0.149,✔️,0.0,False,CPU -46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1050.0,+89/-100,24.517,0.073,41.94,24.43,5.55,0.867,✔️,0.0,False,GPU -47,🧠⚡,Foundation Model,[TabFlex (default) [21.21% IMPUTED]](https://arxiv.org/abs/2506.05584),1024.0,+82/-95,26.178,0.054,43.27,34.1,1.29,0.149,➖,21.21,True,GPU -48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1012.0,+79/-77,24.002,0.028,43.85,38.13,9.39,0.15,✔️,0.0,False,CPU -49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+82/-107,26.75,0.021,44.42,33.54,0.48,0.074,✔️,0.0,False,CPU -50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),994.0,+86/-117,27.387,0.023,44.7,40.11,0.38,0.075,✔️,0.0,False,CPU -51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),984.0,+77/-97,24.643,0.025,45.18,41.29,5.2,0.522,✔️,0.0,False,CPU -52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),955.0,+74/-83,28.217,0.007,46.42,43.79,101.2,1.135,✔️,0.0,False,CPU -53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),912.0,+86/-109,32.259,0.03,48.15,19.91,497.63,0.481,✔️,0.0,False,CPU -54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),887.0,+84/-108,32.749,0.026,49.08,28.85,497.63,0.084,✔️,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),849.0,+98/-121,35.015,0.012,50.36,46.23,2.47,0.121,✔️,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),771.0,+84/-107,34.522,0.0,52.58,51.87,101.2,0.065,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),625.0,+106/-165,45.393,0.0,55.48,55.02,0.28,0.026,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1647.0,+123/-84,0.68,8.58,2.71,4.175,2200.35,8.314,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1626.0,+85/-70,0.622,9.35,5.67,7.065,6.21,0.561,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1595.0,+112/-88,0.599,10.58,4.3,6.692,2200.35,0.882,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1567.0,+96/-80,0.577,11.79,4.89,7.982,625.63,7.662,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1533.0,+100/-76,0.514,13.38,5.51,9.632,3650.56,52.333,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1530.0,+71/-56,0.485,13.52,7.52,10.676,3565.03,23.942,✔️,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1485.0,+61/-50,0.423,15.85,8.85,10.911,3035.75,1.498,✔️,0.0,False,CPU +7,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),1482.0,+83/-67,0.43,15.98,7.5,10.885,137.76,3.046,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1468.0,+91/-79,0.459,16.8,5.9,13.013,4.66,0.611,➖,0.0,False,GPU +9,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1459.0,+95/-74,0.457,17.27,5.0,10.452,7821.64,382.96,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1453.0,+68/-64,0.397,17.62,9.23,12.039,3565.03,0.706,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1420.0,+107/-73,0.413,19.61,6.3,11.708,7821.64,39.414,✔️,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1411.0,+86/-73,0.355,20.15,10.73,12.201,3650.56,0.543,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1399.0,+79/-70,0.326,20.88,11.01,12.887,3407.57,1.449,✔️,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1368.0,+60/-44,0.211,22.8,18.86,14.631,780.12,2.129,✔️,0.0,False,CPU +15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1360.0,+68/-60,0.233,23.35,17.15,14.299,2234.83,0.653,✔️,0.0,False,CPU +16,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1350.0,+85/-61,0.324,24.0,8.95,13.532,4.97,0.545,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1332.0,+83/-70,0.252,25.17,16.03,13.965,3407.57,0.173,✔️,0.0,False,GPU +18,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1327.0,+74/-57,0.224,25.45,17.53,15.058,1216.65,2.007,✔️,0.0,False,GPU +19,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1326.0,+54/-50,0.196,25.52,19.79,14.923,2234.83,0.071,✔️,0.0,False,CPU +20,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1325.0,+103/-84,0.329,25.61,8.04,14.25,51.52,40.033,✔️,0.0,False,GPU +21,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1324.0,+82/-88,0.279,25.65,10.45,16.307,13.95,1.208,➖,0.0,False,GPU +22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1321.0,+92/-73,0.251,25.88,16.85,15.733,6308.39,7.737,✔️,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1320.0,+49/-54,0.175,25.92,21.1,15.678,848.99,1.866,✔️,0.0,False,CPU +24,🧠⚡,Foundation Model,[TabICL (default) [21.21% IMPUTED]](https://arxiv.org/abs/2502.05564),1315.0,+94/-96,0.276,26.29,7.62,14.435,7.68,1.615,✔️,21.21,True,GPU +25,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1312.0,+77/-83,0.222,26.48,12.68,14.889,6308.39,0.477,✔️,0.0,False,GPU +26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1311.0,+58/-45,0.16,26.5,19.87,15.66,8.13,0.126,✔️,0.0,False,CPU +27,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1307.0,+60/-50,0.162,26.79,14.2,15.82,848.99,0.298,✔️,0.0,False,CPU +28,🧠⚡,Foundation Model,[BetaTabPFN (default) [21.21% IMPUTED]](https://arxiv.org/abs/2502.02527),1304.0,+84/-85,0.235,26.97,15.47,16.996,277.28,1.727,➖,21.21,True,GPU +29,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1301.0,+71/-63,0.159,27.18,21.99,15.671,780.12,0.381,✔️,0.0,False,CPU +30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1289.0,+72/-67,0.17,27.97,16.29,16.838,2961.52,0.521,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1285.0,+60/-54,0.16,28.27,17.99,15.815,16.29,4.082,✔️,0.0,False,GPU +32,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1280.0,+76/-76,0.211,28.58,17.98,16.232,13.3,0.146,✔️,0.0,False,GPU +33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1271.0,+68/-65,0.158,29.2,20.68,16.037,1216.65,0.097,✔️,0.0,False,GPU +34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1246.0,+64/-62,0.144,30.82,23.66,16.297,3460.09,2.057,✔️,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1245.0,+73/-65,0.137,30.91,13.46,17.567,2961.52,0.054,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1220.0,+66/-59,0.115,32.5,14.1,19.358,14.5,0.3,✔️,0.0,False,GPU +37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1219.0,+77/-78,0.109,32.61,14.98,18.183,9.77,0.046,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1201.0,+86/-84,0.134,33.74,23.06,19.428,1453.07,6.678,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1195.0,+71/-76,0.104,34.17,26.43,20.536,418.39,0.972,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1179.0,+70/-60,0.106,35.15,28.2,17.9,3460.09,0.121,✔️,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1176.0,+75/-82,0.128,35.33,20.05,20.959,418.39,0.147,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1140.0,+78/-86,0.098,37.61,24.33,20.713,1453.07,0.637,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1133.0,+74/-94,0.07,38.0,28.81,21.857,563.93,0.997,✔️,0.0,False,CPU +44,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1120.0,+52/-60,0.026,38.79,35.75,19.796,2.86,0.19,✔️,0.0,False,CPU +45,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1111.0,+50/-65,0.037,39.29,35.92,20.033,3.28,0.179,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1092.0,+68/-84,0.043,40.42,33.86,22.761,563.93,0.149,✔️,0.0,False,CPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1050.0,+88/-99,0.069,42.73,24.73,24.517,5.55,0.867,✔️,0.0,False,GPU +48,🧠⚡,Foundation Model,[TabFlex (default) [21.21% IMPUTED]](https://arxiv.org/abs/2506.05584),1023.0,+81/-92,0.053,44.12,34.93,26.178,1.29,0.149,➖,21.21,True,GPU +49,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1011.0,+78/-76,0.028,44.7,38.87,24.002,9.39,0.15,✔️,0.0,False,CPU +50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+82/-103,0.022,45.24,35.81,26.75,0.48,0.074,✔️,0.0,False,CPU +51,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),994.0,+86/-111,0.02,45.52,40.92,27.387,0.38,0.075,✔️,0.0,False,CPU +52,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),983.0,+74/-93,0.024,46.03,42.24,24.643,5.2,0.522,✔️,0.0,False,CPU +53,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),957.0,+73/-82,0.007,47.21,44.48,28.217,101.2,1.135,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),912.0,+86/-110,0.03,49.06,20.06,32.259,497.63,0.481,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),888.0,+84/-105,0.026,49.95,29.15,32.749,497.63,0.084,✔️,0.0,False,CPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),851.0,+100/-118,0.012,51.21,46.96,35.015,2.47,0.121,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),772.0,+84/-108,0.0,53.48,52.79,34.522,101.2,0.065,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),624.0,+105/-164,0.0,56.48,56.03,45.393,0.28,0.026,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/winrate_matrix.png.zip index 68cb3aaecac0efe1c1a9a689cf93e39688365978..4e7b257a2ebdd4ed1e5dbb662538c06e0170b0ff 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d22360b40385e6c41366aca2b7b9302a934c01a500b3cf5fccdcf216391e6785 -size 2416990 +oid sha256:4c6ff3c91c7200f273555cad0b9f6a4fba01c03fdb4ca6558e56a10abdb138dd +size 2388302 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index 5b193f667ca433797f81f11f8f8cf1c59a2d2ae2..bfa66ad4b2011e6f8298eea7a39ae99e6a4fe569 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:02dd24cd571aa73fb038155030805c59a3b85ac8289a17f13b55b415c4528588 -size 492020 +oid sha256:9a0770b589f710841cac216b4e4358e0d71f91629a7e116ca6276afd713275e4 +size 505664 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip index 9bde16ed3444ad947590c2b3a9bd840c00470b73..60f4536586385761c8bdc88838f79cd4d49dfde8 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:dae95d50999e4d436a1f0dd33029bc78d5356fe64d0ef5021901bde4157757f0 -size 1046013 +oid sha256:43cc22f839965870f8e9cd2742c692696584731c7361cd60e8a184011702a46c +size 1029599 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/tuning-impact-elo.png.zip index 349855878157c987d99d0d22d310d8b06e80db5d..91e930e83e468343abbb9a633f48c6161c596852 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d2d25057404273423286e27d8748d667742b8a44c939e805d9f6ee66f8a28d96 -size 183658 +oid sha256:215ffe07b8d22328e7dc2a75e157d7552bb2741a5e0674c1eef2f72b88c86f4c +size 182231 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/website_leaderboard.csv index 3f928c74e3514929f8ce21c7398c165f2d063478..97e50695e91ce45c7cea3e3d97efaf52f2722492 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1568.0,+105/-74,5.636,0.589,10.57,3.67,1877.76,10.745,✔️,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1562.0,+112/-85,6.142,0.623,10.8,3.7,442.6,6.737,✔️,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1547.0,+99/-70,7.435,0.567,11.47,4.88,1231.21,1.36,✔️,0.0,False,CPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1546.0,+95/-69,7.006,0.567,11.5,6.22,4.79,0.708,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1537.0,+102/-63,6.667,0.551,11.92,5.24,1877.76,1.035,✔️,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1500.0,+69/-60,9.187,0.477,13.73,8.61,2721.85,9.311,✔️,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1424.0,+76/-72,11.154,0.388,17.92,8.05,2721.85,0.265,✔️,0.0,False,GPU -7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1412.0,+81/-64,11.026,0.382,18.67,7.44,2180.12,1.135,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[TabICL (default) [3.33% IMPUTED]](https://arxiv.org/abs/2502.05564),1394.0,+82/-68,10.012,0.349,19.78,7.26,8.05,2.008,✔️,3.33,True,GPU -9,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1388.0,+52/-50,12.615,0.292,20.12,14.44,328.64,0.774,✔️,0.0,False,CPU -10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1374.0,+71/-61,9.545,0.34,21.03,7.84,5890.6,409.986,✔️,0.0,False,GPU -11,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1368.0,+62/-51,12.209,0.304,21.4,10.27,3.84,0.068,✔️,0.0,False,CPU -12,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1367.0,+70/-70,13.18,0.302,21.45,11.5,462.92,0.613,✔️,0.0,False,CPU -13,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1366.0,+87/-76,12.046,0.331,21.52,10.11,3436.74,0.405,✔️,0.0,False,GPU -14,🧠⚡,Foundation Model,[LimiX (default) [16.67% IMPUTED]](https://arxiv.org/abs/2509.03505),1362.0,+93/-84,13.114,0.365,21.75,6.22,1.95,0.488,➖,16.67,True,GPU -15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1358.0,+69/-73,12.343,0.302,22.05,12.29,1043.89,0.478,✔️,0.0,False,CPU -16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1350.0,+94/-95,12.196,0.353,22.52,8.36,3436.74,8.478,✔️,0.0,False,GPU -17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1340.0,+85/-73,12.161,0.305,23.22,12.12,2180.12,0.117,✔️,0.0,False,GPU -18,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1332.0,+56/-61,12.867,0.276,23.72,14.78,1043.89,0.041,✔️,0.0,False,CPU -19,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1332.0,+76/-80,11.754,0.281,23.72,9.4,5890.6,39.565,✔️,0.0,False,GPU -20,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1332.0,+95/-64,14.412,0.275,23.73,13.6,248.47,1.7,➖,0.0,False,GPU -21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1331.0,+75/-80,13.559,0.262,23.77,10.5,462.92,0.108,✔️,0.0,False,CPU -22,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1320.0,+58/-62,13.786,0.23,24.53,19.55,328.64,0.089,✔️,0.0,False,CPU -23,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1315.0,+94/-79,14.164,0.304,24.85,9.85,2914.83,17.91,✔️,33.33,True,GPU -24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1311.0,+68/-57,14.601,0.213,25.1,15.6,1579.5,0.428,✔️,0.0,False,CPU -25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1304.0,+69/-82,13.468,0.224,25.6,17.15,559.77,1.513,✔️,0.0,False,GPU -26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1299.0,+58/-53,13.707,0.175,25.9,16.67,9.04,1.162,✔️,0.0,False,GPU -27,🧠⚡,Foundation Model,[Mitra (default) [33.33% IMPUTED]](https://arxiv.org/abs/2510.21204),1296.0,+102/-77,14.387,0.272,26.07,9.31,106.33,2.667,✔️,33.33,True,GPU -28,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1285.0,+89/-78,14.697,0.24,26.83,17.89,8.0,0.124,✔️,0.0,False,GPU -29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1273.0,+73/-55,13.865,0.179,27.63,20.43,2206.58,2.301,✔️,0.0,False,CPU -30,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1262.0,+91/-73,15.933,0.221,28.38,13.63,2914.83,0.225,✔️,33.33,True,GPU -31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1258.0,+63/-69,15.134,0.148,28.68,20.29,559.77,0.086,✔️,0.0,False,GPU -32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1257.0,+67/-57,15.369,0.14,28.73,13.78,1579.5,0.05,✔️,0.0,False,CPU -33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1251.0,+70/-60,16.377,0.15,29.1,13.32,4.79,0.045,✔️,0.0,False,CPU -34,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1222.0,+72/-90,14.951,0.179,31.08,11.5,47.89,39.446,✔️,0.0,False,GPU -35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1218.0,+76/-88,16.897,0.139,31.35,21.38,561.29,4.462,✔️,0.0,False,CPU -36,🧠⚡,Foundation Model,[TabPFNv2 (default) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1216.0,+78/-78,17.278,0.196,31.48,10.46,2.61,0.256,✔️,33.33,True,GPU -37,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1200.0,+85/-80,16.455,0.141,32.53,18.88,1.42,0.118,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1191.0,+77/-69,16.272,0.131,33.07,25.9,2206.58,0.107,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1188.0,+63/-62,17.287,0.111,33.25,13.68,11.15,0.312,✔️,0.0,False,GPU -40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1173.0,+74/-68,18.237,0.081,34.25,27.98,122.9,0.583,✔️,0.0,False,CPU -41,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1140.0,+68/-85,19.635,0.061,36.3,29.95,171.61,0.552,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1138.0,+79/-101,19.002,0.096,36.47,23.6,561.29,0.236,✔️,0.0,False,CPU -43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1135.0,+63/-66,17.467,0.082,36.65,32.0,0.93,0.094,✔️,0.0,False,CPU -44,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1130.0,+79/-75,19.849,0.086,36.97,21.52,122.9,0.072,✔️,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1106.0,+64/-69,20.285,0.021,38.42,34.53,171.61,0.053,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1044.0,+74/-69,21.546,0.035,41.93,36.06,4.97,0.092,✔️,0.0,False,CPU -47,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1010.0,+104/-121,26.685,0.059,43.7,33.56,0.81,0.115,➖,0.0,False,GPU -48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+76/-92,24.447,0.024,44.18,32.57,0.29,0.031,✔️,0.0,False,CPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),998.0,+75/-93,22.018,0.031,44.27,40.03,2.79,0.259,✔️,0.0,False,CPU -50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),991.0,+63/-92,24.644,0.006,44.63,42.41,140.66,3.209,✔️,0.0,False,CPU -51,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),973.0,+93/-126,28.274,0.041,45.47,18.11,155.53,0.277,✔️,0.0,False,CPU -52,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),941.0,+96/-133,28.935,0.03,46.88,26.66,155.53,0.05,✔️,0.0,False,CPU -53,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),930.0,+91/-113,27.199,0.037,47.33,41.1,2.71,0.392,✔️,0.0,False,GPU -54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),924.0,+92/-110,26.269,0.016,47.6,43.29,0.18,0.035,✔️,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),904.0,+98/-113,30.667,0.013,48.4,44.34,0.8,0.088,✔️,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),827.0,+62/-96,30.916,0.0,51.03,50.24,140.66,0.188,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),586.0,+96/-207,43.999,0.0,56.0,55.7,0.15,0.024,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1567.0,+108/-75,0.59,10.73,3.68,5.636,1877.76,10.745,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1561.0,+113/-88,0.622,11.0,3.7,6.142,442.6,6.737,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1547.0,+97/-69,0.571,11.63,6.26,7.006,4.79,0.708,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1546.0,+103/-71,0.567,11.67,4.92,7.435,1231.21,1.36,✔️,0.0,False,CPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1537.0,+103/-63,0.552,12.08,5.26,6.667,1877.76,1.035,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1500.0,+70/-60,0.479,13.9,8.76,9.187,2721.85,9.311,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1424.0,+76/-72,0.39,18.22,8.45,11.154,2721.85,0.265,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1411.0,+81/-64,0.382,18.97,7.51,11.026,2180.12,1.135,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabICL (default) [3.33% IMPUTED]](https://arxiv.org/abs/2502.05564),1394.0,+82/-68,0.351,20.05,7.3,10.012,8.05,2.008,✔️,3.33,True,GPU +9,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1388.0,+52/-50,0.295,20.45,14.54,12.615,328.64,0.774,✔️,0.0,False,CPU +10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1374.0,+73/-61,0.344,21.33,7.86,9.545,5890.6,409.986,✔️,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1368.0,+63/-50,0.306,21.7,10.61,12.209,3.84,0.068,✔️,0.0,False,CPU +12,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1367.0,+70/-69,0.305,21.78,11.59,13.18,462.92,0.613,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1366.0,+86/-76,0.334,21.82,10.15,12.046,3436.74,0.405,✔️,0.0,False,GPU +14,🧠🔁,Neural Network,[LimiX (default) [16.67% IMPUTED]](https://arxiv.org/abs/2509.03505),1363.0,+93/-83,0.366,22.05,6.24,13.114,1.95,0.488,➖,16.67,True,GPU +15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1359.0,+69/-73,0.303,22.28,12.75,12.343,1043.89,0.478,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1350.0,+94/-96,0.355,22.92,8.39,12.196,3436.74,8.478,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1339.0,+84/-71,0.307,23.62,12.26,12.161,2180.12,0.117,✔️,0.0,False,GPU +18,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1333.0,+56/-60,0.277,24.02,14.86,12.867,1043.89,0.041,✔️,0.0,False,CPU +19,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1332.0,+95/-64,0.278,24.07,13.76,14.412,248.47,1.7,➖,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1331.0,+76/-78,0.267,24.13,10.6,13.559,462.92,0.108,✔️,0.0,False,CPU +21,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1331.0,+74/-80,0.281,24.15,9.51,11.754,5890.6,39.565,✔️,0.0,False,GPU +22,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1320.0,+58/-61,0.234,24.9,19.67,13.786,328.64,0.089,✔️,0.0,False,CPU +23,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1314.0,+93/-79,0.305,25.32,10.04,14.164,2914.83,17.91,✔️,33.33,True,GPU +24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1312.0,+67/-58,0.214,25.43,15.75,14.601,1579.5,0.428,✔️,0.0,False,CPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1304.0,+68/-83,0.225,26.0,17.3,13.468,559.77,1.513,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1299.0,+57/-52,0.178,26.33,17.02,13.707,9.04,1.162,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[Mitra (default) [33.33% IMPUTED]](https://arxiv.org/abs/2510.21204),1296.0,+100/-78,0.272,26.5,9.34,14.387,106.33,2.667,✔️,33.33,True,GPU +28,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1285.0,+88/-78,0.243,27.27,18.09,14.697,8.0,0.124,✔️,0.0,False,GPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1273.0,+73/-54,0.181,28.1,20.65,13.865,2206.58,2.301,✔️,0.0,False,CPU +30,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1262.0,+90/-73,0.22,28.87,13.73,15.933,2914.83,0.225,✔️,33.33,True,GPU +31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1259.0,+63/-70,0.148,29.05,20.45,15.134,559.77,0.086,✔️,0.0,False,GPU +32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1258.0,+68/-56,0.14,29.1,13.89,15.369,1579.5,0.05,✔️,0.0,False,CPU +33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1252.0,+71/-60,0.151,29.57,13.61,16.377,4.79,0.045,✔️,0.0,False,CPU +34,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1238.0,+85/-82,0.167,30.48,14.8,16.859,13.07,3.169,➖,0.0,False,GPU +35,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1221.0,+71/-90,0.18,31.65,11.57,14.951,47.89,39.446,✔️,0.0,False,GPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1216.0,+76/-89,0.14,31.95,22.12,16.897,561.29,4.462,✔️,0.0,False,CPU +37,🧠⚡,Foundation Model,[TabPFNv2 (default) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1215.0,+77/-78,0.196,32.05,10.51,17.278,2.61,0.256,✔️,33.33,True,GPU +38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1201.0,+86/-79,0.145,33.0,18.98,16.455,1.42,0.118,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1191.0,+76/-69,0.133,33.63,26.36,16.272,2206.58,0.107,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1189.0,+63/-62,0.112,33.82,13.75,17.287,11.15,0.312,✔️,0.0,False,GPU +41,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1174.0,+74/-64,0.08,34.82,28.29,18.237,122.9,0.583,✔️,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1142.0,+67/-83,0.062,36.87,30.49,19.635,171.61,0.552,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1137.0,+79/-100,0.096,37.17,24.06,19.002,561.29,0.236,✔️,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1136.0,+64/-66,0.085,37.25,32.36,17.467,0.93,0.094,✔️,0.0,False,CPU +45,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1130.0,+78/-75,0.086,37.6,21.71,19.849,122.9,0.072,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1107.0,+62/-66,0.021,39.02,35.27,20.285,171.61,0.053,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1044.0,+72/-70,0.035,42.67,36.64,21.546,4.97,0.092,✔️,0.0,False,CPU +48,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1008.0,+103/-123,0.059,44.57,34.4,26.685,0.81,0.115,➖,0.0,False,GPU +49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+76/-91,0.024,44.98,34.88,24.447,0.29,0.031,✔️,0.0,False,CPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),998.0,+73/-92,0.032,45.1,40.86,22.018,2.79,0.259,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),990.0,+63/-92,0.006,45.47,43.27,24.644,140.66,3.209,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),972.0,+92/-126,0.041,46.33,18.24,28.274,155.53,0.277,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),941.0,+95/-132,0.03,47.72,26.9,28.935,155.53,0.05,✔️,0.0,False,CPU +54,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),930.0,+91/-114,0.036,48.2,42.13,27.199,2.71,0.392,✔️,0.0,False,GPU +55,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),924.0,+91/-112,0.016,48.43,44.11,26.269,0.18,0.035,✔️,0.0,False,CPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),904.0,+96/-113,0.013,49.27,45.08,30.667,0.8,0.088,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),826.0,+62/-94,0.0,51.97,51.19,30.916,140.66,0.188,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),584.0,+95/-207,0.0,57.0,56.7,43.999,0.15,0.024,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/winrate_matrix.png.zip index 2a57fa64073675d96ee4edc0a5e0e5ae48d88de6..aac59758ad02681943e84719dfaeb5e81590e251 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:72788737937043034cfe107767f734cf0961225612a691ac783b36e76fc591a5 -size 2329337 +oid sha256:376fbcd6aa690ea9a586a19be6db150a214e9efeb7a3c3c0a7bdd4c893573515 +size 2288077 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index 615ed59abbfe1f021a17bea9594a9c80a66cfb7f..fa378f5abe7573a1f6d9e9d4aa9307fe89615103 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fed291c83460899264064b3f8e1047fb297987310c053a1760d0c448ec24b3a1 -size 448234 +oid sha256:cee258ee00a2fafa2bc3798b900d391320ca33f3cb745088941cd6f7a8d24b41 +size 464120 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip index f2ce5133485bc61bdea30eb66cb5350bddc90b33..09b12b58d51175fbe72e10161ab926d012ceb0b2 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6b0b2cca46e709f2ab78a838fe985612b0010f15d74d7295af91f07b64415511 -size 933385 +oid sha256:d22ff2e0b01980cd38ecac9b23103be21fa5085a57c54053cff695c58f684afc +size 915595 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/tuning-impact-elo.png.zip index 38618f877357fce214fe9817ed1dddd178adf0a7..3e026c78b706acc5289851cee728c0d56ae0ce6f 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6fb355fddb9d884690ab134d30c7c8cbf493dad72c950773c495cc5070c66856 -size 134935 +oid sha256:775aaa8d605114fad5f7179450e92d341f1ba7c2a05de1f0275b790a79fa55e7 +size 141329 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/website_leaderboard.csv index 8150f45cb95757e78b1004b7c2cf1a34d912397e..c1a39ef868a872a2af619edcce79aff5cc732ae2 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/website_leaderboard.csv @@ -1,55 +1,56 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",2055.0,+219/-66,1.559,0.859,3.44,2.17,310.55,1.824,✔️,0.0,False,CPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1995.0,+270/-81,2.117,0.834,4.33,2.22,256.14,2.868,✔️,0.0,False,GPU -2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1747.0,+274/-161,4.475,0.619,10.78,3.85,2149.06,6.373,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1729.0,+219/-166,5.896,0.527,11.44,4.12,841.84,11.882,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1714.0,+130/-48,5.584,0.517,12.0,8.92,1655.56,2.483,✔️,0.0,False,GPU -5,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1708.0,+186/-105,7.366,0.46,12.22,8.36,114.48,0.404,✔️,0.0,False,CPU -6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1689.0,+204/-112,6.578,0.517,13.0,7.63,618.23,0.184,✔️,0.0,False,CPU -7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1675.0,+224/-137,7.894,0.452,13.56,7.63,841.84,1.263,✔️,0.0,False,GPU -8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1670.0,+201/-118,7.403,0.478,13.78,5.14,807.72,0.575,✔️,0.0,False,GPU -9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1661.0,+168/-101,6.676,0.489,14.17,8.72,618.23,0.031,✔️,0.0,False,CPU -10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1660.0,+186/-100,8.502,0.43,14.22,10.24,87.3,0.533,✔️,0.0,False,CPU -11,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1644.0,+181/-117,9.317,0.418,14.89,10.64,2149.06,0.318,✔️,0.0,False,GPU -12,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1635.0,+152/-84,6.592,0.46,15.28,5.61,2.12,0.043,✔️,0.0,False,CPU -13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1617.0,+163/-118,7.688,0.403,16.11,10.87,114.48,0.058,✔️,0.0,False,CPU -14,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1609.0,+189/-106,8.195,0.394,16.44,9.39,1655.56,0.086,✔️,0.0,False,GPU -15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1581.0,+177/-114,9.447,0.326,17.78,13.98,87.3,0.069,✔️,0.0,False,CPU -16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1578.0,+230/-158,8.051,0.396,17.89,8.37,807.72,0.07,✔️,0.0,False,GPU -17,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1548.0,+182/-140,7.804,0.333,19.33,9.01,2.32,0.889,✔️,0.0,False,GPU -18,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1537.0,+176/-155,9.501,0.29,19.89,11.67,9.96,4.847,✔️,0.0,False,GPU -19,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1474.0,+173/-195,10.609,0.249,23.0,18.82,5.77,0.078,✔️,0.0,False,GPU -20,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1461.0,+174/-196,10.169,0.26,23.67,10.33,0.48,0.047,✔️,0.0,False,CPU -21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1461.0,+158/-145,9.627,0.248,23.67,15.34,559.89,1.156,✔️,0.0,False,GPU -22,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1461.0,+119/-70,9.923,0.179,23.67,19.26,1128.19,1.674,✔️,0.0,False,CPU -23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1444.0,+183/-128,13.232,0.215,24.56,20.49,262.92,0.205,✔️,0.0,False,CPU -24,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1442.0,+169/-148,10.124,0.186,24.67,18.8,4560.06,421.968,✔️,0.0,False,GPU -25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1431.0,+163/-120,12.808,0.166,25.22,17.58,559.89,0.056,✔️,0.0,False,GPU -26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1412.0,+85/-76,11.884,0.087,26.22,23.87,6.55,0.295,✔️,0.0,False,GPU -27,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1380.0,+112/-153,12.385,0.112,27.89,22.51,368.88,2.507,✔️,0.0,False,CPU -28,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1366.0,+202/-171,13.822,0.132,28.61,19.08,4560.06,39.452,✔️,0.0,False,GPU -29,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1338.0,+181/-190,13.509,0.13,30.0,19.31,92.9,0.786,➖,0.0,False,GPU -30,🧠⚡,Foundation Model,[LimiX (default) [55.56% IMPUTED]](https://arxiv.org/abs/2509.03505),1338.0,+244/-253,18.727,0.218,30.0,6.8,0.68,0.014,➖,55.56,True,GPU -31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1323.0,+165/-119,16.195,0.077,30.78,27.72,0.92,0.018,✔️,0.0,False,CPU -32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1314.0,+200/-159,14.917,0.097,31.22,25.64,262.92,0.015,✔️,0.0,False,CPU -33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1307.0,+122/-138,13.503,0.083,31.56,27.8,1128.19,0.055,✔️,0.0,False,CPU -34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1300.0,+123/-154,13.474,0.036,31.89,28.72,40.45,0.331,✔️,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1294.0,+55/-72,14.752,0.018,32.22,31.24,43.93,0.204,✔️,0.0,False,CPU -36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1266.0,+109/-148,12.579,0.047,33.56,31.32,0.34,0.052,✔️,0.0,False,CPU -37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1251.0,+51/-62,15.173,0.0,34.28,33.94,43.93,0.018,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1230.0,+108/-121,14.121,0.015,35.22,33.77,9.93,0.224,✔️,0.0,False,GPU -39,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1210.0,+137/-213,18.236,0.022,36.17,32.49,41.8,39.131,✔️,0.0,False,GPU -40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1194.0,+128/-148,17.071,0.02,36.89,34.1,40.45,0.028,✔️,0.0,False,CPU -41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1178.0,+108/-183,16.535,0.02,37.56,35.52,368.88,0.084,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1142.0,+101/-139,16.268,0.013,39.11,37.42,1.81,0.048,✔️,0.0,False,CPU -43,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1104.0,+173/-273,25.579,0.019,40.61,36.3,43.96,0.099,✔️,0.0,False,CPU -44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1102.0,+114/-195,20.929,0.02,40.67,37.79,514.21,52.341,✔️,0.0,False,CPU -45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1067.0,+137/-238,19.282,0.026,42.0,38.71,1.61,0.113,✔️,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1056.0,+170/-258,26.315,0.002,42.39,39.36,43.96,0.024,✔️,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+121/-272,22.467,0.0,44.28,43.24,0.11,0.012,✔️,0.0,False,CPU -48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),896.0,+238/-618,30.82,0.027,47.22,39.51,2.77,0.157,✔️,0.0,False,GPU -49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),887.0,+98/-215,28.058,0.0,47.44,47.08,0.22,0.022,✔️,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),859.0,+70/-166,28.846,0.0,48.11,47.78,514.21,2.49,✔️,0.0,False,CPU -51,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),859.0,+102/-228,25.047,0.0,48.11,47.82,0.1,0.02,✔️,0.0,False,CPU -52,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),823.0,+121/-262,33.192,0.0,48.89,48.33,0.2,0.035,➖,0.0,False,GPU -53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),488.0,+95/-357,46.138,0.0,53.11,53.08,0.11,0.089,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",2059.0,+224/-66,0.861,3.44,2.17,1.559,310.55,1.824,✔️,0.0,False,CPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1998.0,+267/-81,0.836,4.33,2.22,2.117,256.14,2.868,✔️,0.0,False,GPU +2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1747.0,+277/-164,0.62,10.89,3.85,4.475,2149.06,6.373,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1729.0,+224/-164,0.531,11.56,4.13,5.896,841.84,11.882,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1718.0,+130/-47,0.523,12.0,8.92,5.584,1655.56,2.483,✔️,0.0,False,GPU +5,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1712.0,+188/-104,0.466,12.22,8.36,7.366,114.48,0.404,✔️,0.0,False,CPU +6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1693.0,+204/-110,0.522,13.0,7.63,6.578,618.23,0.184,✔️,0.0,False,CPU +7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1676.0,+227/-137,0.456,13.67,7.64,7.894,841.84,1.263,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1671.0,+206/-117,0.481,13.89,5.15,7.403,807.72,0.575,✔️,0.0,False,GPU +9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1665.0,+166/-100,0.493,14.17,8.72,6.676,618.23,0.031,✔️,0.0,False,CPU +10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1661.0,+192/-107,0.435,14.33,10.26,8.502,87.3,0.533,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1646.0,+181/-120,0.422,15.0,10.65,9.317,2149.06,0.318,✔️,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1639.0,+150/-82,0.465,15.28,5.61,6.592,2.12,0.043,✔️,0.0,False,CPU +13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1618.0,+166/-114,0.407,16.22,10.92,7.688,114.48,0.058,✔️,0.0,False,CPU +14,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1611.0,+191/-109,0.4,16.56,9.4,8.195,1655.56,0.086,✔️,0.0,False,GPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1583.0,+181/-116,0.33,17.89,14.01,9.447,87.3,0.069,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1580.0,+235/-167,0.4,18.0,8.38,8.051,807.72,0.07,✔️,0.0,False,GPU +17,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1551.0,+185/-142,0.341,19.44,9.02,7.804,2.32,0.889,✔️,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1537.0,+176/-159,0.296,20.11,11.74,9.501,9.96,4.847,✔️,0.0,False,GPU +19,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1476.0,+178/-197,0.253,23.22,18.87,10.609,5.77,0.078,✔️,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1465.0,+176/-195,0.262,23.78,10.33,10.169,0.48,0.047,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1465.0,+118/-68,0.182,23.78,19.38,9.923,1128.19,1.674,✔️,0.0,False,CPU +22,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1463.0,+155/-146,0.25,23.89,15.38,9.627,559.89,1.156,✔️,0.0,False,GPU +23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1448.0,+185/-125,0.22,24.67,20.52,13.232,262.92,0.205,✔️,0.0,False,CPU +24,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1442.0,+172/-149,0.192,25.0,18.97,10.124,4560.06,421.968,✔️,0.0,False,GPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1437.0,+160/-117,0.168,25.22,17.58,12.808,559.89,0.056,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1414.0,+86/-76,0.095,26.44,23.98,11.884,6.55,0.295,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1383.0,+114/-154,0.113,28.11,22.57,12.385,368.88,2.507,✔️,0.0,False,CPU +28,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1365.0,+203/-172,0.134,29.06,19.19,13.822,4560.06,39.452,✔️,0.0,False,GPU +29,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1342.0,+181/-183,0.13,30.22,19.37,13.509,92.9,0.786,➖,0.0,False,GPU +30,🧠🔁,Neural Network,[LimiX (default) [55.56% IMPUTED]](https://arxiv.org/abs/2509.03505),1338.0,+246/-251,0.218,30.44,6.81,18.727,0.68,0.014,➖,55.56,True,GPU +31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1327.0,+171/-121,0.082,31.0,27.8,16.195,0.92,0.018,✔️,0.0,False,CPU +32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1321.0,+200/-154,0.101,31.33,25.68,14.917,262.92,0.015,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1310.0,+124/-143,0.087,31.89,27.96,13.503,1128.19,0.055,✔️,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1299.0,+123/-159,0.039,32.44,29.07,13.474,40.45,0.331,✔️,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1297.0,+55/-71,0.018,32.56,31.63,14.752,43.93,0.204,✔️,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1267.0,+113/-152,0.051,34.0,31.58,12.579,0.34,0.052,✔️,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1255.0,+50/-62,0.0,34.61,34.23,15.173,43.93,0.018,✔️,0.0,False,CPU +38,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1237.0,+124/-131,0.029,35.44,31.72,17.033,16.58,16.469,➖,0.0,False,GPU +39,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1230.0,+113/-126,0.019,35.78,34.15,14.121,9.93,0.224,✔️,0.0,False,GPU +40,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1210.0,+139/-216,0.023,36.72,32.83,18.236,41.8,39.131,✔️,0.0,False,GPU +41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1190.0,+128/-151,0.02,37.67,34.69,17.071,40.45,0.028,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1177.0,+110/-189,0.022,38.22,36.02,16.535,368.88,0.084,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1144.0,+104/-142,0.015,39.67,37.81,16.268,1.81,0.048,✔️,0.0,False,CPU +44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1102.0,+175/-273,0.021,41.39,36.9,25.579,43.96,0.099,✔️,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1095.0,+114/-196,0.02,41.67,38.92,20.929,514.21,52.341,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1062.0,+140/-242,0.028,42.89,39.38,19.282,1.61,0.113,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1055.0,+172/-260,0.003,43.17,39.98,26.315,43.96,0.024,✔️,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+124/-271,0.0,45.06,43.95,22.467,0.11,0.012,✔️,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),895.0,+233/-623,0.027,48.11,40.81,30.82,2.77,0.157,✔️,0.0,False,GPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),881.0,+98/-215,0.0,48.44,48.08,28.058,0.22,0.022,✔️,0.0,False,CPU +51,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),858.0,+106/-232,0.0,49.0,48.68,25.047,0.1,0.02,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),853.0,+67/-160,0.0,49.11,48.79,28.846,514.21,2.49,✔️,0.0,False,CPU +53,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),817.0,+121/-262,0.0,49.89,49.34,33.192,0.2,0.035,➖,0.0,False,GPU +54,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),484.0,+96/-360,0.0,54.11,54.08,46.138,0.11,0.089,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/winrate_matrix.png.zip index 9ba3e054d0850001f43e29c9121bb0b7d0902e10..24e49ab8daed39a78723187da15e51cbe224948b 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:21109011fe8ebcd72973055231212590179b02ef38fddaf5e6f1df1a53ba5009 -size 1568808 +oid sha256:fc76d33f2660dbff471fefe01e81290cce71bbceff6da3e304ff5fd2ee043203 +size 1678520 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index be9d6e9439894c154d755e2e95498f96129508b5..c54b82e4ba9612040403e25cc1122ee1d8540b39 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:86bb4df1cdfd81af44084f4ac48081b6a74088b02f509a42cff9b013ecb9e62a -size 508699 +oid sha256:22c2a3785d2da6cefa1f987f4337d10840e8cd4019a13523d6abfde9223ce53f +size 524137 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip index c76773acca9a5dafc2e5bc8be22102bce07d3673..7985a5f0968b504961a5b1fe3d5272d56ee70a20 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3de61c20ed929d8cb707f15ea5588b90007245ee0a0136774c1d41d3aae841f0 -size 1027994 +oid sha256:8a562120531707fbaab5411483b8ee0f6744b04273262b4c23d07be795eb8adb +size 996846 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/tuning-impact-elo.png.zip index e980fc5c4ffc6de3b5c14b69147d0e7fd1c72c9b..ddd9f0813c82f7b5cade6bdd75a96ff3899cedbd 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:87b52d1cf05d68b635f6a122d29b928c3401516bb6fff0491443da12e66368c1 -size 163970 +oid sha256:f11341f433d8893b0de609f9eed929f89ae379eceba0d9c638c62adb050984f6 +size 167665 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/website_leaderboard.csv index ce3f87366f5c688e1fc65a1f5fe93fcf02ef2eb2..2941715ed19c367788a90493e82ddbfe189dbeab 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1622.0,+93/-51,6.664,0.644,7.95,5.49,5.53,0.55,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1562.0,+146/-85,5.525,0.606,10.19,3.5,2125.17,9.785,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1538.0,+139/-103,6.141,0.583,11.21,4.62,2125.17,0.882,✔️,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1488.0,+128/-92,7.867,0.525,13.57,5.17,613.33,8.472,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1470.0,+81/-86,10.731,0.441,14.48,8.48,2791.97,23.942,✔️,0.0,False,GPU -5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1462.0,+91/-82,9.953,0.435,14.9,10.49,1886.83,1.333,✔️,0.0,False,CPU -6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1447.0,+142/-74,10.606,0.434,15.69,7.38,3275.74,52.333,✔️,4.76,True,GPU -7,🧠⚡,Foundation Model,[Mitra (default) [4.76% IMPUTED]](https://arxiv.org/abs/2510.21204),1417.0,+139/-66,10.924,0.388,17.4,6.94,145.41,3.958,✔️,4.76,True,GPU -8,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1412.0,+107/-82,10.708,0.425,17.69,5.99,4.14,0.557,➖,0.0,False,GPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1401.0,+111/-76,12.422,0.365,18.36,7.57,2791.97,0.544,✔️,0.0,False,GPU -10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1382.0,+100/-80,9.297,0.382,19.48,6.28,6616.87,408.881,✔️,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabICL (default) [4.76% IMPUTED]](https://arxiv.org/abs/2502.05564),1378.0,+81/-87,10.231,0.353,19.74,6.25,7.68,1.811,✔️,4.76,True,GPU -12,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1363.0,+114/-83,14.798,0.314,20.67,12.04,298.33,2.036,➖,0.0,False,GPU -13,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1362.0,+102/-68,13.132,0.315,20.74,10.49,3275.74,0.512,✔️,4.76,True,GPU -14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1361.0,+104/-87,12.578,0.334,20.76,9.2,2237.77,1.273,✔️,0.0,False,GPU -15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1351.0,+96/-92,10.868,0.339,21.43,7.72,6616.87,39.713,✔️,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1332.0,+66/-54,14.378,0.219,22.64,17.52,512.11,1.17,✔️,0.0,False,CPU -17,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1310.0,+81/-60,14.616,0.216,24.02,15.97,4.51,0.124,✔️,0.0,False,CPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1305.0,+138/-85,13.216,0.279,24.36,9.9,4618.5,0.426,✔️,0.0,False,GPU -19,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1296.0,+92/-85,15.188,0.196,24.95,14.11,1640.58,0.552,✔️,0.0,False,CPU -20,🧠⚡,Foundation Model,[TabPFNv2 (default) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1294.0,+109/-93,15.054,0.28,25.14,7.87,4.16,0.543,✔️,4.76,True,GPU -21,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1291.0,+114/-102,13.922,0.256,25.31,14.98,2237.77,0.147,✔️,0.0,False,GPU -22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1290.0,+88/-79,15.671,0.216,25.4,13.7,700.96,0.743,✔️,0.0,False,CPU -23,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1287.0,+80/-61,14.488,0.185,25.57,14.74,11.09,2.46,✔️,0.0,False,GPU -24,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1282.0,+89/-84,14.814,0.194,25.93,16.64,1103.84,0.559,✔️,0.0,False,CPU -25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1274.0,+82/-84,15.114,0.208,26.43,18.06,536.23,1.738,✔️,0.0,False,GPU -26,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1265.0,+79/-77,16.075,0.186,27.05,10.35,700.96,0.203,✔️,0.0,False,CPU -27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1261.0,+77/-68,15.562,0.147,27.29,11.48,1640.58,0.059,✔️,0.0,False,CPU -28,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1259.0,+68/-70,15.646,0.173,27.43,23.57,512.11,0.122,✔️,0.0,False,CPU -29,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1257.0,+114/-91,15.505,0.233,27.55,16.83,4618.5,9.218,✔️,0.0,False,GPU -30,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1253.0,+85/-82,15.521,0.168,27.81,21.05,1103.84,0.051,✔️,0.0,False,CPU -31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1250.0,+90/-86,16.455,0.162,28.0,10.87,7.16,0.046,✔️,0.0,False,CPU -32,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1246.0,+98/-87,16.449,0.219,28.29,17.5,11.3,0.149,✔️,0.0,False,GPU -33,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1245.0,+106/-102,13.544,0.239,28.33,8.98,52.04,39.82,✔️,0.0,False,GPU -34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1230.0,+98/-88,15.554,0.175,29.33,20.98,2535.0,2.546,✔️,0.0,False,CPU -35,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1221.0,+86/-94,16.131,0.137,29.98,21.67,536.23,0.097,✔️,0.0,False,GPU -36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1187.0,+108/-107,18.644,0.133,32.21,10.89,13.39,0.329,✔️,0.0,False,GPU -37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1180.0,+101/-110,18.831,0.147,32.64,20.89,650.63,5.616,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1170.0,+94/-93,17.459,0.146,33.33,25.03,2535.0,0.112,✔️,0.0,False,CPU -39,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1139.0,+87/-99,20.279,0.096,35.26,27.67,206.04,0.946,✔️,0.0,False,CPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1134.0,+104/-151,20.059,0.124,35.62,20.56,650.63,0.442,✔️,0.0,False,CPU -41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1125.0,+83/-84,19.149,0.081,36.14,29.19,1.98,0.122,✔️,0.0,False,CPU -42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1118.0,+108/-111,21.039,0.114,36.62,18.51,206.04,0.098,✔️,0.0,False,CPU -43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1098.0,+74/-90,19.563,0.078,37.79,32.21,2.47,0.168,✔️,0.0,False,CPU -44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1097.0,+93/-108,21.728,0.079,37.86,29.32,494.48,0.936,✔️,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1061.0,+86/-101,22.476,0.028,40.0,34.61,494.48,0.093,✔️,0.0,False,CPU -46,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1051.0,+115/-148,23.895,0.085,40.52,29.4,1.29,0.149,➖,0.0,False,GPU -47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1010.0,+91/-135,23.809,0.044,42.76,35.3,7.76,0.147,✔️,0.0,False,CPU -48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+105/-148,25.296,0.034,43.29,29.07,0.39,0.041,✔️,0.0,False,CPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),976.0,+86/-127,23.191,0.029,44.48,40.11,3.48,0.427,✔️,0.0,False,CPU -50,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),947.0,+100/-143,25.646,0.041,45.86,40.9,2.66,0.702,✔️,0.0,False,GPU -51,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),947.0,+119/-146,26.793,0.023,45.86,40.57,0.33,0.044,✔️,0.0,False,CPU -52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),944.0,+72/-114,26.237,0.0,45.95,44.39,89.61,1.627,✔️,0.0,False,CPU -53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),926.0,+112/-152,29.429,0.048,46.79,14.84,280.42,0.382,✔️,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),900.0,+118/-175,31.785,0.018,47.86,42.54,1.48,0.127,✔️,0.0,False,CPU -55,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),895.0,+118/-156,30.058,0.04,48.05,23.27,280.42,0.072,✔️,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),793.0,+89/-139,31.804,0.0,51.52,50.65,89.61,0.08,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),608.0,+105/-184,43.082,0.0,55.52,55.12,0.24,0.02,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1621.0,+96/-53,0.644,8.1,5.53,6.664,5.53,0.55,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1560.0,+150/-85,0.605,10.38,3.52,5.525,2125.17,9.785,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1537.0,+139/-103,0.583,11.4,4.64,6.141,2125.17,0.882,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1486.0,+126/-93,0.523,13.86,5.19,7.867,613.33,8.472,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1469.0,+80/-85,0.44,14.71,8.7,10.731,2791.97,23.942,✔️,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1460.0,+89/-80,0.434,15.19,10.75,9.953,1886.83,1.333,✔️,0.0,False,CPU +6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1445.0,+143/-73,0.436,16.02,7.52,10.606,3275.74,52.333,✔️,4.76,True,GPU +7,🧠🔁,Neural Network,[Mitra (default) [4.76% IMPUTED]](https://arxiv.org/abs/2510.21204),1416.0,+140/-66,0.388,17.69,6.96,10.924,145.41,3.958,✔️,4.76,True,GPU +8,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1412.0,+106/-80,0.425,17.93,6.01,10.708,4.14,0.557,➖,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1399.0,+108/-73,0.364,18.74,8.09,12.422,2791.97,0.544,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1382.0,+99/-80,0.383,19.76,6.28,9.297,6616.87,408.881,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabICL (default) [4.76% IMPUTED]](https://arxiv.org/abs/2502.05564),1378.0,+80/-86,0.351,20.02,6.28,10.231,7.68,1.811,✔️,4.76,True,GPU +12,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1362.0,+114/-83,0.315,21.05,12.21,14.798,298.33,2.036,➖,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1361.0,+104/-67,0.314,21.1,10.56,13.132,3275.74,0.512,✔️,4.76,True,GPU +14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1361.0,+103/-86,0.333,21.14,9.35,12.578,2237.77,1.273,✔️,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1350.0,+95/-90,0.338,21.86,7.81,10.868,6616.87,39.713,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1331.0,+68/-53,0.221,23.07,17.7,14.378,512.11,1.17,✔️,0.0,False,CPU +17,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1310.0,+78/-59,0.215,24.45,17.2,14.616,4.51,0.124,✔️,0.0,False,CPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1306.0,+138/-85,0.28,24.74,9.94,13.216,4618.5,0.426,✔️,0.0,False,GPU +19,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1296.0,+91/-84,0.194,25.38,14.28,15.188,1640.58,0.552,✔️,0.0,False,CPU +20,🧠⚡,Foundation Model,[TabPFNv2 (default) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1293.0,+109/-90,0.28,25.62,7.9,15.054,4.16,0.543,✔️,4.76,True,GPU +21,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1290.0,+114/-104,0.257,25.83,15.27,13.922,2237.77,0.147,✔️,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1289.0,+86/-78,0.218,25.88,13.89,15.671,700.96,0.743,✔️,0.0,False,CPU +23,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1286.0,+77/-62,0.183,26.1,15.11,14.488,11.09,2.46,✔️,0.0,False,GPU +24,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1283.0,+86/-83,0.192,26.26,17.89,14.814,1103.84,0.559,✔️,0.0,False,CPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1274.0,+83/-85,0.208,26.9,18.29,15.114,536.23,1.738,✔️,0.0,False,GPU +26,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1265.0,+79/-75,0.188,27.52,10.47,16.075,700.96,0.203,✔️,0.0,False,CPU +27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1261.0,+78/-66,0.145,27.76,11.58,15.562,1640.58,0.059,✔️,0.0,False,CPU +28,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1259.0,+68/-71,0.175,27.9,23.79,15.646,512.11,0.122,✔️,0.0,False,CPU +29,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1258.0,+106/-116,0.226,27.98,12.02,16.785,8.99,1.724,➖,0.0,False,GPU +30,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1256.0,+114/-91,0.234,28.07,16.97,15.505,4618.5,9.218,✔️,0.0,False,GPU +31,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1254.0,+84/-78,0.166,28.24,21.28,15.521,1103.84,0.051,✔️,0.0,False,CPU +32,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1249.0,+88/-84,0.161,28.57,11.15,16.455,7.16,0.046,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1246.0,+97/-85,0.22,28.81,17.75,16.449,11.3,0.149,✔️,0.0,False,GPU +34,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1244.0,+107/-102,0.237,28.9,9.03,13.544,52.04,39.82,✔️,0.0,False,GPU +35,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1229.0,+97/-89,0.177,29.95,21.25,15.554,2535.0,2.546,✔️,0.0,False,CPU +36,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1221.0,+86/-95,0.138,30.5,21.94,16.131,536.23,0.097,✔️,0.0,False,GPU +37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1187.0,+108/-106,0.132,32.79,10.94,18.644,13.39,0.329,✔️,0.0,False,GPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1178.0,+100/-110,0.146,33.4,21.89,18.831,650.63,5.616,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1169.0,+94/-91,0.147,34.0,25.6,17.459,2535.0,0.112,✔️,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1140.0,+85/-95,0.094,35.83,27.97,20.279,206.04,0.946,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1133.0,+103/-150,0.123,36.33,20.99,20.059,650.63,0.442,✔️,0.0,False,CPU +42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1126.0,+81/-86,0.082,36.76,29.54,19.149,1.98,0.122,✔️,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1119.0,+110/-111,0.114,37.19,18.64,21.039,206.04,0.098,✔️,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1099.0,+76/-92,0.079,38.45,32.63,19.563,2.47,0.168,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1098.0,+92/-105,0.079,38.52,29.91,21.728,494.48,0.936,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1061.0,+84/-100,0.028,40.71,35.55,22.476,494.48,0.093,✔️,0.0,False,CPU +47,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1050.0,+113/-149,0.084,41.33,30.16,23.895,1.29,0.149,➖,0.0,False,GPU +48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1010.0,+89/-132,0.044,43.57,35.95,23.809,7.76,0.147,✔️,0.0,False,CPU +49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+104/-148,0.034,44.1,31.6,25.296,0.39,0.041,✔️,0.0,False,CPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),976.0,+85/-125,0.03,45.29,41.0,23.191,3.48,0.427,✔️,0.0,False,CPU +51,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),948.0,+119/-145,0.023,46.67,41.36,26.793,0.33,0.044,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),947.0,+71/-116,0.0,46.71,45.09,26.237,89.61,1.627,✔️,0.0,False,CPU +53,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),947.0,+98/-141,0.04,46.71,41.79,25.646,2.66,0.702,✔️,0.0,False,GPU +54,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),925.0,+111/-152,0.048,47.69,14.93,29.429,280.42,0.382,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),902.0,+119/-171,0.019,48.67,43.2,31.785,1.48,0.127,✔️,0.0,False,CPU +56,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),896.0,+119/-156,0.04,48.9,23.45,30.058,280.42,0.072,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),794.0,+88/-140,0.0,52.43,51.57,31.804,89.61,0.08,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),606.0,+105/-182,0.0,56.52,56.13,43.082,0.24,0.02,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/winrate_matrix.png.zip index 90aab53ce8fb826e16c54c297db7489c4ac92ec4..70412a1e13c7b8cc468ba5305156ec34c95a9ab2 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:baab5a61999f6d4b54e898f99fd248358ac7582727315823ff037b02103e8622 -size 2224691 +oid sha256:b5a3eae4b206e7e53912feacf98c69c5a60a95b5923abda9f640376873ca88b6 +size 2177769 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 14023fc6c79e9904baec3842d26277a69dca93fd..f614ae1ea3ae8d4d77d193912f7501f871b12633 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5d6ce6e272b4e836c71bb78b1cb5d125a165250ec2a62b167f214804abd0f95e -size 495281 +oid sha256:4f0883a4c8baf0958c117bdbc31b4dcff46985a03977fc6890bd9b943d51e157 +size 509399 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip index 5c198406cb7f71e78f2411cb8c463e2881bb3d81..923295b1b6c2fdf870893b37f42f2a6ca56e3f47 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:246bf21b0ba13194a86dd66f40b576c97a7c436f97ece80ba84569d5ee70c022 -size 1020703 +oid sha256:bcdd80adc440ec2e8b862068d9f90385dd2cf9f26d188dded3aa0c4dc52c2ac0 +size 990322 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip index f90f7802bdaf9696c349c281e97ec6b3afd15d90..ae48cc0f1dc344e664c1ce3ab031748e115cdb71 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:21075d9a3c478af30efd8f013140ea58d94236bd6cff4b50b01e0895ba12e1b2 -size 118622 +oid sha256:d919f2e3586b4dff162e0e1f535be8e753f3a8c10a067556c486f043bf72139e +size 124432 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/website_leaderboard.csv index 28185c72fd447069c3a0aa2fa1086235606fd735..ad0d5b94633c5368981d7b7cca58e58c3a012162 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1644.0,+93/-60,6.865,0.637,7.8,5.36,5.25,0.517,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1567.0,+153/-98,5.801,0.586,10.65,4.0,2092.56,8.655,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1556.0,+140/-99,6.297,0.575,11.12,4.48,2092.56,0.867,✔️,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1493.0,+130/-78,8.203,0.506,14.1,5.36,598.6,8.421,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1492.0,+91/-82,10.934,0.441,14.2,8.25,2773.9,21.391,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1476.0,+124/-90,10.58,0.456,15.02,7.12,3361.32,65.579,✔️,0.0,False,GPU -6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1471.0,+85/-74,10.37,0.413,15.3,10.76,1881.65,1.26,✔️,0.0,False,CPU -7,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1443.0,+115/-72,10.913,0.407,16.82,6.69,145.75,4.071,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1426.0,+123/-84,10.972,0.419,17.77,5.81,4.09,0.544,➖,0.0,False,GPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1420.0,+120/-99,12.618,0.371,18.18,7.33,2773.9,0.534,✔️,0.0,False,GPU -10,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1402.0,+115/-88,14.276,0.33,19.25,11.6,292.15,1.882,➖,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1401.0,+118/-76,10.185,0.371,19.27,6.01,8.05,1.864,✔️,0.0,False,GPU -12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1399.0,+132/-82,9.42,0.38,19.4,6.06,6408.95,397.026,✔️,0.0,False,GPU -13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1389.0,+117/-88,12.618,0.351,20.05,8.88,2217.3,1.258,✔️,0.0,False,GPU -14,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1384.0,+124/-88,13.232,0.331,20.32,10.16,3361.32,0.527,✔️,0.0,False,GPU -15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1368.0,+131/-93,10.945,0.347,21.3,7.46,6408.95,39.606,✔️,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1334.0,+58/-48,15.024,0.186,23.52,20.03,457.0,1.064,✔️,0.0,False,CPU -17,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1333.0,+110/-76,13.112,0.293,23.58,9.54,4442.23,0.405,✔️,0.0,False,GPU -18,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1320.0,+84/-79,15.346,0.206,24.4,13.69,1618.79,0.502,✔️,0.0,False,CPU -19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1315.0,+112/-89,13.982,0.269,24.72,14.55,2217.3,0.137,✔️,0.0,False,GPU -20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1315.0,+77/-60,15.224,0.186,24.72,16.46,4.49,0.115,✔️,0.0,False,CPU -21,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1314.0,+80/-69,14.446,0.194,24.8,14.28,10.77,2.16,✔️,0.0,False,GPU -22,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1312.0,+104/-98,15.25,0.295,24.95,7.59,3.89,0.544,✔️,0.0,False,GPU -23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1287.0,+83/-75,16.445,0.178,26.58,19.37,673.88,0.681,✔️,0.0,False,CPU -24,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1284.0,+85/-81,15.452,0.162,26.78,17.38,1103.74,0.549,✔️,0.0,False,CPU -25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1281.0,+88/-92,15.662,0.186,27.0,18.24,516.54,1.513,✔️,0.0,False,GPU -26,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1278.0,+83/-91,16.268,0.17,27.2,10.48,6.77,0.046,✔️,0.0,False,CPU -27,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1276.0,+99/-108,15.707,0.245,27.28,16.43,4442.23,8.478,✔️,0.0,False,GPU -28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1275.0,+83/-79,15.789,0.154,27.4,11.18,1618.79,0.055,✔️,0.0,False,CPU -29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1272.0,+120/-112,16.267,0.23,27.55,17.0,10.18,0.149,✔️,0.0,False,GPU -30,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1268.0,+136/-120,13.464,0.251,27.85,8.65,51.78,39.866,✔️,0.0,False,GPU -31,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1264.0,+74/-64,16.249,0.146,28.1,24.41,457.0,0.111,✔️,0.0,False,CPU -32,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1262.0,+76/-68,16.819,0.15,28.2,11.24,673.88,0.177,✔️,0.0,False,CPU -33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1255.0,+98/-87,15.396,0.183,28.7,20.47,2533.55,2.301,✔️,0.0,False,CPU -34,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1252.0,+87/-54,16.217,0.133,28.9,24.07,1103.74,0.05,✔️,0.0,False,CPU -35,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1227.0,+80/-86,16.637,0.12,30.58,21.9,516.54,0.086,✔️,0.0,False,GPU -36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1212.0,+84/-118,18.467,0.139,31.52,10.49,11.78,0.312,✔️,0.0,False,GPU -37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1204.0,+96/-119,18.721,0.154,32.02,20.34,622.79,5.389,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1196.0,+97/-94,17.128,0.153,32.6,24.44,2533.55,0.11,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1157.0,+98/-138,19.935,0.13,35.05,20.0,622.79,0.398,✔️,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1141.0,+94/-87,20.989,0.076,36.08,28.32,204.47,0.853,✔️,0.0,False,CPU -41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1121.0,+117/-112,21.645,0.109,37.3,18.33,204.47,0.091,✔️,0.0,False,CPU -42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1117.0,+81/-93,20.016,0.043,37.55,33.65,1.88,0.121,✔️,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1094.0,+98/-125,22.532,0.057,38.9,30.42,448.77,0.841,✔️,0.0,False,CPU -44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1091.0,+80/-100,20.402,0.043,39.08,35.18,2.39,0.159,✔️,0.0,False,CPU -45,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1078.0,+103/-162,23.197,0.089,39.85,28.74,1.17,0.148,➖,0.0,False,GPU -46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1065.0,+93/-110,23.044,0.03,40.55,34.95,448.77,0.092,✔️,0.0,False,CPU -47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1033.0,+109/-126,23.554,0.046,42.3,34.74,7.36,0.138,✔️,0.0,False,CPU -48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+98/-164,26.004,0.035,44.0,29.07,0.37,0.04,✔️,0.0,False,CPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),997.0,+107/-123,22.94,0.031,44.15,39.68,3.3,0.395,✔️,0.0,False,CPU -50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),963.0,+65/-111,26.281,0.0,45.75,44.14,89.46,1.579,✔️,0.0,False,CPU -51,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),953.0,+121/-156,26.17,0.044,46.2,41.0,2.66,0.553,✔️,0.0,False,GPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),949.0,+114/-129,29.0,0.05,46.38,14.32,276.0,0.368,✔️,0.0,False,CPU -53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),948.0,+121/-161,27.548,0.024,46.45,40.96,0.29,0.041,✔️,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),926.0,+106/-145,31.115,0.019,47.35,41.98,1.45,0.123,✔️,0.0,False,CPU -55,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),920.0,+122/-145,29.565,0.042,47.6,22.6,276.0,0.065,✔️,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),810.0,+84/-146,31.853,0.0,51.45,50.54,89.46,0.075,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),624.0,+110/-269,43.289,0.0,55.5,55.08,0.22,0.02,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1641.0,+95/-63,0.637,7.95,5.4,6.865,5.25,0.517,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1565.0,+152/-97,0.586,10.85,4.02,5.801,2092.56,8.655,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1554.0,+136/-99,0.574,11.32,4.49,6.297,2092.56,0.867,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1491.0,+127/-79,0.504,14.4,5.38,8.203,598.6,8.421,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1490.0,+91/-80,0.441,14.45,8.46,10.934,2773.9,21.391,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1473.0,+126/-90,0.457,15.32,7.25,10.58,3361.32,65.579,✔️,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1468.0,+84/-74,0.412,15.6,11.04,10.37,1881.65,1.26,✔️,0.0,False,CPU +7,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),1442.0,+115/-71,0.408,17.08,6.7,10.913,145.75,4.071,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1426.0,+122/-82,0.419,18.02,5.83,10.972,4.09,0.544,➖,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1418.0,+117/-96,0.369,18.52,7.84,12.618,2773.9,0.534,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1401.0,+117/-74,0.368,19.52,6.04,10.185,8.05,1.864,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1400.0,+114/-90,0.331,19.6,11.77,14.276,292.15,1.882,➖,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1399.0,+133/-83,0.382,19.7,6.07,9.42,6408.95,397.026,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1387.0,+115/-87,0.349,20.4,9.02,12.618,2217.3,1.258,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1383.0,+125/-87,0.33,20.65,10.22,13.232,3361.32,0.527,✔️,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1367.0,+128/-92,0.346,21.7,7.55,10.945,6408.95,39.606,✔️,0.0,False,GPU +16,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1333.0,+107/-76,0.294,23.92,9.58,13.112,4442.23,0.405,✔️,0.0,False,GPU +17,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1332.0,+58/-49,0.188,23.98,20.28,15.024,457.0,1.064,✔️,0.0,False,CPU +18,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1320.0,+82/-77,0.204,24.8,13.85,15.346,1618.79,0.502,✔️,0.0,False,CPU +19,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1314.0,+76/-59,0.186,25.18,17.84,15.224,4.49,0.115,✔️,0.0,False,CPU +20,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1313.0,+110/-88,0.27,25.22,14.83,13.982,2217.3,0.137,✔️,0.0,False,GPU +21,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1312.0,+79/-70,0.192,25.3,14.64,14.446,10.77,2.16,✔️,0.0,False,GPU +22,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1311.0,+103/-98,0.294,25.4,7.62,15.25,3.89,0.544,✔️,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1286.0,+83/-74,0.18,27.08,19.76,16.445,673.88,0.681,✔️,0.0,False,CPU +24,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1285.0,+84/-79,0.16,27.12,18.82,15.452,1103.74,0.549,✔️,0.0,False,CPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1280.0,+87/-92,0.185,27.5,18.49,15.662,516.54,1.513,✔️,0.0,False,GPU +26,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1276.0,+80/-90,0.169,27.75,10.75,16.268,6.77,0.046,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1276.0,+99/-108,0.246,27.78,16.56,15.707,4442.23,8.478,✔️,0.0,False,GPU +28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1274.0,+84/-80,0.152,27.85,11.27,15.789,1618.79,0.055,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1271.0,+120/-110,0.231,28.05,17.24,16.267,10.18,0.149,✔️,0.0,False,GPU +30,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1268.0,+108/-108,0.22,28.28,11.75,17.243,8.92,1.501,➖,0.0,False,GPU +31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1266.0,+137/-121,0.249,28.4,8.7,13.464,51.78,39.866,✔️,0.0,False,GPU +32,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1263.0,+76/-64,0.148,28.6,24.65,16.249,457.0,0.111,✔️,0.0,False,CPU +33,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1262.0,+74/-66,0.153,28.7,11.39,16.819,673.88,0.177,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1253.0,+97/-86,0.185,29.3,20.73,15.396,2533.55,2.301,✔️,0.0,False,CPU +35,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1252.0,+87/-53,0.131,29.35,24.38,16.217,1103.74,0.05,✔️,0.0,False,CPU +36,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1226.0,+80/-86,0.12,31.12,22.18,16.637,516.54,0.086,✔️,0.0,False,GPU +37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1212.0,+83/-118,0.139,32.08,10.53,18.467,11.78,0.312,✔️,0.0,False,GPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1201.0,+96/-120,0.154,32.78,21.33,18.721,622.79,5.389,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1194.0,+98/-95,0.155,33.25,25.0,17.128,2533.55,0.11,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1156.0,+98/-137,0.129,35.75,20.42,19.935,622.79,0.398,✔️,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1141.0,+95/-85,0.075,36.67,28.64,20.989,204.47,0.853,✔️,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1123.0,+117/-109,0.108,37.85,18.44,21.645,204.47,0.091,✔️,0.0,False,CPU +43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1117.0,+81/-93,0.043,38.2,34.13,20.016,1.88,0.121,✔️,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1094.0,+95/-124,0.057,39.6,31.09,22.532,448.77,0.841,✔️,0.0,False,CPU +45,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1091.0,+80/-102,0.045,39.78,35.69,20.402,2.39,0.159,✔️,0.0,False,CPU +46,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1076.0,+103/-158,0.088,40.65,29.49,23.197,1.17,0.148,➖,0.0,False,GPU +47,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1066.0,+92/-107,0.03,41.25,35.88,23.044,448.77,0.092,✔️,0.0,False,CPU +48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1033.0,+109/-126,0.046,43.1,35.38,23.554,7.36,0.138,✔️,0.0,False,CPU +49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+99/-165,0.036,44.8,31.69,26.004,0.37,0.04,✔️,0.0,False,CPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),997.0,+107/-123,0.031,44.95,40.57,22.94,3.3,0.395,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),965.0,+65/-115,0.001,46.5,44.83,26.281,89.46,1.579,✔️,0.0,False,CPU +52,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),953.0,+119/-157,0.042,47.05,41.88,26.17,2.66,0.553,✔️,0.0,False,GPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),948.0,+120/-158,0.025,47.25,41.74,27.548,0.29,0.041,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),948.0,+116/-129,0.05,47.28,14.4,29.0,276.0,0.368,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),928.0,+104/-142,0.02,48.15,42.62,31.115,1.45,0.123,✔️,0.0,False,CPU +56,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),921.0,+122/-141,0.042,48.45,22.77,29.565,276.0,0.065,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),811.0,+79/-142,0.0,52.35,51.45,31.853,89.46,0.075,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),622.0,+109/-264,0.0,56.5,56.09,43.289,0.22,0.02,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip index a38c97493472f144f3cac754be7a9530f1853b29..a7853bbe7ff73c9a849007c6059566c372f9f557 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:52a0c714e4310e9214f6cd09f32813abb9f538deaa4b26a1dbb368b8cab31d36 -size 2250455 +oid sha256:50edab1d7a59d99d1c4c089aebb9de19bbd728d83ad85b6b1ad4f16753c41700 +size 2209336 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index 258f800e1f5f6a56785fe5b110269007e06da238..badc59b20855b8733c7811e1f61d4095cf2084ba 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:57f1be2af2aaddc57542aa565f43865bd4981375bd1b23692d849a1057bc5055 -size 488004 +oid sha256:2d36ecb3b61e6ed27029f6b240bec4f9cf36ae1891bc3cfc074d5129c82a51ca +size 502533 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip index 7f51178b43ecb249d45467145bc7c512a0128ec3..19db188f6989069e1e1d57ff5222377bab90c911 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:61e44cde308ff17f8f809983b69a11d2b30091b184fc7fb3e76aecee5da8bed4 -size 1021494 +oid sha256:98379d451347f5589bc8373d0b48ea4281bb3054fd82b12ffd2cdc792143c933 +size 1009116 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/tuning-impact-elo.png.zip index feee90328ae54403ff2c48e944aa2468ca09da65..f221d6443902a1739151f3f20a65161e038128ee 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:556e7d4f401c6d869bacb8535295606afb961d8936bd3cd5cec58f8d4eec6a64 -size 175033 +oid sha256:4a03d55c11c0b8d801d18c84ca92cb5677bf3460343a448586125d5745773c02 +size 179418 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/website_leaderboard.csv index ea7ee08add81b5e52ede347a0b3eb4f1459328a9..f14790f02699380dff1c1d7ffbc82074306e8e9b 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1590.0,+102/-84,4.83,0.632,9.61,2.92,2063.62,9.938,✔️,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1577.0,+90/-78,7.182,0.643,10.13,3.95,521.51,7.239,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1555.0,+94/-77,7.35,0.577,11.08,5.09,2063.62,1.087,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1537.0,+80/-70,7.714,0.559,11.88,6.08,5.62,0.771,✔️,0.0,False,GPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1525.0,+94/-61,8.832,0.552,12.42,4.74,1815.7,1.456,✔️,0.0,False,CPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1480.0,+56/-49,10.513,0.461,14.68,7.76,2773.9,16.363,✔️,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1415.0,+53/-51,12.166,0.381,18.38,8.94,2773.9,0.39,✔️,0.0,False,GPU -7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1404.0,+75/-60,12.641,0.369,19.08,8.25,2461.81,1.283,✔️,0.0,False,GPU -8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1389.0,+56/-44,13.379,0.297,19.96,14.18,391.23,1.243,✔️,0.0,False,CPU -9,🧠⚡,Foundation Model,[LimiX (default) [15.79% IMPUTED]](https://arxiv.org/abs/2509.03505),1384.0,+73/-74,12.947,0.384,20.29,6.54,3.46,0.517,➖,15.79,True,GPU -10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1373.0,+95/-58,11.63,0.347,21.0,8.39,6067.13,404.225,✔️,0.0,False,GPU -11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1369.0,+72/-56,12.907,0.314,21.22,12.73,1326.58,0.531,✔️,0.0,False,CPU -12,🧠⚡,Foundation Model,[TabICL (default) [5.26% IMPUTED]](https://arxiv.org/abs/2502.05564),1367.0,+79/-72,12.807,0.332,21.37,7.5,9.07,2.008,✔️,5.26,True,GPU -13,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1360.0,+69/-72,13.973,0.284,21.83,12.73,682.62,0.971,✔️,0.0,False,CPU -14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1350.0,+64/-68,13.402,0.3,22.47,11.37,4853.07,0.473,✔️,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1347.0,+66/-56,13.432,0.289,22.67,14.9,1326.58,0.051,✔️,0.0,False,CPU -16,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1346.0,+50/-54,13.55,0.272,22.74,11.73,5.74,0.107,✔️,0.0,False,CPU -17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1344.0,+83/-62,13.609,0.307,22.86,12.22,2461.81,0.138,✔️,0.0,False,GPU -18,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1342.0,+93/-83,13.448,0.326,22.97,9.57,4853.07,9.386,✔️,0.0,False,GPU -19,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1334.0,+93/-80,15.007,0.334,23.51,8.96,2914.83,17.597,✔️,31.58,True,GPU -20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1333.0,+64/-47,14.32,0.249,23.58,17.69,391.23,0.186,✔️,0.0,False,CPU -21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1331.0,+63/-69,14.222,0.256,23.68,11.74,682.62,0.183,✔️,0.0,False,CPU -22,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1323.0,+75/-74,13.66,0.285,24.22,8.48,6067.13,39.565,✔️,0.0,False,GPU -23,🧠⚡,Foundation Model,[Mitra (default) [31.58% IMPUTED]](https://arxiv.org/abs/2510.21204),1316.0,+86/-83,15.339,0.295,24.66,9.93,118.41,2.667,✔️,31.58,True,GPU -24,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1312.0,+66/-58,15.997,0.256,24.92,14.82,250.94,2.036,➖,0.0,False,GPU -25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1310.0,+69/-58,14.71,0.239,25.11,14.61,1053.72,2.067,✔️,0.0,False,GPU -26,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1303.0,+63/-51,15.784,0.207,25.53,15.73,2399.23,0.558,✔️,0.0,False,CPU -27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1282.0,+72/-66,16.264,0.237,26.95,17.43,10.31,0.136,✔️,0.0,False,GPU -28,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1273.0,+76/-76,16.721,0.233,27.54,13.68,2914.83,0.271,✔️,31.58,True,GPU -29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1264.0,+54/-51,15.515,0.169,28.13,21.55,2480.32,2.175,✔️,0.0,False,CPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1260.0,+67/-50,16.137,0.152,28.42,18.64,10.77,1.787,✔️,0.0,False,GPU -31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1258.0,+70/-56,16.368,0.166,28.59,13.64,1053.72,0.105,✔️,0.0,False,GPU -32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1249.0,+66/-51,16.707,0.146,29.18,15.17,2399.23,0.056,✔️,0.0,False,CPU -33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1242.0,+59/-54,17.609,0.147,29.63,14.98,6.77,0.052,✔️,0.0,False,CPU -34,🧠⚡,Foundation Model,[TabPFNv2 (default) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1239.0,+82/-80,17.473,0.216,29.83,10.67,3.3,0.305,✔️,31.58,True,GPU -35,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1212.0,+81/-70,16.869,0.186,31.62,11.33,47.89,39.446,✔️,0.0,False,GPU -36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1210.0,+68/-91,18.74,0.145,31.75,21.46,622.79,5.037,✔️,0.0,False,CPU -37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1194.0,+64/-63,18.948,0.114,32.82,23.16,197.16,0.721,✔️,0.0,False,CPU -38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1193.0,+65/-71,17.316,0.126,32.84,20.53,1.88,0.121,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1192.0,+69/-62,17.8,0.13,32.89,24.82,2480.32,0.117,✔️,0.0,False,CPU -40,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1170.0,+65/-60,19.393,0.092,34.36,15.73,13.35,0.353,✔️,0.0,False,GPU -41,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1167.0,+67/-75,19.874,0.104,34.53,24.3,327.31,0.708,✔️,0.0,False,CPU -42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1155.0,+66/-76,20.247,0.113,35.3,19.39,197.16,0.083,✔️,0.0,False,CPU -43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1142.0,+57/-62,18.628,0.092,36.12,30.52,2.25,0.159,✔️,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1127.0,+68/-78,20.748,0.093,37.03,24.79,622.79,0.346,✔️,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1124.0,+58/-60,20.709,0.056,37.2,30.05,327.31,0.085,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1030.0,+62/-79,23.445,0.028,42.55,37.41,5.73,0.148,✔️,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+61/-86,26.787,0.019,44.07,34.16,0.37,0.038,✔️,0.0,False,CPU -48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),995.0,+74/-102,27.556,0.032,44.29,38.03,140.66,2.876,✔️,0.0,False,CPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),986.0,+72/-69,24.653,0.029,44.76,40.61,3.12,0.334,✔️,0.0,False,CPU -50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),972.0,+82/-138,29.787,0.047,45.39,36.21,1.04,0.192,➖,0.0,False,GPU -51,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),966.0,+90/-91,27.581,0.042,45.66,39.33,3.48,0.677,✔️,0.0,False,GPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),961.0,+83/-99,31.022,0.032,45.88,20.78,256.15,0.36,✔️,0.0,False,CPU -53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),932.0,+89/-110,31.745,0.023,47.13,29.33,256.15,0.07,✔️,0.0,False,CPU -54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),917.0,+77/-108,29.31,0.014,47.74,43.73,0.24,0.037,✔️,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),872.0,+91/-108,34.338,0.01,49.42,45.92,1.32,0.117,✔️,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),819.0,+93/-118,35.052,0.015,51.11,48.48,140.66,0.188,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),540.0,+117/-145,49.668,0.0,56.42,56.17,0.18,0.028,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1589.0,+104/-82,0.635,9.74,2.93,4.83,2063.62,9.938,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1574.0,+91/-78,0.642,10.34,3.97,7.182,521.51,7.239,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1553.0,+97/-78,0.58,11.24,5.12,7.35,2063.62,1.087,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1536.0,+80/-69,0.562,12.04,6.28,7.714,5.62,0.771,✔️,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1523.0,+91/-61,0.553,12.66,4.78,8.832,1815.7,1.456,✔️,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1479.0,+56/-49,0.462,14.89,7.89,10.513,2773.9,16.363,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1413.0,+53/-51,0.381,18.72,9.36,12.166,2773.9,0.39,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1402.0,+77/-58,0.37,19.39,8.33,12.641,2461.81,1.283,✔️,0.0,False,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1388.0,+59/-44,0.3,20.3,14.29,13.379,391.23,1.243,✔️,0.0,False,CPU +9,🧠🔁,Neural Network,[LimiX (default) [15.79% IMPUTED]](https://arxiv.org/abs/2509.03505),1383.0,+73/-75,0.385,20.61,6.62,12.947,3.46,0.517,➖,15.79,True,GPU +10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1372.0,+95/-59,0.351,21.34,8.42,11.63,6067.13,404.225,✔️,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1369.0,+71/-54,0.314,21.51,13.2,12.907,1326.58,0.531,✔️,0.0,False,CPU +12,🧠⚡,Foundation Model,[TabICL (default) [5.26% IMPUTED]](https://arxiv.org/abs/2502.05564),1366.0,+80/-74,0.334,21.71,7.54,12.807,9.07,2.008,✔️,5.26,True,GPU +13,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1358.0,+69/-71,0.287,22.2,12.86,13.973,682.62,0.971,✔️,0.0,False,CPU +14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1349.0,+66/-68,0.303,22.79,11.41,13.402,4853.07,0.473,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1346.0,+66/-55,0.29,23.01,15.02,13.432,1326.58,0.051,✔️,0.0,False,CPU +16,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1345.0,+48/-54,0.276,23.08,12.1,13.55,5.74,0.107,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1342.0,+84/-62,0.309,23.25,12.35,13.609,2461.81,0.138,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1340.0,+95/-82,0.327,23.39,9.6,13.448,4853.07,9.386,✔️,0.0,False,GPU +19,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1332.0,+66/-46,0.252,23.95,17.83,14.32,391.23,0.186,✔️,0.0,False,CPU +20,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1331.0,+92/-78,0.334,23.99,9.13,15.007,2914.83,17.597,✔️,31.58,True,GPU +21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1330.0,+64/-67,0.259,24.08,11.87,14.222,682.62,0.183,✔️,0.0,False,CPU +22,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1321.0,+76/-75,0.286,24.67,8.56,13.66,6067.13,39.565,✔️,0.0,False,GPU +23,🧠🔁,Neural Network,[Mitra (default) [31.58% IMPUTED]](https://arxiv.org/abs/2510.21204),1314.0,+84/-82,0.295,25.13,10.02,15.339,118.41,2.667,✔️,31.58,True,GPU +24,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1312.0,+67/-57,0.26,25.32,15.02,15.997,250.94,2.036,➖,0.0,False,GPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1308.0,+69/-58,0.239,25.55,14.77,14.71,1053.72,2.067,✔️,0.0,False,GPU +26,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1303.0,+63/-52,0.21,25.87,15.87,15.784,2399.23,0.558,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1281.0,+70/-65,0.24,27.42,17.62,16.264,10.31,0.136,✔️,0.0,False,GPU +28,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1271.0,+77/-75,0.232,28.05,14.04,16.721,2914.83,0.271,✔️,31.58,True,GPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1263.0,+56/-52,0.173,28.63,21.8,15.515,2480.32,2.175,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1259.0,+65/-50,0.154,28.92,19.04,16.137,10.77,1.787,✔️,0.0,False,GPU +31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1257.0,+70/-58,0.166,29.01,13.77,16.368,1053.72,0.105,✔️,0.0,False,GPU +32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1249.0,+65/-50,0.147,29.61,15.31,16.707,2399.23,0.056,✔️,0.0,False,CPU +33,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1248.0,+63/-89,0.209,29.67,12.67,17.661,13.62,1.985,➖,0.0,False,GPU +34,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1241.0,+58/-53,0.151,30.13,15.3,17.609,6.77,0.052,✔️,0.0,False,CPU +35,🧠⚡,Foundation Model,[TabPFNv2 (default) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1237.0,+81/-79,0.216,30.41,11.03,17.473,3.3,0.305,✔️,31.58,True,GPU +36,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1211.0,+81/-70,0.187,32.17,11.39,16.869,47.89,39.446,✔️,0.0,False,GPU +37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1208.0,+68/-90,0.146,32.36,22.16,18.74,622.79,5.037,✔️,0.0,False,CPU +38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1194.0,+66/-70,0.13,33.32,20.66,17.316,1.88,0.121,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1193.0,+62/-64,0.115,33.37,23.42,18.948,197.16,0.721,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1191.0,+69/-63,0.134,33.47,25.21,17.8,2480.32,0.117,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1169.0,+65/-61,0.094,34.93,15.83,19.393,13.35,0.353,✔️,0.0,False,GPU +42,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1167.0,+66/-71,0.105,35.08,24.99,19.874,327.31,0.708,✔️,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1154.0,+64/-74,0.114,35.91,19.62,20.247,197.16,0.083,✔️,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1141.0,+58/-62,0.094,36.72,30.92,18.628,2.25,0.159,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1125.0,+68/-78,0.093,37.74,25.37,20.748,622.79,0.346,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1125.0,+59/-60,0.058,37.78,30.86,20.709,327.31,0.085,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1030.0,+60/-80,0.028,43.29,38.01,23.445,5.73,0.148,✔️,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+63/-87,0.019,44.83,36.23,26.787,0.37,0.038,✔️,0.0,False,CPU +49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),995.0,+72/-102,0.032,45.08,38.64,27.556,140.66,2.876,✔️,0.0,False,CPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),984.0,+71/-68,0.03,45.61,41.47,24.653,3.12,0.334,✔️,0.0,False,CPU +51,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),970.0,+82/-134,0.046,46.26,37.07,29.787,1.04,0.192,➖,0.0,False,GPU +52,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),966.0,+89/-92,0.04,46.47,40.22,27.581,3.48,0.677,✔️,0.0,False,GPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),960.0,+83/-101,0.032,46.75,20.96,31.022,256.15,0.36,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),932.0,+89/-108,0.024,47.97,29.64,31.745,256.15,0.07,✔️,0.0,False,CPU +55,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),918.0,+77/-108,0.014,48.55,44.5,29.31,0.24,0.037,✔️,0.0,False,CPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),872.0,+90/-107,0.01,50.29,46.68,34.338,1.32,0.117,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),819.0,+94/-115,0.016,52.0,49.29,35.052,140.66,0.188,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),538.0,+116/-143,0.0,57.42,57.17,49.668,0.18,0.028,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/winrate_matrix.png.zip index c0c23bef329c91b1b2db85cdc6219b478b292983..d85d12260029ebaeb59deff784f6f69783e197cb 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a413a48103ee85da34baa22e24b81a4cb173674a3ff108e90319ba96ee6e04e7 -size 2410880 +oid sha256:b7195c7cd5dcb2fd3418e2c9cc216b72a04129e59d7a8b1f904fb533f8c55568 +size 2373252 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index bc2541c40f01f960ac2694aad9401898acdb23dd..a1a0dcdc051c85c127cf37c6d4b0b0907fc00b8b 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8dce2ac911711eccf79ff9be6354e33cff14b3f36ad02eb84c71e63a9b317b22 -size 444464 +oid sha256:b0f07f2d7cbb830797682325fd1ebcea559e400b0601172fb206754a6d95a9bb +size 458926 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip index 537ed23db875cf43de1dcc0fff06f3fc8593459e..2e6a420b668969da0f013340278d3b7b3222f7f6 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:dcc442e3aafecb3f06c824ca97efb971aa165a3590b58f606e023e22b84da031 -size 975039 +oid sha256:deaa0c7210f6f9c0ef857b251dfac8453ecedc6b694f0fb05078aa9fb2a17438 +size 948663 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/tuning-impact-elo.png.zip index eaf8c900a6bc89a8ea181c7a33fd34c063f74b5e..c1b258e062fd5ec46a5b1c2986f0be600ebd1131 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3696de77e0a739df957caaf9b09aab43f3de8881a09c37fab94769f256ae5b10 -size 133309 +oid sha256:2eadcc74e36c3a1dc16bb28c42c84849e099ff1459b0750fbd570d93b4a54e43 +size 138556 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/website_leaderboard.csv index 3c9a8357f37c04c30f185c15b3568180dbb5e0c0..b9c1e81a254fda3e213b6de2ecdfc587b974296e 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/website_leaderboard.csv @@ -1,55 +1,56 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",2023.0,+176/-73,1.576,0.867,3.3,2.15,307.34,2.186,✔️,0.0,False,CPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1959.0,+218/-71,2.126,0.842,4.2,2.28,266.49,2.963,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1712.0,+263/-137,5.306,0.575,10.4,3.14,780.95,11.64,✔️,0.0,False,GPU -3,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1684.0,+240/-162,5.366,0.601,11.4,4.17,2335.82,14.145,✔️,0.0,False,GPU -4,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1661.0,+156/-106,7.774,0.467,12.3,8.67,202.53,0.43,✔️,0.0,False,CPU -5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1654.0,+240/-119,7.407,0.495,12.6,6.99,780.95,1.493,✔️,0.0,False,GPU -6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1646.0,+177/-92,7.059,0.519,12.9,7.92,688.25,0.192,✔️,0.0,False,CPU -7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1632.0,+173/-79,8.682,0.445,13.5,9.79,91.02,0.568,✔️,0.0,False,CPU -8,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1626.0,+168/-86,7.132,0.494,13.75,8.83,688.25,0.036,✔️,0.0,False,CPU -9,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1615.0,+129/-89,7.485,0.466,14.2,9.63,1635.73,2.146,✔️,0.0,False,GPU -10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1592.0,+183/-102,8.859,0.43,15.2,5.6,1400.65,0.571,✔️,0.0,False,GPU -11,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1590.0,+170/-111,9.79,0.417,15.3,11.13,2335.82,0.502,✔️,0.0,False,GPU -12,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1589.0,+136/-69,7.247,0.459,15.35,6.0,2.2,0.044,✔️,0.0,False,CPU -13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1576.0,+176/-103,8.091,0.414,15.9,11.12,202.53,0.055,✔️,0.0,False,CPU -14,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1557.0,+174/-105,9.535,0.352,16.8,13.01,91.02,0.074,✔️,0.0,False,CPU -15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1525.0,+159/-111,9.894,0.355,18.3,10.13,1635.73,0.082,✔️,0.0,False,GPU -16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1511.0,+196/-166,9.451,0.356,19.0,9.01,1400.65,0.065,✔️,0.0,False,GPU -17,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1502.0,+191/-126,8.488,0.337,19.4,9.54,2.3,0.818,✔️,0.0,False,GPU -18,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1490.0,+167/-135,10.137,0.292,20.0,12.21,8.46,3.879,✔️,0.0,False,GPU -19,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1439.0,+205/-127,12.923,0.253,22.6,15.64,899.24,0.221,✔️,0.0,False,CPU -20,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1435.0,+228/-155,10.324,0.285,22.8,10.66,0.47,0.047,✔️,0.0,False,CPU -21,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1412.0,+111/-69,11.033,0.164,24.0,19.83,1128.73,1.573,✔️,0.0,False,CPU -22,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1400.0,+180/-172,12.728,0.224,24.6,19.84,5.79,0.084,✔️,0.0,False,GPU -23,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1400.0,+137/-131,11.114,0.176,24.6,19.22,4546.1,406.11,✔️,0.0,False,GPU -24,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1393.0,+156/-139,11.586,0.223,25.0,16.3,713.0,2.317,✔️,0.0,False,GPU -25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1364.0,+159/-138,14.611,0.15,26.5,18.58,713.0,0.155,✔️,0.0,False,GPU -26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1340.0,+93/-103,14.578,0.078,27.8,24.94,6.53,0.294,✔️,0.0,False,GPU -27,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1318.0,+162/-158,14.846,0.118,28.95,19.88,4546.1,39.323,✔️,0.0,False,GPU -28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1313.0,+170/-130,14.555,0.141,29.2,22.63,899.24,0.015,✔️,0.0,False,CPU -29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1307.0,+107/-154,15.444,0.101,29.5,23.67,387.73,2.446,✔️,0.0,False,CPU -30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1307.0,+156/-121,15.979,0.11,29.5,26.3,1.95,0.02,✔️,0.0,False,CPU -31,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1305.0,+130/-103,14.301,0.074,29.6,21.99,52.7,0.202,✔️,0.0,False,CPU -32,🧠⚡,Foundation Model,[LimiX (default) [60.00% IMPUTED]](https://arxiv.org/abs/2509.03505),1296.0,+236/-216,19.254,0.196,30.05,7.37,0.46,0.013,➖,60.0,True,GPU -33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1279.0,+119/-104,14.214,0.08,30.9,27.5,1128.73,0.054,✔️,0.0,False,CPU -34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1279.0,+136/-144,13.793,0.06,30.9,27.87,38.59,0.337,✔️,0.0,False,CPU -35,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1276.0,+179/-173,15.572,0.117,31.1,20.39,82.52,0.97,➖,0.0,False,GPU -36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1262.0,+124/-96,14.723,0.057,31.75,26.58,52.7,0.017,✔️,0.0,False,CPU -37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1220.0,+104/-127,13.965,0.042,33.8,31.74,0.33,0.05,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1210.0,+91/-114,14.804,0.016,34.3,32.79,11.03,0.489,✔️,0.0,False,GPU -39,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1184.0,+145/-162,17.278,0.031,35.5,32.53,38.59,0.028,✔️,0.0,False,CPU -40,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1176.0,+131/-135,18.831,0.02,35.85,32.54,41.63,39.259,✔️,0.0,False,GPU -41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1125.0,+100/-144,19.014,0.018,38.1,36.15,387.73,0.089,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1099.0,+96/-108,17.823,0.011,39.2,37.66,1.84,0.047,✔️,0.0,False,CPU -43,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1046.0,+175/-253,28.308,0.017,41.25,37.15,43.41,0.111,✔️,0.0,False,CPU -44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1037.0,+113/-202,24.557,0.018,41.6,38.74,332.4,33.481,✔️,0.0,False,CPU -45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1014.0,+145/-195,22.583,0.024,42.4,39.33,1.63,0.11,✔️,0.0,False,CPU -46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+112/-243,22.62,0.0,42.9,41.5,0.11,0.012,✔️,0.0,False,CPU -47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),998.0,+162/-268,29.28,0.002,42.95,40.08,43.41,0.02,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),837.0,+96/-216,30.878,0.0,47.6,47.26,0.21,0.019,✔️,0.0,False,CPU -49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),837.0,+242/-553,33.515,0.024,47.6,40.42,2.96,0.183,✔️,0.0,False,GPU -50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),828.0,+85/-214,27.598,0.0,47.8,47.52,0.1,0.02,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),792.0,+67/-191,33.031,0.0,48.6,48.26,332.4,1.631,✔️,0.0,False,CPU -52,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),762.0,+124/-263,35.655,0.0,49.2,48.67,0.18,0.032,➖,0.0,False,GPU -53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),424.0,+108/-391,49.897,0.0,53.2,53.17,0.12,0.094,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",2025.0,+174/-74,0.868,3.3,2.15,1.576,307.34,2.186,✔️,0.0,False,CPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1961.0,+218/-69,0.844,4.2,2.28,2.126,266.49,2.963,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1712.0,+268/-137,0.578,10.5,3.14,5.306,780.95,11.64,✔️,0.0,False,GPU +3,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1684.0,+242/-165,0.603,11.5,4.17,5.366,2335.82,14.145,✔️,0.0,False,GPU +4,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1664.0,+156/-105,0.474,12.3,8.67,7.774,202.53,0.43,✔️,0.0,False,CPU +5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1654.0,+243/-120,0.499,12.7,7.0,7.407,780.95,1.493,✔️,0.0,False,GPU +6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1649.0,+176/-93,0.524,12.9,7.92,7.059,688.25,0.192,✔️,0.0,False,CPU +7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1632.0,+179/-81,0.451,13.6,9.81,8.682,91.02,0.568,✔️,0.0,False,CPU +8,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1628.0,+167/-84,0.499,13.75,8.83,7.132,688.25,0.036,✔️,0.0,False,CPU +9,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1618.0,+129/-88,0.471,14.2,9.63,7.485,1635.73,2.146,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1593.0,+187/-100,0.433,15.3,5.61,8.859,1400.65,0.571,✔️,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1592.0,+135/-68,0.465,15.35,6.0,7.247,2.2,0.044,✔️,0.0,False,CPU +12,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1590.0,+172/-116,0.422,15.4,11.14,9.79,2335.82,0.502,✔️,0.0,False,GPU +13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1577.0,+179/-102,0.419,16.0,11.17,8.091,202.53,0.055,✔️,0.0,False,CPU +14,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1558.0,+178/-110,0.356,16.9,13.03,9.535,91.02,0.074,✔️,0.0,False,CPU +15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1524.0,+160/-112,0.36,18.5,10.15,9.894,1635.73,0.082,✔️,0.0,False,GPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1512.0,+199/-166,0.36,19.1,9.02,9.451,1400.65,0.065,✔️,0.0,False,GPU +17,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1504.0,+190/-128,0.346,19.5,9.54,8.488,2.3,0.818,✔️,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1490.0,+168/-135,0.299,20.2,12.28,10.137,8.46,3.879,✔️,0.0,False,GPU +19,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1441.0,+202/-125,0.258,22.7,15.66,12.923,899.24,0.221,✔️,0.0,False,CPU +20,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1437.0,+230/-155,0.289,22.9,10.66,10.324,0.47,0.047,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1414.0,+110/-66,0.169,24.1,19.95,11.033,1128.73,1.573,✔️,0.0,False,CPU +22,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1400.0,+140/-132,0.184,24.9,19.38,11.114,4546.1,406.11,✔️,0.0,False,GPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1400.0,+183/-172,0.228,24.9,19.92,12.728,5.79,0.084,✔️,0.0,False,GPU +24,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1392.0,+158/-142,0.225,25.3,16.35,11.586,713.0,2.317,✔️,0.0,False,GPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1368.0,+159/-138,0.151,26.6,18.6,14.611,713.0,0.155,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1340.0,+98/-105,0.086,28.1,25.09,14.578,6.53,0.294,✔️,0.0,False,GPU +27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1317.0,+170/-127,0.146,29.3,22.65,14.555,899.24,0.015,✔️,0.0,False,CPU +28,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1316.0,+163/-162,0.12,29.35,19.99,14.846,4546.1,39.323,✔️,0.0,False,GPU +29,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1310.0,+158/-122,0.116,29.7,26.36,15.979,1.95,0.02,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1308.0,+106/-155,0.102,29.8,23.76,15.444,387.73,2.446,✔️,0.0,False,CPU +31,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1306.0,+128/-99,0.075,29.9,22.16,14.301,52.7,0.202,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[LimiX (default) [60.00% IMPUTED]](https://arxiv.org/abs/2509.03505),1296.0,+234/-217,0.196,30.45,7.39,19.254,0.46,0.013,➖,60.0,True,GPU +33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1281.0,+120/-104,0.086,31.2,27.63,14.214,1128.73,0.054,✔️,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1277.0,+139/-150,0.064,31.4,28.17,13.793,38.59,0.337,✔️,0.0,False,CPU +35,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1277.0,+178/-170,0.117,31.4,20.47,15.572,82.52,0.97,➖,0.0,False,GPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1264.0,+126/-100,0.058,32.05,26.73,14.723,52.7,0.017,✔️,0.0,False,CPU +37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1219.0,+107/-128,0.046,34.3,32.05,13.965,0.33,0.05,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1209.0,+94/-118,0.023,34.8,33.11,14.804,11.03,0.489,✔️,0.0,False,GPU +39,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1197.0,+108/-106,0.026,35.4,32.02,17.798,14.93,13.661,➖,0.0,False,GPU +40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1180.0,+146/-165,0.034,36.2,33.01,17.278,38.59,0.028,✔️,0.0,False,CPU +41,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1177.0,+131/-136,0.021,36.35,32.85,18.831,41.63,39.259,✔️,0.0,False,GPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1122.0,+104/-148,0.019,38.8,36.69,19.014,387.73,0.089,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1099.0,+97/-111,0.013,39.8,38.11,17.823,1.84,0.047,✔️,0.0,False,CPU +44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1042.0,+176/-254,0.019,42.05,37.77,28.308,43.41,0.111,✔️,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1028.0,+113/-202,0.018,42.6,39.86,24.557,332.4,33.481,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1008.0,+146/-197,0.025,43.3,40.03,22.583,1.63,0.11,✔️,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+115/-242,0.0,43.6,42.09,22.62,0.11,0.012,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),996.0,+164/-266,0.002,43.75,40.73,29.28,43.41,0.02,✔️,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),834.0,+238/-559,0.024,48.5,41.71,33.515,2.96,0.183,✔️,0.0,False,GPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),830.0,+96/-217,0.0,48.6,48.27,30.878,0.21,0.019,✔️,0.0,False,CPU +51,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),825.0,+89/-219,0.0,48.7,48.4,27.598,0.1,0.02,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),785.0,+67/-189,0.0,49.6,49.27,33.031,332.4,1.631,✔️,0.0,False,CPU +53,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),755.0,+125/-262,0.0,50.2,49.68,35.655,0.18,0.032,➖,0.0,False,GPU +54,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),419.0,+107/-394,0.0,54.2,54.17,49.897,0.12,0.094,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/winrate_matrix.png.zip index 41b09f8b8fed655f1ce918139e9a2083d1bcbbb2..4fd812934b283667500a6e0e7bd42d6d2cbf770f 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b6fefa7813c3765b91e4fffb19866742ff952318f962d00c019381b21b958f27 -size 1795058 +oid sha256:4a79c4e06bf8fd85d19e771092bf3441b3179292732cae1364a0b72817a9f0b3 +size 1920407 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index a50c399ce5ad735254f462395c82662a04bad72b..8e6b008e6975ba789c03ab59027c76094ac71fe4 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:10a7198765b50a0cb683b00e28ad2a4fa8aeccae2fe0d83aea7cfdd6e6f14280 -size 473361 +oid sha256:28e84987d410dac77f88b8fb220fb5ba1d2bda781e198d059af2481482e7fe59 +size 487806 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip index 1456b35535e51545230faa4072a2bbc548b13bc5..97e1055a95dbe7f4cabc720af6decc28adf52ee2 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5d78bdd8bae05a2ba7d11979fd2de96071783b2325367e044f678aacf92600ad -size 1031428 +oid sha256:94be942cd62c160ec0c590941542f36505a9cfdcce279befed6997f87914c8fc +size 1028991 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/tuning-impact-elo.png.zip index c52aea9e00f56ddf6e84741a932cbd0f8cabbad9..bdd28e1f87154ef4b799b09bbe91e577e798f21d 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8a8ff0e7c0fabb1c4e5c23a75048ffb83e3da4ced9015351dfb48d575802da44 -size 179441 +oid sha256:6852b864e6684eedc581d0d9ee250c681e295843d4862f6a643c3c7509096fae +size 189324 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/website_leaderboard.csv index 566bd9e01dc146fbd7bc9f4e129039e8490607b3..915e90202cce7c701baa16130d893c47de73f952 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1603.0,+95/-74,7.437,0.618,9.05,5.38,6.03,0.703,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1596.0,+129/-95,4.66,0.646,9.32,2.85,2279.85,9.666,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1565.0,+125/-84,7.33,0.598,10.54,4.64,2279.85,0.968,✔️,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1526.0,+107/-86,8.987,0.567,12.25,5.35,635.11,8.569,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1476.0,+84/-64,11.594,0.445,14.71,7.26,3486.44,22.867,✔️,0.0,False,GPU -5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1459.0,+89/-66,11.423,0.434,15.68,8.29,2993.17,1.401,✔️,0.0,False,CPU -6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1457.0,+132/-93,12.285,0.454,15.79,6.98,3226.71,49.21,✔️,7.14,True,GPU -7,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1446.0,+87/-62,10.694,0.449,16.38,6.27,4.53,0.611,➖,0.0,False,GPU -8,🧠⚡,Foundation Model,[Mitra (default) [7.14% IMPUTED]](https://arxiv.org/abs/2510.21204),1427.0,+107/-80,12.738,0.401,17.46,7.78,145.75,4.071,✔️,7.14,True,GPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1416.0,+74/-72,12.977,0.375,18.12,8.57,3486.44,1.002,✔️,0.0,False,GPU -10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1389.0,+84/-72,11.814,0.387,19.71,6.98,7432.95,404.225,✔️,0.0,False,GPU -11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1377.0,+76/-76,13.992,0.34,20.46,9.93,3083.36,1.48,✔️,0.0,False,GPU -12,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1365.0,+108/-81,14.612,0.316,21.25,10.98,3226.71,0.494,✔️,7.14,True,GPU -13,🧠⚡,Foundation Model,[TabICL (default) [7.14% IMPUTED]](https://arxiv.org/abs/2502.05564),1355.0,+72/-64,13.761,0.329,21.86,6.59,9.07,1.864,✔️,7.14,True,GPU -14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1349.0,+62/-52,15.057,0.233,22.27,16.88,774.51,1.455,✔️,0.0,False,CPU -15,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1348.0,+96/-74,16.148,0.289,22.29,13.46,303.47,2.324,➖,0.0,False,GPU -16,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1348.0,+85/-89,13.237,0.341,22.32,7.03,7432.95,39.797,✔️,0.0,False,GPU -17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1320.0,+89/-67,15.094,0.281,24.09,13.98,3083.36,0.189,✔️,0.0,False,GPU -18,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1319.0,+71/-61,14.996,0.228,24.2,16.26,1612.87,0.724,✔️,0.0,False,CPU -19,🧠⚡,Foundation Model,[TabPFNv2 (default) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1314.0,+94/-71,15.634,0.293,24.48,8.41,4.32,0.469,✔️,7.14,True,GPU -20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1307.0,+78/-74,15.826,0.241,25.0,14.07,1165.19,2.067,✔️,0.0,False,GPU -21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1306.0,+82/-77,14.692,0.246,25.04,11.45,6286.73,0.473,✔️,0.0,False,GPU -22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1303.0,+68/-70,16.187,0.205,25.23,15.29,829.32,1.642,✔️,0.0,False,CPU -23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1301.0,+57/-56,15.801,0.188,25.38,17.8,6.97,0.144,✔️,0.0,False,CPU -24,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1294.0,+67/-47,15.683,0.202,25.86,19.76,1612.87,0.061,✔️,0.0,False,CPU -25,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1291.0,+70/-66,16.028,0.199,26.0,20.3,774.51,0.282,✔️,0.0,False,CPU -26,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1287.0,+72/-64,16.805,0.178,26.29,15.72,2989.36,0.609,✔️,0.0,False,CPU -27,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1284.0,+69/-67,16.412,0.186,26.46,11.97,829.32,0.241,✔️,0.0,False,CPU -28,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1275.0,+83/-79,16.335,0.222,27.11,17.78,6286.73,9.386,✔️,0.0,False,GPU -29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1269.0,+81/-80,17.527,0.229,27.5,16.66,11.86,0.149,✔️,0.0,False,GPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1256.0,+70/-70,16.694,0.157,28.36,17.05,14.59,4.035,✔️,0.0,False,GPU -31,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1248.0,+60/-67,17.476,0.138,28.89,13.54,2989.36,0.063,✔️,0.0,False,CPU -32,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1246.0,+78/-82,16.995,0.17,29.05,12.43,1165.19,0.105,✔️,0.0,False,GPU -33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1240.0,+80/-64,18.192,0.145,29.39,12.97,8.12,0.053,✔️,0.0,False,CPU -34,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1238.0,+86/-106,16.168,0.24,29.54,9.17,51.78,39.927,✔️,0.0,False,GPU -35,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1237.0,+81/-73,17.116,0.165,29.61,22.23,2894.94,2.42,✔️,0.0,False,CPU -36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1197.0,+102/-91,19.917,0.158,32.27,20.71,1425.28,7.355,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1180.0,+87/-76,19.081,0.141,33.32,23.9,2894.94,0.145,✔️,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1178.0,+83/-85,20.789,0.129,33.5,21.84,402.38,1.373,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1166.0,+69/-85,21.032,0.102,34.23,13.26,14.22,0.353,✔️,0.0,False,GPU -40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1155.0,+84/-85,21.307,0.139,34.95,16.9,402.38,0.163,✔️,0.0,False,CPU -41,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1136.0,+80/-98,21.865,0.113,36.14,25.19,562.0,1.155,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1135.0,+91/-92,21.368,0.117,36.21,22.19,1425.28,0.561,✔️,0.0,False,CPU -43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1134.0,+67/-66,19.814,0.061,36.29,30.61,2.85,0.19,✔️,0.0,False,CPU -44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1128.0,+64/-81,20.294,0.095,36.66,29.97,3.27,0.193,✔️,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1088.0,+72/-83,22.847,0.054,39.0,31.4,562.0,0.138,✔️,0.0,False,CPU -46,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1012.0,+95/-122,27.691,0.064,43.18,32.86,2.45,0.519,➖,0.0,False,GPU -47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1010.0,+82/-100,25.453,0.033,43.32,37.04,9.06,0.196,✔️,0.0,False,CPU -48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1002.0,+96/-132,25.461,0.048,43.68,38.27,5.43,0.836,✔️,0.0,False,GPU -49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+85/-113,28.276,0.025,43.8,31.74,0.46,0.07,✔️,0.0,False,CPU -50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),979.0,+92/-121,28.627,0.037,44.82,37.52,123.01,1.579,✔️,0.0,False,CPU -51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),978.0,+84/-106,25.392,0.029,44.89,40.58,5.03,0.601,✔️,0.0,False,CPU -52,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),944.0,+94/-123,29.921,0.018,46.43,41.59,0.38,0.072,✔️,0.0,False,CPU -53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),934.0,+89/-116,31.991,0.036,46.82,17.87,479.28,0.495,✔️,0.0,False,CPU -54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),908.0,+89/-126,32.625,0.03,47.91,26.58,479.28,0.098,✔️,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),873.0,+104/-131,35.574,0.014,49.21,44.74,2.34,0.132,✔️,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),809.0,+108/-156,35.773,0.02,51.29,47.97,123.01,0.092,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),565.0,+114/-194,49.586,0.0,56.14,55.81,0.28,0.024,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1600.0,+94/-70,0.618,9.23,5.6,7.437,6.03,0.703,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1594.0,+128/-95,0.648,9.46,2.86,4.66,2279.85,9.666,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1563.0,+124/-83,0.599,10.71,4.67,7.33,2279.85,0.968,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1522.0,+104/-87,0.565,12.54,5.39,8.987,635.11,8.569,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1474.0,+83/-61,0.444,15.0,7.4,11.594,3486.44,22.867,✔️,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1456.0,+87/-65,0.435,16.0,8.46,11.423,2993.17,1.401,✔️,0.0,False,CPU +6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1452.0,+130/-91,0.454,16.18,7.11,12.285,3226.71,49.21,✔️,7.14,True,GPU +7,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1444.0,+86/-63,0.45,16.66,6.37,10.694,4.53,0.611,➖,0.0,False,GPU +8,🧠🔁,Neural Network,[Mitra (default) [7.14% IMPUTED]](https://arxiv.org/abs/2510.21204),1424.0,+104/-79,0.4,17.86,7.86,12.738,145.75,4.071,✔️,7.14,True,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1413.0,+71/-70,0.373,18.52,9.09,12.977,3486.44,1.002,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1388.0,+85/-73,0.388,20.07,7.0,11.814,7432.95,404.225,✔️,0.0,False,GPU +11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1375.0,+75/-76,0.338,20.86,10.09,13.992,3083.36,1.48,✔️,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1362.0,+108/-81,0.315,21.7,11.28,14.612,3226.71,0.494,✔️,7.14,True,GPU +13,🧠⚡,Foundation Model,[TabICL (default) [7.14% IMPUTED]](https://arxiv.org/abs/2502.05564),1353.0,+72/-65,0.326,22.25,6.62,13.761,9.07,1.864,✔️,7.14,True,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1347.0,+62/-51,0.235,22.7,17.07,15.057,774.51,1.455,✔️,0.0,False,CPU +15,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1346.0,+97/-72,0.291,22.71,13.68,16.148,303.47,2.324,➖,0.0,False,GPU +16,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1345.0,+83/-88,0.341,22.79,7.1,13.237,7432.95,39.797,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1318.0,+88/-67,0.28,24.59,14.21,15.094,3083.36,0.189,✔️,0.0,False,GPU +18,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1318.0,+69/-59,0.225,24.59,17.33,14.996,1612.87,0.724,✔️,0.0,False,CPU +19,🧠⚡,Foundation Model,[TabPFNv2 (default) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1312.0,+92/-71,0.293,25.02,8.7,15.634,4.32,0.469,✔️,7.14,True,GPU +20,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1306.0,+81/-78,0.246,25.43,11.51,14.692,6286.73,0.473,✔️,0.0,False,GPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1304.0,+79/-73,0.24,25.5,14.26,15.826,1165.19,2.067,✔️,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1301.0,+65/-70,0.205,25.73,15.54,16.187,829.32,1.642,✔️,0.0,False,CPU +23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1299.0,+56/-54,0.189,25.84,19.02,15.801,6.97,0.144,✔️,0.0,False,CPU +24,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1292.0,+68/-44,0.2,26.32,20.04,15.683,1612.87,0.061,✔️,0.0,False,CPU +25,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1290.0,+70/-65,0.201,26.46,20.53,16.028,774.51,0.282,✔️,0.0,False,CPU +26,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1286.0,+72/-62,0.178,26.71,15.9,16.805,2989.36,0.609,✔️,0.0,False,CPU +27,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1283.0,+68/-64,0.186,26.96,12.15,16.412,829.32,0.241,✔️,0.0,False,CPU +28,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1279.0,+85/-91,0.274,27.2,10.4,17.612,13.4,1.64,➖,0.0,False,GPU +29,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1273.0,+82/-78,0.221,27.64,17.96,16.335,6286.73,9.386,✔️,0.0,False,GPU +30,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1267.0,+80/-81,0.23,28.04,16.89,17.527,11.86,0.149,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1254.0,+71/-71,0.155,28.93,17.48,16.694,14.59,4.035,✔️,0.0,False,GPU +32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1247.0,+60/-67,0.138,29.43,13.69,17.476,2989.36,0.063,✔️,0.0,False,CPU +33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1244.0,+78/-82,0.17,29.59,12.57,16.995,1165.19,0.105,✔️,0.0,False,GPU +34,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1238.0,+79/-64,0.146,30.0,13.28,18.192,8.12,0.053,✔️,0.0,False,CPU +35,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1237.0,+88/-107,0.239,30.11,9.21,16.168,51.78,39.927,✔️,0.0,False,GPU +36,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1234.0,+81/-72,0.167,30.25,22.55,17.116,2894.94,2.42,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1194.0,+101/-90,0.158,32.98,21.58,19.917,1425.28,7.355,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1178.0,+87/-76,0.143,34.0,24.35,19.081,2894.94,0.145,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1178.0,+81/-83,0.127,34.07,22.09,20.789,402.38,1.373,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1166.0,+70/-85,0.101,34.84,13.33,21.032,14.22,0.353,✔️,0.0,False,GPU +41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1155.0,+83/-84,0.138,35.52,17.09,21.307,402.38,0.163,✔️,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1136.0,+78/-98,0.114,36.79,26.12,21.865,562.0,1.155,✔️,0.0,False,CPU +43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1134.0,+68/-66,0.061,36.89,31.01,19.814,2.85,0.19,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1133.0,+90/-90,0.115,36.93,22.75,21.368,1425.28,0.561,✔️,0.0,False,CPU +45,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1127.0,+62/-81,0.096,37.3,30.39,20.294,3.27,0.193,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1089.0,+69/-82,0.055,39.68,32.55,22.847,562.0,0.138,✔️,0.0,False,CPU +47,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1012.0,+95/-121,0.063,44.0,33.67,27.691,2.45,0.519,➖,0.0,False,GPU +48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1009.0,+81/-102,0.033,44.11,37.69,25.453,9.06,0.196,✔️,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1002.0,+96/-130,0.045,44.46,39.03,25.461,5.43,0.836,✔️,0.0,False,GPU +50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+83/-112,0.026,44.59,34.08,28.276,0.46,0.07,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),981.0,+92/-121,0.037,45.54,37.96,28.627,123.01,1.579,✔️,0.0,False,CPU +52,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),977.0,+84/-106,0.029,45.71,41.51,25.392,5.03,0.601,✔️,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),945.0,+95/-124,0.019,47.21,42.32,29.921,0.38,0.072,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),933.0,+89/-114,0.036,47.71,17.99,31.991,479.28,0.495,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),908.0,+90/-123,0.03,48.77,26.83,32.625,479.28,0.098,✔️,0.0,False,CPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),875.0,+105/-131,0.014,50.04,45.44,35.574,2.34,0.132,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),811.0,+108/-152,0.021,52.14,48.72,35.773,123.01,0.092,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),563.0,+112/-194,0.0,57.14,56.82,49.586,0.28,0.024,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/winrate_matrix.png.zip index bf53bf2673e41905932c647be0d77e6b2f6fd592..7d9cbbaa7a101289aea3a35416fdecfc3f703096 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4009c2691e3a3a767625bde867c12c6f3b19d60e22b8c8e88b81226da5967ca2 -size 2402746 +oid sha256:887f178fb9032c637fa57d89fa620e8a8c2a1f1a9bbf4f7b8ce3cebdbdc4848d +size 2349782 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 5ef1882182376979e2af8064911f4df016c7156f..0c06a27314305ef9a19180907e5c6cf42c5e14c8 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3660b9a319ecbb36b1bc8d75e5c01c63b3a33a712f8025022e7ff2c27fb888f4 -size 481575 +oid sha256:47345199e71c7618aa1230de53db11930b9d5e26c9af8e0cf5c3512ad0c0ae3d +size 496077 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip index c5d35aba0b8430d6c4c66e3b77bf9a03e78544db..66ed38dc416d982179a2dc322129d13d759eb3d8 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:66ac13813be4eaa8dbc496584a35acd482640c60a3b49abcd863aff40c9fafcc -size 1023492 +oid sha256:246b2f715d4c64aea9b268b35efbb5901aa066f8491390520281d25f53d47357 +size 1015404 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip index ea17501ee0a4ce45fe7f4aa59f86c57319feb69d..e85a834a0a05365c08d360d007a150e72120c335 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:eb2ec143bac9618492218a8f500c0fb3ecc5f0c4f2e388985062f3d813b2871e -size 121062 +oid sha256:85fb8b83a6c0d7da0be7b54d9d794fc1b89fb90c16fa588a122f166012dd981f +size 126122 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/website_leaderboard.csv index 1fe395cd30ca8ffe789f7e6c60ff47aaab3d9097..7864d15319412f0ba8ab42040dc2f659f7abf2d7 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1648.0,+88/-68,7.839,0.635,8.1,5.12,5.78,0.586,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1624.0,+124/-100,4.968,0.653,8.92,2.96,2162.76,8.93,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1601.0,+124/-104,7.727,0.61,9.81,4.39,2162.76,0.894,✔️,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1533.0,+113/-82,9.618,0.548,12.73,5.43,620.85,8.569,✔️,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1510.0,+93/-63,12.159,0.463,13.81,6.88,3159.9,21.159,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1509.0,+107/-89,11.53,0.488,13.87,6.57,3361.32,51.669,✔️,0.0,False,GPU -6,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1472.0,+108/-68,12.029,0.432,15.79,7.34,150.4,4.403,✔️,0.0,False,GPU -7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1467.0,+82/-70,12.218,0.41,16.08,8.22,2628.27,1.401,✔️,0.0,False,CPU -8,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1460.0,+100/-66,11.286,0.439,16.44,6.0,4.3,0.584,➖,0.0,False,GPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1446.0,+89/-57,13.535,0.394,17.25,8.14,3159.9,0.646,✔️,0.0,False,GPU -10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1403.0,+84/-80,14.036,0.341,19.75,10.41,3361.32,0.527,✔️,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1403.0,+103/-85,12.433,0.381,19.77,6.66,6883.9,405.331,✔️,0.0,False,GPU -12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1392.0,+106/-73,14.604,0.335,20.42,9.82,2899.02,1.461,✔️,0.0,False,GPU -13,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1390.0,+84/-96,13.131,0.354,20.52,6.2,9.07,2.008,✔️,0.0,False,GPU -14,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1377.0,+96/-83,16.382,0.298,21.31,12.92,303.47,2.098,➖,0.0,False,GPU -15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1360.0,+81/-99,13.867,0.341,22.42,6.69,6883.9,39.606,✔️,0.0,False,GPU -16,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1345.0,+83/-90,15.148,0.316,23.35,7.94,4.28,0.511,✔️,0.0,False,GPU -17,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1345.0,+53/-41,16.138,0.192,23.37,19.01,691.46,1.455,✔️,0.0,False,CPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1336.0,+73/-88,15.115,0.265,23.88,10.86,6020.89,0.454,✔️,0.0,False,GPU -19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1330.0,+102/-70,15.756,0.27,24.33,14.64,2899.02,0.171,✔️,0.0,False,GPU -20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1326.0,+85/-72,16.007,0.213,24.52,16.47,1394.45,0.618,✔️,0.0,False,CPU -21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1308.0,+69/-57,17.318,0.182,25.71,19.95,785.9,1.281,✔️,0.0,False,CPU -22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1306.0,+76/-53,16.868,0.169,25.83,18.07,5.74,0.139,✔️,0.0,False,CPU -23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1305.0,+73/-64,17.587,0.185,25.92,15.16,2399.23,0.565,✔️,0.0,False,CPU -24,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1300.0,+79/-73,16.877,0.2,26.27,18.27,1053.72,1.872,✔️,0.0,False,GPU -25,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1297.0,+91/-84,17.031,0.239,26.42,17.12,6020.89,8.478,✔️,0.0,False,GPU -26,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1293.0,+74/-51,16.785,0.173,26.69,21.51,1394.45,0.053,✔️,0.0,False,CPU -27,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1290.0,+72/-60,17.51,0.166,26.88,12.61,785.9,0.213,✔️,0.0,False,CPU -28,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1288.0,+71/-52,17.102,0.163,27.0,21.23,691.46,0.264,✔️,0.0,False,CPU -29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1285.0,+102/-73,18.07,0.23,27.19,16.18,11.6,0.148,✔️,0.0,False,GPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1284.0,+83/-65,17.108,0.17,27.27,16.3,12.75,3.502,✔️,0.0,False,GPU -31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1257.0,+98/-127,16.756,0.259,29.04,8.67,51.52,40.126,✔️,0.0,False,GPU -32,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1254.0,+90/-93,18.783,0.139,29.23,12.47,7.41,0.052,✔️,0.0,False,CPU -33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1254.0,+60/-73,18.357,0.136,29.27,13.11,2399.23,0.061,✔️,0.0,False,CPU -34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1253.0,+73/-74,17.668,0.169,29.31,21.75,2792.14,2.175,✔️,0.0,False,CPU -35,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1234.0,+85/-76,18.071,0.126,30.56,21.73,1053.72,0.103,✔️,0.0,False,GPU -36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1221.0,+93/-108,20.456,0.17,31.37,19.9,1350.04,6.57,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1191.0,+72/-113,21.041,0.109,33.29,12.56,13.67,0.339,✔️,0.0,False,GPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1183.0,+80/-82,19.614,0.119,33.81,26.42,2792.14,0.133,✔️,0.0,False,CPU -39,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1160.0,+84/-81,22.147,0.086,35.21,27.55,378.34,1.373,✔️,0.0,False,CPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1154.0,+96/-109,21.966,0.126,35.58,21.37,1350.04,0.613,✔️,0.0,False,CPU -41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1137.0,+91/-92,22.595,0.107,36.62,19.58,378.34,0.154,✔️,0.0,False,CPU -42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1128.0,+60/-59,21.116,0.033,37.19,33.98,2.66,0.17,✔️,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1118.0,+107/-106,23.308,0.078,37.77,26.65,527.27,1.155,✔️,0.0,False,CPU -44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1103.0,+69/-69,21.73,0.047,38.63,34.62,3.01,0.177,✔️,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1079.0,+91/-89,24.138,0.045,40.04,32.03,527.27,0.128,✔️,0.0,False,CPU -46,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1029.0,+86/-152,28.036,0.069,42.69,32.05,2.28,0.454,➖,0.0,False,GPU -47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1013.0,+91/-94,26.238,0.036,43.5,36.97,8.93,0.172,✔️,0.0,False,CPU -48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+86/-108,28.762,0.027,44.15,31.42,0.44,0.065,✔️,0.0,False,CPU -49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),999.0,+93/-122,26.76,0.051,44.19,38.38,5.0,0.753,✔️,0.0,False,GPU -50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),990.0,+94/-109,25.606,0.031,44.62,40.08,4.68,0.547,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),959.0,+80/-107,29.843,0.009,46.04,43.24,118.4,1.479,✔️,0.0,False,CPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),956.0,+94/-99,31.197,0.038,46.17,16.98,429.17,0.495,✔️,0.0,False,CPU -53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),946.0,+87/-146,30.28,0.02,46.62,41.57,0.36,0.068,✔️,0.0,False,CPU -54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),930.0,+99/-111,31.807,0.032,47.27,25.54,429.17,0.081,✔️,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),897.0,+108/-120,34.214,0.015,48.58,43.99,2.16,0.129,✔️,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),766.0,+80/-151,37.314,0.0,52.58,51.76,118.4,0.082,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),576.0,+118/-291,49.46,0.0,56.08,55.73,0.27,0.024,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1643.0,+86/-65,0.635,8.29,5.33,7.839,5.78,0.586,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1621.0,+125/-100,0.652,9.08,2.97,4.968,2162.76,8.93,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1597.0,+126/-103,0.609,10.0,4.42,7.727,2162.76,0.894,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1528.0,+113/-80,0.546,13.04,5.48,9.618,620.85,8.569,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1506.0,+86/-60,0.461,14.12,7.02,12.159,3159.9,21.159,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1504.0,+104/-88,0.489,14.21,6.69,11.53,3361.32,51.669,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),1468.0,+101/-67,0.431,16.13,7.41,12.029,150.4,4.403,✔️,0.0,False,GPU +7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1463.0,+80/-69,0.41,16.42,8.4,12.218,2628.27,1.401,✔️,0.0,False,CPU +8,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1457.0,+98/-64,0.439,16.75,6.1,11.286,4.3,0.584,➖,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1441.0,+88/-57,0.391,17.63,8.64,13.535,3159.9,0.646,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1400.0,+84/-79,0.339,20.15,10.7,14.036,3361.32,0.527,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1400.0,+103/-86,0.38,20.15,6.68,12.433,6883.9,405.331,✔️,0.0,False,GPU +12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1389.0,+100/-72,0.332,20.81,9.98,14.604,2899.02,1.461,✔️,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1388.0,+84/-94,0.351,20.87,6.23,13.131,9.07,2.008,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1374.0,+94/-84,0.298,21.73,13.13,16.382,303.47,2.098,➖,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1356.0,+80/-98,0.34,22.88,6.75,13.867,6883.9,39.606,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1342.0,+51/-42,0.194,23.83,19.27,16.138,691.46,1.455,✔️,0.0,False,CPU +17,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1341.0,+82/-91,0.315,23.85,8.22,15.148,4.28,0.511,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1335.0,+77/-86,0.265,24.27,10.91,15.115,6020.89,0.454,✔️,0.0,False,GPU +19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1326.0,+98/-71,0.268,24.83,14.9,15.756,2899.02,0.171,✔️,0.0,False,GPU +20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1325.0,+83/-68,0.21,24.94,17.66,16.007,1394.45,0.618,✔️,0.0,False,CPU +21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1305.0,+68/-56,0.183,26.25,20.41,17.318,785.9,1.281,✔️,0.0,False,CPU +22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1304.0,+74/-52,0.167,26.33,19.44,16.868,5.74,0.139,✔️,0.0,False,CPU +23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1303.0,+73/-63,0.183,26.35,15.34,17.587,2399.23,0.565,✔️,0.0,False,CPU +24,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1298.0,+124/-117,0.282,26.71,9.9,18.367,12.42,1.417,➖,0.0,False,GPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1296.0,+77/-71,0.198,26.81,18.62,16.877,1053.72,1.872,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1294.0,+92/-83,0.238,26.96,17.29,17.031,6020.89,8.478,✔️,0.0,False,GPU +27,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1291.0,+70/-49,0.17,27.19,21.87,16.785,1394.45,0.053,✔️,0.0,False,CPU +28,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1287.0,+71/-58,0.166,27.42,12.81,17.51,785.9,0.213,✔️,0.0,False,CPU +29,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1286.0,+69/-54,0.164,27.5,21.5,17.102,691.46,0.264,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1283.0,+100/-71,0.23,27.73,16.4,18.07,11.6,0.148,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1281.0,+81/-65,0.167,27.85,16.72,17.108,12.75,3.502,✔️,0.0,False,GPU +32,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1254.0,+97/-127,0.258,29.62,8.71,16.756,51.52,40.126,✔️,0.0,False,GPU +33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1252.0,+60/-71,0.134,29.81,13.25,18.357,2399.23,0.061,✔️,0.0,False,CPU +34,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1251.0,+88/-92,0.139,29.85,12.79,18.783,7.41,0.052,✔️,0.0,False,CPU +35,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1249.0,+72/-74,0.169,29.96,22.06,17.668,2792.14,2.175,✔️,0.0,False,CPU +36,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1232.0,+84/-74,0.126,31.13,22.19,18.071,1053.72,0.103,✔️,0.0,False,GPU +37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1217.0,+92/-107,0.17,32.1,20.76,20.456,1350.04,6.57,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1190.0,+71/-115,0.109,33.87,12.63,21.041,13.67,0.339,✔️,0.0,False,GPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1180.0,+80/-84,0.121,34.5,27.01,19.614,2792.14,0.133,✔️,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1159.0,+82/-82,0.085,35.83,27.98,22.147,378.34,1.373,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1152.0,+95/-107,0.124,36.31,21.92,21.966,1350.04,0.613,✔️,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1138.0,+92/-91,0.107,37.19,19.83,22.595,378.34,0.154,✔️,0.0,False,CPU +43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1127.0,+60/-60,0.033,37.85,34.5,21.116,2.66,0.17,✔️,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1117.0,+107/-103,0.077,38.46,27.77,23.308,527.27,1.155,✔️,0.0,False,CPU +45,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1102.0,+70/-70,0.047,39.33,35.23,21.73,3.01,0.177,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1078.0,+88/-89,0.045,40.73,33.27,24.138,527.27,0.128,✔️,0.0,False,CPU +47,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1028.0,+85/-152,0.068,43.5,32.84,28.036,2.28,0.454,➖,0.0,False,GPU +48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1012.0,+90/-94,0.035,44.31,37.65,26.238,8.93,0.172,✔️,0.0,False,CPU +49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+82/-103,0.028,44.92,33.83,28.762,0.44,0.065,✔️,0.0,False,CPU +50,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),998.0,+92/-120,0.049,45.0,39.17,26.76,5.0,0.753,✔️,0.0,False,GPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),990.0,+92/-104,0.031,45.42,41.01,25.606,4.68,0.547,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),961.0,+81/-103,0.009,46.77,43.84,29.843,118.4,1.479,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),954.0,+94/-96,0.038,47.06,17.1,31.197,429.17,0.495,✔️,0.0,False,CPU +54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),947.0,+86/-142,0.02,47.38,42.29,30.28,0.36,0.068,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),930.0,+98/-110,0.033,48.12,25.77,31.807,429.17,0.081,✔️,0.0,False,CPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),898.0,+103/-116,0.015,49.38,44.67,34.214,2.16,0.129,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),768.0,+83/-149,0.0,53.46,52.66,37.314,118.4,0.082,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),574.0,+118/-291,0.0,57.08,56.73,49.46,0.27,0.024,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip index 65dbd80994f0ff2942dd25b37c99ac0133805069..27d58110903d3c8b887f3a4f4b31e05fe3c9d324 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c668582b4db6db0ea6b4685d4f1e16f2c54f5a894dc20c85607658823a7bc2f0 -size 2336052 +oid sha256:524eb46d68d327d1a6d777a5e42c0f4b7db148f6082f19796926140652c67412 +size 2299751 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index b23a5a8143368347b56a7d1e05146d7d8df155c0..a2f74acb617580fa0c84ad0b35466298017ab009 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c41dfb1ce8978e97fb425592288545d25aeace828f367d002f0014fd97098778 -size 466924 +oid sha256:f6cefd752e59363417d51f7d6be24d9f192735cd0878dea83a1de8b3ca5d334e +size 480312 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip index b28b654a37bdf4d1c043741fef7b98de9cecdc63..9e7e8619ffe8e41038d7664876ebec4d017a8368 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e6e5faf6f26a0a508e2d9ef7a144442b343171e7089cde1e45d5dd466c0191fb -size 1007719 +oid sha256:766ac9faa5866695778540a808be2b9ebb78723b1f39677a503812ee2db6b666 +size 1027173 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip index ea164860b05c7b92ddd8d519f89bce38c4daf84c..153363f19879f89dd246f4a9a418b818b02ecb7d 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:46cb4af198af897080ad8f1eaac66ac6551c563b91f5a795d894335eb142641b -size 169156 +oid sha256:6f0cd8c17fe6b9edfe30e9256c0074d002ea9089e499d696e4cdfdeef3a28c05 +size 172882 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/website_leaderboard.csv index da5264ea182bfefca3100fbc2173060f912ef7a0..e87c69e972109ad79b927d7cbe24127bfa6bd3ab 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1724.0,+707/-194,1.808,0.796,6.0,1.66,2793.65,8.93,✔️,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1666.0,+231/-99,11.082,0.72,7.62,5.34,1008.06,9.121,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1657.0,+269/-152,9.911,0.675,7.94,4.58,2793.65,1.513,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1523.0,+311/-140,10.367,0.527,13.31,5.61,9.88,0.825,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[LimiX (default) [12.50% IMPUTED]](https://arxiv.org/abs/2509.03505),1492.0,+207/-87,12.322,0.456,14.81,8.07,5.93,0.989,➖,12.5,True,GPU -5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1470.0,+196/-107,14.071,0.496,16.0,4.26,5546.65,3.268,✔️,0.0,False,CPU -6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1432.0,+176/-88,15.024,0.359,18.12,14.73,3307.58,1.182,✔️,0.0,False,CPU -7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1430.0,+204/-111,15.484,0.401,18.25,5.67,4334.8,21.159,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1425.0,+318/-180,18.166,0.447,18.5,6.69,2584.13,12.372,✔️,25.0,True,GPU -9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1421.0,+173/-68,15.552,0.336,18.75,15.39,3307.58,0.14,✔️,0.0,False,CPU -10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1410.0,+136/-93,16.245,0.316,19.38,13.27,1280.01,4.078,✔️,0.0,False,CPU -11,🧠⚡,Foundation Model,[Mitra (default) [25.00% IMPUTED]](https://arxiv.org/abs/2510.21204),1410.0,+215/-171,18.908,0.384,19.38,13.23,159.43,3.545,✔️,25.0,True,GPU -12,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1400.0,+134/-112,16.321,0.32,20.0,13.04,1280.01,1.051,✔️,0.0,False,CPU -13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1398.0,+193/-119,15.962,0.354,20.12,15.39,4334.8,3.529,✔️,0.0,False,GPU -14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1389.0,+142/-108,18.699,0.32,20.62,13.94,5568.31,1.781,✔️,0.0,False,GPU -15,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1385.0,+215/-130,19.449,0.374,20.88,11.34,9411.86,344.009,✔️,0.0,False,GPU -16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1375.0,+148/-106,19.038,0.317,21.5,12.59,5568.31,0.372,✔️,0.0,False,GPU -17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1347.0,+112/-85,16.949,0.219,23.25,21.3,2029.77,4.113,✔️,0.0,False,CPU -18,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1347.0,+193/-161,19.369,0.296,23.25,9.4,2079.21,3.347,✔️,0.0,False,GPU -19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1345.0,+124/-90,16.708,0.234,23.38,21.0,2029.77,1.04,✔️,0.0,False,CPU -20,🧠⚡,Foundation Model,[TabPFNv2 (default) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1341.0,+193/-143,18.203,0.291,23.62,11.54,5.48,0.354,✔️,25.0,True,GPU -21,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1330.0,+214/-150,19.679,0.28,24.38,13.89,2584.13,0.408,✔️,25.0,True,GPU -22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1325.0,+143/-104,18.145,0.225,24.69,20.75,6684.65,9.59,✔️,0.0,False,GPU -23,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1303.0,+157/-112,18.488,0.184,26.06,21.31,6684.65,0.752,✔️,0.0,False,GPU -24,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1302.0,+213/-134,20.809,0.302,26.12,6.19,9411.86,39.507,✔️,0.0,False,GPU -25,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1287.0,+146/-134,20.217,0.185,27.12,16.25,27108.48,0.742,✔️,0.0,False,CPU -26,🧠⚡,Foundation Model,[TabICL (default) [12.50% IMPUTED]](https://arxiv.org/abs/2502.05564),1284.0,+236/-185,23.291,0.269,27.31,8.56,11.51,1.948,✔️,12.5,True,GPU -27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1283.0,+144/-138,22.14,0.227,27.38,15.88,17.09,0.145,✔️,0.0,False,GPU -28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1282.0,+193/-166,21.612,0.24,27.44,14.07,728.32,2.441,✔️,0.0,False,CPU -29,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1277.0,+114/-76,18.577,0.153,27.75,24.97,43.1,0.254,✔️,0.0,False,CPU -30,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1275.0,+224/-214,20.771,0.266,27.88,14.24,729.17,1.833,✔️,0.0,False,CPU -31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1269.0,+242/-179,20.994,0.234,28.25,6.12,2079.21,0.319,✔️,0.0,False,GPU -32,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1256.0,+205/-148,21.742,0.211,29.06,14.14,728.32,0.356,✔️,0.0,False,CPU -33,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1251.0,+123/-146,21.941,0.187,29.38,22.35,283.57,4.104,➖,0.0,False,GPU -34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1241.0,+123/-109,21.705,0.132,30.0,27.09,3646.83,2.162,✔️,0.0,False,CPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1228.0,+122/-158,21.727,0.169,30.88,24.3,27108.48,0.066,✔️,0.0,False,CPU -36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1216.0,+122/-136,22.231,0.139,31.62,28.2,17.86,0.099,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1206.0,+148/-164,23.531,0.126,32.25,21.47,3646.83,0.194,✔️,0.0,False,CPU -38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1200.0,+198/-186,22.297,0.191,32.62,20.2,729.17,0.325,✔️,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1190.0,+204/-185,25.651,0.168,33.25,21.78,2721.87,12.593,✔️,0.0,False,CPU -40,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1184.0,+252/-168,24.06,0.213,33.62,10.77,46.36,39.812,✔️,0.0,False,GPU -41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1178.0,+130/-119,20.547,0.07,34.0,30.54,4.93,0.592,✔️,0.0,False,CPU -42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1176.0,+155/-160,22.982,0.13,34.12,26.02,5.12,0.444,✔️,0.0,False,CPU -43,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1112.0,+125/-146,25.251,0.066,37.88,33.48,19.12,6.922,✔️,0.0,False,GPU -44,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1101.0,+125/-156,27.29,0.022,38.5,35.99,17.24,0.565,✔️,0.0,False,GPU -45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1089.0,+164/-171,27.296,0.085,39.12,30.54,2721.87,1.083,✔️,0.0,False,CPU -46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1085.0,+163/-237,29.013,0.058,39.38,33.84,8.99,1.372,✔️,0.0,False,GPU -47,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1013.0,+264/-406,38.476,0.128,43.0,27.42,155.52,2.046,✔️,0.0,False,CPU -48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+99/-219,35.562,0.0,43.62,41.77,0.74,0.148,✔️,0.0,False,CPU -49,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),972.0,+107/-162,30.565,0.0,44.88,43.55,14.37,0.363,✔️,0.0,False,CPU -50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),929.0,+156/-270,34.532,0.024,46.62,42.92,8.37,0.659,✔️,0.0,False,CPU -51,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),908.0,+73/-232,41.326,0.0,47.44,46.53,909.67,0.609,✔️,0.0,False,CPU -52,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),890.0,+83/-217,42.282,0.0,48.06,47.0,909.67,0.107,✔️,0.0,False,CPU -53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),885.0,+164/-240,40.712,0.003,48.25,45.44,0.76,0.153,✔️,0.0,False,CPU -54,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),780.0,+283/-1302,50.559,0.071,51.38,42.87,155.52,0.191,✔️,0.0,False,CPU -55,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),765.0,+104/-224,41.42,0.0,51.75,51.44,4.67,0.855,➖,0.0,False,GPU -56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),698.0,+93/-331,48.104,0.0,53.25,52.96,4.62,0.135,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-283.0,+165/-1047,70.927,0.0,58.0,58.0,0.52,0.075,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1720.0,+699/-192,0.804,6.0,1.66,1.808,2793.65,8.93,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1655.0,+207/-99,0.719,7.88,5.43,11.082,1008.06,9.121,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1649.0,+244/-150,0.681,8.06,4.67,9.911,2793.65,1.513,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1515.0,+283/-134,0.53,13.56,6.37,10.367,9.88,0.825,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[LimiX (default) [12.50% IMPUTED]](https://arxiv.org/abs/2509.03505),1483.0,+182/-83,0.458,15.19,8.58,12.322,5.93,0.989,➖,12.5,True,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1461.0,+185/-108,0.5,16.38,4.31,14.071,5546.65,3.268,✔️,0.0,False,CPU +6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1422.0,+166/-82,0.356,18.62,15.23,15.024,3307.58,1.182,✔️,0.0,False,CPU +7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1422.0,+190/-106,0.397,18.62,5.74,15.484,4334.8,21.159,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1416.0,+292/-183,0.444,19.0,6.82,18.166,2584.13,12.372,✔️,25.0,True,GPU +9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1411.0,+152/-64,0.339,19.25,15.68,15.552,3307.58,0.14,✔️,0.0,False,CPU +10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1403.0,+134/-94,0.322,19.75,13.4,16.245,1280.01,4.078,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[Mitra (default) [25.00% IMPUTED]](https://arxiv.org/abs/2510.21204),1399.0,+208/-172,0.381,20.0,13.82,18.908,159.43,3.545,✔️,25.0,True,GPU +12,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1393.0,+132/-112,0.323,20.38,13.2,16.321,1280.01,1.051,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1389.0,+175/-116,0.349,20.62,15.68,15.962,4334.8,3.529,✔️,0.0,False,GPU +14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1382.0,+140/-108,0.322,21.0,14.15,18.699,5568.31,1.781,✔️,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1376.0,+209/-130,0.378,21.38,11.45,19.449,9411.86,344.009,✔️,0.0,False,GPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1369.0,+149/-106,0.318,21.88,12.69,19.038,5568.31,0.372,✔️,0.0,False,GPU +17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1340.0,+101/-80,0.217,23.75,21.83,16.949,2029.77,4.113,✔️,0.0,False,CPU +18,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1338.0,+189/-166,0.293,23.88,9.54,19.369,2079.21,3.347,✔️,0.0,False,GPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1338.0,+116/-86,0.229,23.88,21.57,16.708,2029.77,1.04,✔️,0.0,False,CPU +20,🧠⚡,Foundation Model,[TabPFNv2 (default) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1332.0,+180/-144,0.289,24.25,13.55,18.203,5.48,0.354,✔️,25.0,True,GPU +21,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1321.0,+197/-145,0.277,25.0,15.36,19.679,2584.13,0.408,✔️,25.0,True,GPU +22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1318.0,+134/-101,0.223,25.19,21.04,18.145,6684.65,9.59,✔️,0.0,False,GPU +23,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1299.0,+150/-114,0.186,26.44,21.47,18.488,6684.65,0.752,✔️,0.0,False,GPU +24,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1296.0,+220/-257,0.366,26.62,8.23,20.668,21.84,1.702,➖,0.0,False,GPU +25,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1296.0,+204/-138,0.304,26.62,6.21,20.809,9411.86,39.507,✔️,0.0,False,GPU +26,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1283.0,+143/-137,0.195,27.5,16.34,20.217,27108.48,0.742,✔️,0.0,False,CPU +27,🧠⚡,Foundation Model,[TabICL (default) [12.50% IMPUTED]](https://arxiv.org/abs/2502.05564),1276.0,+216/-186,0.27,27.94,8.59,23.291,11.51,1.948,✔️,12.5,True,GPU +28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1276.0,+190/-166,0.244,27.94,14.23,21.612,728.32,2.441,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1275.0,+144/-140,0.228,28.0,16.08,22.14,17.09,0.145,✔️,0.0,False,GPU +30,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1271.0,+101/-75,0.162,28.25,25.54,18.577,43.1,0.254,✔️,0.0,False,CPU +31,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1269.0,+221/-210,0.27,28.38,14.91,20.771,729.17,1.833,✔️,0.0,False,CPU +32,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1262.0,+236/-179,0.232,28.88,6.19,20.994,2079.21,0.319,✔️,0.0,False,GPU +33,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1251.0,+204/-150,0.216,29.56,14.41,21.742,728.32,0.356,✔️,0.0,False,CPU +34,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1244.0,+122/-145,0.192,30.0,22.89,21.941,283.57,4.104,➖,0.0,False,GPU +35,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1235.0,+113/-110,0.14,30.62,27.53,21.705,3646.83,2.162,✔️,0.0,False,CPU +36,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1221.0,+122/-157,0.177,31.5,24.81,21.727,27108.48,0.066,✔️,0.0,False,CPU +37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1210.0,+117/-141,0.15,32.25,28.55,22.231,17.86,0.099,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1200.0,+144/-165,0.136,32.88,21.67,23.531,3646.83,0.194,✔️,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1196.0,+194/-182,0.197,33.12,21.01,22.297,729.17,0.325,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1185.0,+181/-186,0.167,33.88,22.31,25.651,2721.87,12.593,✔️,0.0,False,CPU +41,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1181.0,+242/-168,0.214,34.12,10.79,24.06,46.36,39.812,✔️,0.0,False,GPU +42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1175.0,+115/-125,0.072,34.5,30.92,20.547,4.93,0.592,✔️,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1171.0,+155/-163,0.127,34.75,26.49,22.982,5.12,0.444,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1106.0,+123/-146,0.063,38.62,34.3,25.251,19.12,6.922,✔️,0.0,False,GPU +45,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1098.0,+118/-156,0.027,39.12,36.54,27.29,17.24,0.565,✔️,0.0,False,GPU +46,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1084.0,+145/-168,0.081,39.88,31.85,27.296,2721.87,1.083,✔️,0.0,False,CPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1082.0,+162/-224,0.055,40.0,34.37,29.013,8.99,1.372,✔️,0.0,False,GPU +48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1013.0,+261/-397,0.13,43.62,27.58,38.476,155.52,2.046,✔️,0.0,False,CPU +49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+97/-198,0.0,44.25,42.35,35.562,0.74,0.148,✔️,0.0,False,CPU +50,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),970.0,+108/-152,0.0,45.62,44.22,30.565,14.37,0.363,✔️,0.0,False,CPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),925.0,+147/-261,0.022,47.5,43.96,34.532,8.37,0.659,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),904.0,+67/-219,0.0,48.31,47.45,41.326,909.67,0.609,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),887.0,+67/-207,0.0,48.94,47.93,42.282,909.67,0.107,✔️,0.0,False,CPU +54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),886.0,+164/-237,0.003,49.0,46.02,40.712,0.76,0.153,✔️,0.0,False,CPU +55,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),785.0,+282/-876,0.075,52.12,43.3,50.559,155.52,0.191,✔️,0.0,False,CPU +56,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),765.0,+98/-218,0.0,52.62,52.31,41.42,4.67,0.855,➖,0.0,False,GPU +57,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),699.0,+85/-257,0.0,54.12,53.86,48.104,4.62,0.135,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-304.0,+191/-698,0.0,59.0,59.0,70.927,0.52,0.075,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/winrate_matrix.png.zip index 68d95d1cbdfe193f2191496de484f58e51919f18..52aa4daae1cd081cb30c486caf8cd3ab8904e39c 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4c179e7e2c619545fb7a3953c137ad7953f31b7de66374a5e7a4cb9672eb0694 -size 1788887 +oid sha256:db738577b3c1c80d4f67061c65b8ec694978215595d79d0b3d421c37bc4fc8c1 +size 1697509 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index 502f9965ec35ce1828d9e48438bd7cebbace2e1a..bf4e6402c4693caccc0e69d86c7b3d580a20c620 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:134a614466a30a212c4dfab23d74e399f90ed8d3d055621a4f9858988bab216f -size 451105 +oid sha256:4926bc8cd9ad69d5c1b78f470dbc129bc8ebcb1763e22ff4c34efd1ab6d2dcd6 +size 465220 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip index 44a02f1b113e9d79778ba384c02adfd72e79446f..303bfe595b9ad2c66c9a49ccfbdfb4a35d6f4a3e 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0a6d78d440415703b94c598e6f2551520cb543b71edc689143987ebc44807b3b -size 948654 +oid sha256:66d5c4cab65869afa38ae38989be2885aaf186c528be45a004bd53fa2ff7ee96 +size 986657 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip index 344d8cb5b00a349dfe02f719d089f1eb0a7a6889..17ddd653f26e56cc231da7f2f63aeead92924aeb 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c3100cd7f8a801aac04d2c58754b8b5767e466c2f9adb9f5598c30335f2b8fb8 -size 107975 +oid sha256:6951d6de26841bcfe9539f4e192a58fcad95cdeebfdb941464b73ae9f2fd7311 +size 113816 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/website_leaderboard.csv index f680a6b8392e345fe931fd78494d7ac947fec6fd..d0fe50aa14cb9ce5a4b243ea5ac30d0a27b22e02 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/website_leaderboard.csv @@ -1,54 +1,55 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),25228.0,+0/-0,0.0,1.0,1.0,1.0,134.59,8.308,✔️,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",23917.0,+0/-0,1.729,0.934,2.0,2.0,276.98,4.897,✔️,0.0,False,CPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",22310.0,+0/-0,2.208,0.915,3.0,3.0,276.84,3.303,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),20483.0,+0/-0,3.024,0.883,4.0,4.0,134.59,2.819,✔️,0.0,False,GPU -4,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),19432.0,+0/-0,10.141,0.576,5.0,5.0,7855.1,0.565,✔️,0.0,False,CPU -5,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),18258.0,+0/-0,10.238,0.572,6.0,6.0,61.47,0.154,✔️,0.0,False,CPU -6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),17510.0,+0/-0,10.303,0.569,7.0,7.0,157.4,4.579,✔️,0.0,False,CPU -7,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),16643.0,+0/-0,10.333,0.567,8.0,8.0,157.4,0.401,✔️,0.0,False,CPU -8,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),15885.0,+0/-0,10.67,0.551,9.0,9.0,61.47,0.015,✔️,0.0,False,CPU -9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),15137.0,+0/-0,11.231,0.525,10.0,10.0,2687.46,0.361,✔️,0.0,False,CPU -10,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),14345.0,+0/-0,11.295,0.522,11.0,11.0,7855.1,0.048,✔️,0.0,False,CPU -11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),13528.0,+0/-0,11.389,0.517,12.0,12.0,2687.46,0.793,✔️,0.0,False,CPU -12,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),12768.0,+0/-0,11.441,0.515,13.0,13.0,456.33,1.171,✔️,0.0,False,CPU -13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),12036.0,+0/-0,11.722,0.501,14.0,14.0,456.33,0.047,✔️,0.0,False,CPU -14,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),11311.0,+0/-0,11.722,0.501,15.0,15.0,0.32,0.047,✔️,0.0,False,CPU -15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),10518.0,+0/-0,13.143,0.432,16.0,16.0,8.22,0.049,✔️,0.0,False,CPU -16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),9807.0,+0/-0,13.383,0.42,17.0,17.0,4378.82,25.595,✔️,0.0,False,GPU -17,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),9111.0,+0/-0,14.026,0.387,18.0,18.0,20.73,0.079,✔️,0.0,False,CPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),8390.0,+0/-0,14.047,0.386,19.0,19.0,4378.82,2.216,✔️,0.0,False,GPU -19,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),7635.0,+0/-0,14.639,0.356,20.0,20.0,1.48,0.535,✔️,0.0,False,GPU -20,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),6953.0,+0/-0,15.857,0.292,21.0,21.0,2.79,1.057,✔️,0.0,False,GPU -21,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),6264.0,+0/-0,16.662,0.249,22.0,22.0,20.88,0.359,✔️,0.0,False,CPU -22,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),5562.0,+0/-0,19.143,0.111,23.0,23.0,20.88,0.026,✔️,0.0,False,CPU -23,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),5007.0,+0/-0,20.028,0.059,24.0,24.0,4390.96,275.957,✔️,0.0,False,GPU -24,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),4306.0,+0/-0,20.621,0.024,25.0,25.0,2341.91,0.027,✔️,0.0,False,CPU -25,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),3678.0,+0/-0,20.955,0.004,26.0,26.0,14.53,1.747,✔️,0.0,False,GPU -26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),3004.0,+0/-0,21.025,0.0,27.0,27.0,2341.91,0.172,✔️,0.0,False,CPU -27,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),2329.0,+0/-0,21.961,0.0,28.0,28.0,1993.57,0.469,✔️,0.0,False,GPU -28,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1701.0,+0/-0,22.049,0.0,29.0,29.0,1993.57,0.055,✔️,0.0,False,GPU -29,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+0/-0,24.001,0.0,30.0,30.0,0.18,0.007,✔️,0.0,False,CPU -30,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),445.0,+0/-0,24.06,0.0,31.0,31.0,4390.96,38.525,✔️,0.0,False,GPU -31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),-256.0,+0/-0,24.187,0.0,32.0,32.0,41.21,39.406,✔️,0.0,False,GPU -32,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),-946.0,+0/-0,24.591,0.0,33.0,33.0,1322.48,0.546,✔️,0.0,False,GPU -33,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),-1628.0,+0/-0,25.183,0.0,34.0,34.0,1322.48,0.055,✔️,0.0,False,GPU -34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),-2382.0,+0/-0,26.439,0.0,35.0,35.0,0.21,0.047,✔️,0.0,False,CPU -35,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),-3104.0,+0/-0,29.215,0.0,36.0,36.0,2009.11,3.767,✔️,0.0,False,GPU -36,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),-3800.0,+0/-0,30.84,0.0,37.0,37.0,2009.11,0.974,✔️,0.0,False,GPU -37,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),-4511.0,+0/-0,31.796,0.0,38.0,38.0,6.84,0.09,✔️,0.0,False,GPU -38,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),-5304.0,+0/-0,31.813,0.0,39.0,39.0,3.34,0.028,✔️,0.0,False,CPU -39,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),-6029.0,+0/-0,34.136,0.0,40.0,40.0,66.06,1.783,➖,0.0,False,GPU -40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),-6760.0,+0/-0,38.819,0.0,41.0,41.0,6.51,0.162,✔️,0.0,False,GPU -41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),-7521.0,+0/-0,41.321,0.0,42.0,42.0,553.48,0.174,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),-8338.0,+0/-0,42.976,0.0,43.0,43.0,553.48,1.636,✔️,0.0,False,CPU -43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),-9130.0,+0/-0,50.562,0.0,44.0,44.0,0.08,0.017,✔️,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),-9878.0,+0/-0,52.296,0.0,45.0,45.0,2.73,0.083,✔️,0.0,False,CPU -45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),-10636.0,+0/-0,52.871,0.0,46.0,46.0,30.19,0.134,✔️,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),-11503.0,+0/-0,55.965,0.0,47.0,47.0,30.19,0.017,✔️,0.0,False,CPU -47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),-12250.0,+0/-0,56.26,0.0,48.0,48.0,0.13,0.015,✔️,0.0,False,CPU -48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),-13425.0,+0/-0,57.211,0.0,49.0,49.0,76.41,2.853,✔️,0.0,False,CPU -49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),-14476.0,+0/-0,57.769,0.0,50.0,50.0,3.14,0.921,✔️,0.0,False,GPU -50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),-16303.0,+0/-0,57.822,0.0,51.0,51.0,0.11,0.023,➖,0.0,False,GPU -51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),-17910.0,+0/-0,70.703,0.0,52.0,52.0,76.41,0.258,✔️,0.0,False,CPU -52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-19221.0,+0/-0,83.729,0.0,53.0,53.0,0.14,0.123,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),25529.0,+0/-0,1.0,1.0,1.0,0.0,134.59,8.308,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",24211.0,+0/-0,0.936,2.0,2.0,1.729,276.98,4.897,✔️,0.0,False,CPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",22594.0,+0/-0,0.918,3.0,3.0,2.208,276.84,3.303,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),20725.0,+0/-0,0.886,4.0,4.0,3.024,134.59,2.819,✔️,0.0,False,GPU +4,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),19652.0,+0/-0,0.588,5.0,5.0,10.141,7855.1,0.565,✔️,0.0,False,CPU +5,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),18457.0,+0/-0,0.583,6.0,6.0,10.238,61.47,0.154,✔️,0.0,False,CPU +6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),17698.0,+0/-0,0.58,7.0,7.0,10.303,157.4,4.579,✔️,0.0,False,CPU +7,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),16836.0,+0/-0,0.579,8.0,8.0,10.333,157.4,0.401,✔️,0.0,False,CPU +8,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),16053.0,+0/-0,0.564,9.0,9.0,10.67,61.47,0.015,✔️,0.0,False,CPU +9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),15299.0,+0/-0,0.538,10.0,10.0,11.231,2687.46,0.361,✔️,0.0,False,CPU +10,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),14521.0,+0/-0,0.535,11.0,11.0,11.295,7855.1,0.048,✔️,0.0,False,CPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),13703.0,+0/-0,0.531,12.0,12.0,11.389,2687.46,0.793,✔️,0.0,False,CPU +12,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),12920.0,+0/-0,0.528,13.0,13.0,11.441,456.33,1.171,✔️,0.0,False,CPU +13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),12173.0,+0/-0,0.515,14.0,14.0,11.722,456.33,0.047,✔️,0.0,False,CPU +14,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),11454.0,+0/-0,0.515,15.0,15.0,11.722,0.32,0.047,✔️,0.0,False,CPU +15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),10661.0,+0/-0,0.447,16.0,16.0,13.143,8.22,0.049,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),9930.0,+0/-0,0.436,17.0,17.0,13.383,4378.82,25.595,✔️,0.0,False,GPU +17,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),9235.0,+0/-0,0.404,18.0,18.0,14.026,20.73,0.079,✔️,0.0,False,CPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),8520.0,+0/-0,0.403,19.0,19.0,14.047,4378.82,2.216,✔️,0.0,False,GPU +19,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),7765.0,+0/-0,0.374,20.0,20.0,14.639,1.48,0.535,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),7045.0,+0/-0,0.312,21.0,21.0,15.857,2.79,1.057,✔️,0.0,False,GPU +21,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),6363.0,+0/-0,0.27,22.0,22.0,16.662,20.88,0.359,✔️,0.0,False,CPU +22,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),5667.0,+0/-0,0.135,23.0,23.0,19.143,20.88,0.026,✔️,0.0,False,CPU +23,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),4962.0,+0/-0,0.085,24.0,24.0,20.028,4390.96,275.957,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),4493.0,+0/-0,0.051,25.0,25.0,20.621,2341.91,0.027,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),3732.0,+0/-0,0.032,26.0,26.0,20.955,14.53,1.747,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),3107.0,+0/-0,0.028,27.0,27.0,21.025,2341.91,0.172,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),2386.0,+0/-0,0.0,28.0,28.0,21.961,1993.57,0.469,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1761.0,+0/-0,0.0,29.0,29.0,22.049,1993.57,0.055,✔️,0.0,False,GPU +29,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+0/-0,0.0,30.0,30.0,24.001,0.18,0.007,✔️,0.0,False,CPU +30,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),531.0,+0/-0,0.0,31.0,31.0,24.06,4390.96,38.525,✔️,0.0,False,GPU +31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),-174.0,+0/-0,0.0,32.0,32.0,24.187,41.21,39.406,✔️,0.0,False,GPU +32,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),-870.0,+0/-0,0.0,33.0,33.0,24.591,1322.48,0.546,✔️,0.0,False,GPU +33,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),-1552.0,+0/-0,0.0,34.0,34.0,24.684,9.89,9.237,➖,0.0,False,GPU +34,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),-2272.0,+0/-0,0.0,35.0,35.0,25.183,1322.48,0.055,✔️,0.0,False,GPU +35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),-3027.0,+0/-0,0.0,36.0,36.0,26.439,0.21,0.047,✔️,0.0,False,CPU +36,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),-3742.0,+0/-0,0.0,37.0,37.0,29.215,2009.11,3.767,✔️,0.0,False,GPU +37,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),-4436.0,+0/-0,0.0,38.0,38.0,30.84,2009.11,0.974,✔️,0.0,False,GPU +38,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),-5168.0,+0/-0,0.0,39.0,39.0,31.796,6.84,0.09,✔️,0.0,False,GPU +39,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),-5961.0,+0/-0,0.0,40.0,40.0,31.813,3.34,0.028,✔️,0.0,False,CPU +40,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),-6680.0,+0/-0,0.0,41.0,41.0,34.136,66.06,1.783,➖,0.0,False,GPU +41,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),-7427.0,+0/-0,0.0,42.0,42.0,38.819,6.51,0.162,✔️,0.0,False,GPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),-8210.0,+0/-0,0.0,43.0,43.0,41.321,553.48,0.174,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),-9028.0,+0/-0,0.0,44.0,44.0,42.976,553.48,1.636,✔️,0.0,False,CPU +44,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),-9806.0,+0/-0,0.0,45.0,45.0,50.562,0.08,0.017,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),-10560.0,+0/-0,0.0,46.0,46.0,52.296,2.73,0.083,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),-11343.0,+0/-0,0.0,47.0,47.0,52.871,30.19,0.134,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),-12204.0,+0/-0,0.0,48.0,48.0,55.965,30.19,0.017,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),-12964.0,+0/-0,0.0,49.0,49.0,56.26,0.13,0.015,✔️,0.0,False,CPU +49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),-14158.0,+0/-0,0.0,50.0,50.0,57.211,76.41,2.853,✔️,0.0,False,CPU +50,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),-15232.0,+0/-0,0.0,51.0,51.0,57.769,3.14,0.921,✔️,0.0,False,GPU +51,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),-17101.0,+0/-0,0.0,52.0,52.0,57.822,0.11,0.023,➖,0.0,False,GPU +52,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),-18718.0,+0/-0,0.0,53.0,53.0,70.703,76.41,0.258,✔️,0.0,False,CPU +53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-20036.0,+0/-0,0.0,54.0,54.0,83.729,0.14,0.123,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/winrate_matrix.png.zip index b706401afd56356367eb31c5e1fb43b7394accf6..b23b5e87d15e590bc10f680990d228c74205ede8 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d96450ca89ffc04b8498ec7d98857fb904961de1611b0d6e5de17de8d237ca08 -size 1057411 +oid sha256:cf3060ec083576e3908f44df7c2a31a5e36901041ed57986a0e5c3ef37251cc5 +size 1130689 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 118639deedc524385647c65b198cc300bc631099..3601c7cee7ab3c5adf9dfac7daaa9755a1833899 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5ef2246f5f6815f60519a7e53de59bcd848f41cbd41bdd5e81dd5eeec9054530 -size 456356 +oid sha256:293f8cba2c837da90d0f21382ab51b35659aee2007254f456d0af82ae0c441a7 +size 470888 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip index e374ce11c33babde741329caf088aadc52d179e2..d7457c19cc63a01b6eb2678230aa16caeb365189 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c017602f05184744047042707352b4576c95f3e423b260076db517fe4a6fec60 -size 993438 +oid sha256:4d5f759cee27aab9be8453cdbc21532f9b260dc463ebb9b93782a23b17c0bfb5 +size 1015321 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip index 4f6369e3758f2afe3082f21d8bf0fe3229523432..77d1d8aa68dc6c24a82ae450cd118c4eadd5cb71 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:599e9ccc99003612a37a33b2a05e7b0337df2597990fa0689e5016b2af4c276f -size 168480 +oid sha256:e9cf01356e5e609e53284a3f4315f307f85d01143ebe1ca1a6d65dd282f837f4 +size 169294 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/website_leaderboard.csv index ad9448e0369114b4df4e22d38d152e72a8590650..9f63349068c60e46d98d7bf9e36ac14cf4d58c3d 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1738.0,+465/-200,2.066,0.767,6.71,1.83,2837.96,9.546,✔️,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1688.0,+223/-89,12.349,0.691,8.29,6.0,1060.41,10.928,✔️,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1681.0,+348/-165,10.895,0.644,8.5,4.68,2837.96,1.134,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1584.0,+371/-164,9.756,0.542,12.36,5.08,10.97,0.856,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1583.0,+198/-94,10.653,0.521,12.43,7.3,6.19,1.301,➖,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1521.0,+248/-121,14.183,0.458,15.43,5.06,4655.15,21.791,✔️,0.0,False,GPU -6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1509.0,+372/-201,17.32,0.511,16.07,5.99,2969.66,17.821,✔️,14.29,True,GPU -7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1485.0,+187/-130,14.644,0.405,17.43,14.17,4655.15,4.031,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[Mitra (default) [14.29% IMPUTED]](https://arxiv.org/abs/2510.21204),1481.0,+229/-184,18.18,0.439,17.64,12.22,202.6,4.622,✔️,14.29,True,GPU -9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1474.0,+269/-114,15.834,0.432,18.0,5.08,6318.92,3.195,✔️,0.0,False,CPU -10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1457.0,+221/-107,15.543,0.329,19.0,15.22,3375.11,1.208,✔️,0.0,False,CPU -11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1447.0,+150/-114,18.233,0.357,19.57,13.01,5729.39,1.817,✔️,0.0,False,GPU -12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1440.0,+174/-63,16.169,0.302,20.0,16.67,3375.11,0.132,✔️,0.0,False,CPU -13,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1433.0,+218/-186,19.367,0.404,20.43,10.55,10182.26,399.569,✔️,0.0,False,GPU -14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1433.0,+163/-119,18.608,0.354,20.43,11.64,5729.39,0.373,✔️,0.0,False,GPU -15,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1428.0,+171/-184,17.962,0.339,20.71,8.46,2149.3,2.928,✔️,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1421.0,+193/-93,17.093,0.273,21.14,15.22,1394.84,3.578,✔️,0.0,False,CPU -17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1411.0,+214/-96,17.176,0.278,21.71,14.33,1394.84,1.087,✔️,0.0,False,CPU -18,🧠⚡,Foundation Model,[TabPFNv2 (default) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1398.0,+221/-189,17.374,0.332,22.5,10.58,6.0,0.388,✔️,14.29,True,GPU -19,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1394.0,+259/-194,19.05,0.32,22.79,12.78,2969.66,0.477,✔️,14.29,True,GPU -20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1363.0,+102/-64,17.736,0.169,24.71,23.44,2342.58,5.131,✔️,0.0,False,CPU -21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1363.0,+102/-82,17.42,0.187,24.71,22.61,2342.58,1.065,✔️,0.0,False,CPU -22,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1359.0,+245/-202,20.345,0.345,25.0,5.55,10182.26,40.124,✔️,0.0,False,GPU -23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1357.0,+188/-136,20.76,0.26,25.14,14.57,19.13,0.146,✔️,0.0,False,GPU -24,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1347.0,+159/-119,18.825,0.188,25.79,21.43,6994.17,9.554,✔️,0.0,False,GPU -25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1339.0,+191/-199,19.587,0.267,26.29,5.46,2149.3,0.147,✔️,0.0,False,GPU -26,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1326.0,+160/-143,19.122,0.145,27.07,21.69,6994.17,0.583,✔️,0.0,False,GPU -27,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1325.0,+160/-127,20.198,0.214,27.14,20.85,326.23,4.282,➖,0.0,False,GPU -28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1308.0,+189/-237,22.319,0.227,28.21,13.38,772.2,2.495,✔️,0.0,False,CPU -29,🧠⚡,Foundation Model,[TabICL (default) [14.29% IMPUTED]](https://arxiv.org/abs/2502.05564),1308.0,+256/-194,24.353,0.254,28.21,7.89,11.7,2.132,✔️,14.29,True,GPU -30,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1289.0,+122/-76,19.353,0.104,29.43,27.14,57.99,0.317,✔️,0.0,False,CPU -31,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1281.0,+180/-226,22.113,0.212,29.93,13.4,772.2,0.373,✔️,0.0,False,CPU -32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1276.0,+135/-105,21.656,0.122,30.29,23.95,44054.5,0.843,✔️,0.0,False,CPU -33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1273.0,+139/-119,21.802,0.135,30.43,27.11,3833.57,2.294,✔️,0.0,False,CPU -34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1264.0,+194/-273,22.276,0.215,31.0,17.71,731.01,2.021,✔️,0.0,False,CPU -35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1262.0,+252/-221,23.176,0.192,31.14,20.2,2750.22,13.569,✔️,0.0,False,CPU -36,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1230.0,+249/-225,24.042,0.244,33.14,9.78,51.52,40.219,✔️,0.0,False,GPU -37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1228.0,+182/-165,23.947,0.125,33.29,21.05,3833.57,0.22,✔️,0.0,False,CPU -38,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1228.0,+182/-225,22.488,0.148,33.29,24.78,5.15,0.526,✔️,0.0,False,CPU -39,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1223.0,+115/-138,23.403,0.094,33.57,30.68,14.99,0.113,✔️,0.0,False,CPU -40,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1221.0,+124/-126,23.217,0.111,33.71,29.37,44054.5,0.068,✔️,0.0,False,CPU -41,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1183.0,+134/-218,23.958,0.132,36.0,24.57,731.01,0.334,✔️,0.0,False,CPU -42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1170.0,+54/-72,21.807,0.0,36.71,35.84,5.38,0.624,✔️,0.0,False,CPU -43,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1170.0,+140/-177,23.313,0.076,36.71,32.22,20.63,7.532,✔️,0.0,False,GPU -44,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1163.0,+150/-236,24.905,0.067,37.14,32.08,9.4,1.444,✔️,0.0,False,GPU -45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1148.0,+220/-223,25.292,0.097,38.0,29.08,2750.22,1.239,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1106.0,+117/-141,28.196,0.008,40.29,38.08,19.64,0.478,✔️,0.0,False,GPU -47,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1084.0,+250/-449,35.799,0.146,41.43,25.62,180.38,1.24,✔️,0.0,False,CPU -48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1008.0,+129/-210,30.387,0.0,45.0,43.49,14.87,0.391,✔️,0.0,False,CPU -49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+105/-290,37.213,0.0,45.36,43.81,0.78,0.155,✔️,0.0,False,CPU -50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),981.0,+192/-316,31.994,0.028,46.14,42.07,8.78,0.688,✔️,0.0,False,CPU -51,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),961.0,+74/-238,39.676,0.0,46.93,45.95,913.1,0.738,✔️,0.0,False,CPU -52,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),946.0,+82/-283,40.327,0.0,47.5,46.37,913.1,0.13,✔️,0.0,False,CPU -53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),929.0,+180/-311,39.305,0.004,48.14,44.97,0.77,0.156,✔️,0.0,False,CPU -54,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),853.0,+284/-1300,47.681,0.081,50.57,41.4,180.38,0.136,✔️,0.0,False,CPU -55,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),832.0,+80/-228,39.077,0.0,51.14,50.85,5.48,0.91,➖,0.0,False,GPU -56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),736.0,+100/-302,46.939,0.0,53.29,52.96,4.65,0.14,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-464.0,+388/-687,69.099,0.0,58.0,58.0,0.58,0.071,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1734.0,+485/-194,0.776,6.71,1.83,2.066,2837.96,9.546,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1675.0,+206/-86,0.689,8.57,6.14,12.349,1060.41,10.928,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1673.0,+321/-157,0.65,8.64,4.79,10.895,2837.96,1.134,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1575.0,+331/-155,0.54,12.64,5.81,9.756,10.97,0.856,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1571.0,+183/-87,0.523,12.86,7.77,10.653,6.19,1.301,➖,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1512.0,+227/-112,0.453,15.86,5.12,14.183,4655.15,21.791,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1497.0,+341/-196,0.508,16.64,6.11,17.32,2969.66,17.821,✔️,14.29,True,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1475.0,+178/-124,0.399,17.86,14.43,14.644,4655.15,4.031,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[Mitra (default) [14.29% IMPUTED]](https://arxiv.org/abs/2510.21204),1467.0,+220/-179,0.435,18.36,12.79,18.18,202.6,4.622,✔️,14.29,True,GPU +9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1466.0,+260/-108,0.437,18.43,5.16,15.834,6318.92,3.195,✔️,0.0,False,CPU +10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1446.0,+194/-100,0.322,19.57,15.84,15.543,3375.11,1.208,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1439.0,+147/-109,0.353,20.0,13.21,18.233,5729.39,1.817,✔️,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1430.0,+158/-56,0.301,20.57,17.06,16.169,3375.11,0.132,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1426.0,+160/-115,0.348,20.86,11.75,18.608,5729.39,0.373,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1424.0,+213/-178,0.403,21.0,10.65,19.367,10182.26,399.569,✔️,0.0,False,GPU +15,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1419.0,+169/-183,0.335,21.29,8.59,17.962,2149.3,2.928,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1414.0,+179/-97,0.277,21.57,15.42,17.093,1394.84,3.578,✔️,0.0,False,CPU +17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1406.0,+206/-97,0.278,22.14,14.55,17.176,1394.84,1.087,✔️,0.0,False,CPU +18,🧠⚡,Foundation Model,[TabPFNv2 (default) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1388.0,+206/-183,0.33,23.21,12.53,17.374,6.0,0.388,✔️,14.29,True,GPU +19,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1384.0,+239/-188,0.316,23.5,14.2,19.05,2969.66,0.477,✔️,14.29,True,GPU +20,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1362.0,+276/-242,0.418,24.86,7.39,20.095,22.26,1.556,➖,0.0,False,GPU +21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1356.0,+85/-58,0.164,25.29,24.18,17.736,2342.58,5.131,✔️,0.0,False,CPU +22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1356.0,+94/-75,0.179,25.29,23.38,17.42,2342.58,1.065,✔️,0.0,False,CPU +23,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1351.0,+252/-195,0.348,25.57,5.56,20.345,10182.26,40.124,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1349.0,+182/-133,0.26,25.71,14.75,20.76,19.13,0.146,✔️,0.0,False,GPU +25,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1339.0,+147/-114,0.182,26.36,21.78,18.825,6994.17,9.554,✔️,0.0,False,GPU +26,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1331.0,+193/-191,0.265,26.86,5.51,19.587,2149.3,0.147,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1321.0,+154/-140,0.144,27.5,21.88,19.122,6994.17,0.583,✔️,0.0,False,GPU +28,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1318.0,+162/-121,0.219,27.71,21.35,20.198,326.23,4.282,➖,0.0,False,GPU +29,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1302.0,+191/-229,0.226,28.79,13.55,22.319,772.2,2.495,✔️,0.0,False,CPU +30,🧠⚡,Foundation Model,[TabICL (default) [14.29% IMPUTED]](https://arxiv.org/abs/2502.05564),1300.0,+249/-187,0.251,28.93,7.93,24.353,11.7,2.132,✔️,14.29,True,GPU +31,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1284.0,+117/-70,0.111,30.0,27.91,19.353,57.99,0.317,✔️,0.0,False,CPU +32,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1277.0,+183/-217,0.211,30.5,13.67,22.113,772.2,0.373,✔️,0.0,False,CPU +33,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1273.0,+125/-98,0.131,30.71,24.16,21.656,44054.5,0.843,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1267.0,+135/-120,0.138,31.14,27.61,21.802,3833.57,2.294,✔️,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1260.0,+192/-270,0.218,31.57,18.92,22.276,731.01,2.021,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1258.0,+250/-215,0.191,31.71,20.7,23.176,2750.22,13.569,✔️,0.0,False,CPU +37,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1226.0,+236/-230,0.245,33.71,9.8,24.042,51.52,40.219,✔️,0.0,False,GPU +38,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1223.0,+182/-211,0.146,33.86,25.21,22.488,5.15,0.526,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1221.0,+182/-164,0.13,34.0,21.26,23.947,3833.57,0.22,✔️,0.0,False,CPU +40,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1216.0,+118/-133,0.102,34.29,31.15,23.403,14.99,0.113,✔️,0.0,False,CPU +41,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1214.0,+121/-121,0.117,34.43,30.23,23.217,44054.5,0.068,✔️,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1180.0,+136/-212,0.137,36.57,25.97,23.958,731.01,0.334,✔️,0.0,False,CPU +43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1168.0,+54/-73,0.0,37.29,36.45,21.807,5.38,0.624,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1166.0,+123/-178,0.072,37.43,33.02,23.313,20.63,7.532,✔️,0.0,False,GPU +45,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1161.0,+152/-221,0.063,37.71,32.57,24.905,9.4,1.444,✔️,0.0,False,GPU +46,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1144.0,+205/-212,0.093,38.71,30.39,25.292,2750.22,1.239,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1103.0,+121/-140,0.008,41.0,38.78,28.196,19.64,0.478,✔️,0.0,False,GPU +48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1084.0,+253/-448,0.148,42.0,25.75,35.799,180.38,1.24,✔️,0.0,False,CPU +49,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1008.0,+131/-208,0.0,45.71,44.11,30.387,14.87,0.391,✔️,0.0,False,CPU +50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+98/-268,0.0,46.07,44.54,37.213,0.78,0.155,✔️,0.0,False,CPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),978.0,+191/-292,0.025,47.0,43.11,31.994,8.78,0.688,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),959.0,+69/-219,0.0,47.79,46.87,39.676,913.1,0.738,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),944.0,+77/-256,0.0,48.36,47.28,40.327,913.1,0.13,✔️,0.0,False,CPU +54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),931.0,+172/-292,0.004,48.86,45.5,39.305,0.77,0.156,✔️,0.0,False,CPU +55,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),858.0,+282/-873,0.086,51.29,41.78,47.681,180.38,0.136,✔️,0.0,False,CPU +56,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),833.0,+82/-234,0.0,52.0,51.7,39.077,5.48,0.91,➖,0.0,False,GPU +57,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),741.0,+91/-267,0.0,54.14,53.83,46.939,4.65,0.14,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-282.0,+208/-523,0.0,59.0,59.0,69.099,0.58,0.071,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/winrate_matrix.png.zip index 37ef65eac2a3383690bfe1d0b4c340d40a9210df..3b9779f487e1b3ff312b8be70d1a9a71e627e796 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9b14f973849d0610508336e3cf37a5b7baba3b5749a01df77b92523541cce68a -size 1680410 +oid sha256:7aa02d31b8644ba935ae0b32dd8ceefc111df6b03476360dc0a031bd22111720 +size 1580710 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index f284860070fde89fe42c530436ff00de1185ce45..38dff2a9be3f3e953cf789572c9d8e056978a8d6 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0f7ae8bbbc3e75c7d49b8d2e4e66accc5c121fffc70cf9c4a09870837940041f -size 456644 +oid sha256:ef5db3b2eb900e313fdca371bcaf490675289b1a9a211b81364a0a9a2c90f02c +size 471777 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip index ecabc30996850927bb9172d488865868af89d1cf..9862bd8030a0b9107c00f3f5c699bdd02b76e3f5 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:62c9926537293f74a3c1f74893e269fe8d879ad4a61581db4a1e4cd5392c5a86 -size 975573 +oid sha256:1fb1cf46606fa26fdf077cd92ab99b22b766b17b97b05ddb1b91f24cf2e07fac +size 997903 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip index cb59ecdaf10f09e6b00bb0bec2055fc6bca66797..9da953c59a60d7210489a8a3320396426ecf806f 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:184bd71f14228e7409c6749e7953b06e4ee609f934c3b1e43df6acf04a02c4ac -size 125861 +oid sha256:04c46eebbfd3d2ab196667115b5992ba5222db712269afd32a65db94fc1d735b +size 131002 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv index a4c2ec6e6121067ab7b4a9e628750c998dfa2ed1..de4beda66c35f1f0daaabadfa2bfe9e326870a75 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,60 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1997.0,+809/-100,2.191,0.875,3.17,1.58,2793.65,8.93,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1857.0,+337/-82,12.495,0.729,5.42,4.12,2793.65,1.02,✔️,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1746.0,+356/-135,14.332,0.689,8.17,5.69,1295.16,14.354,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1716.0,+553/-144,11.083,0.632,9.08,4.46,9.88,0.825,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1688.0,+378/-116,14.698,0.596,10.0,5.22,3561.2,34.413,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1634.0,+358/-88,12.332,0.506,12.0,6.72,5.93,0.989,➖,0.0,False,GPU -6,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1626.0,+204/-38,15.748,0.513,12.33,10.85,206.59,4.778,✔️,0.0,False,GPU -7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1622.0,+272/-96,16.24,0.534,12.5,4.43,4334.8,21.159,✔️,0.0,False,GPU -8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1583.0,+233/-63,16.59,0.472,14.17,12.85,4334.8,3.529,✔️,0.0,False,GPU -9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1522.0,+224/-109,17.857,0.384,17.0,14.03,3307.58,1.182,✔️,0.0,False,CPU -10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1506.0,+327/-108,16.716,0.373,17.83,11.35,3561.2,0.612,✔️,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1502.0,+371/-116,14.808,0.388,18.0,9.36,5.48,0.434,✔️,0.0,False,GPU -12,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1489.0,+378/-152,18.377,0.401,18.67,4.6,6700.69,3.268,✔️,0.0,False,CPU -13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1476.0,+201/-74,18.68,0.307,19.33,15.87,3307.58,0.108,✔️,0.0,False,CPU -14,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1445.0,+344/-241,22.477,0.382,21.0,9.92,9411.86,433.416,✔️,0.0,False,GPU -15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1433.0,+268/-147,21.223,0.283,21.67,15.18,5568.31,1.781,✔️,0.0,False,GPU -16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1418.0,+175/-44,19.816,0.219,22.5,21.18,2029.77,1.073,✔️,0.0,False,CPU -17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1412.0,+142/-39,20.228,0.197,22.83,22.15,2029.77,6.631,✔️,0.0,False,CPU -18,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1412.0,+262/-84,19.85,0.213,22.83,16.26,1280.01,5.479,✔️,0.0,False,CPU -19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1409.0,+244/-141,21.671,0.272,23.0,14.96,5568.31,0.372,✔️,0.0,False,GPU -20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1403.0,+285/-92,19.946,0.219,23.33,14.81,1280.01,1.143,✔️,0.0,False,CPU -21,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1398.0,+205/-127,21.447,0.219,23.58,19.93,6829.34,7.818,✔️,0.0,False,GPU -22,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1394.0,+268/-198,20.929,0.246,23.83,18.35,2059.45,3.769,✔️,0.0,False,GPU -23,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1379.0,+437/-228,22.95,0.297,24.67,6.92,13.87,2.152,✔️,0.0,False,GPU -24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1375.0,+264/-171,21.793,0.169,24.92,20.15,6829.34,0.53,✔️,0.0,False,GPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1356.0,+255/-176,24.08,0.229,26.0,13.94,17.09,0.145,✔️,0.0,False,GPU -26,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1353.0,+340/-277,23.608,0.32,26.17,4.97,9411.86,42.924,✔️,0.0,False,GPU -27,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1319.0,+250/-164,23.403,0.191,28.17,20.83,410.08,4.104,➖,0.0,False,GPU -28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1302.0,+373/-354,26.238,0.223,29.17,18.56,2721.87,14.595,✔️,0.0,False,CPU -29,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1296.0,+161/-125,22.348,0.109,29.5,26.85,155.57,0.254,✔️,0.0,False,CPU -30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1278.0,+292/-226,22.852,0.145,30.5,21.21,2059.45,0.127,✔️,0.0,False,GPU -31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1270.0,+202/-155,25.058,0.115,31.0,23.64,27108.48,0.742,✔️,0.0,False,CPU -32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1264.0,+222/-200,25.243,0.119,31.33,27.49,4370.77,2.162,✔️,0.0,False,CPU -33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1246.0,+253/-279,26.005,0.12,32.33,25.27,728.32,2.548,✔️,0.0,False,CPU -34,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1235.0,+397/-370,27.727,0.284,33.0,8.74,42.75,45.46,✔️,0.0,False,GPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1217.0,+342/-353,25.893,0.148,34.0,18.85,729.17,2.154,✔️,0.0,False,CPU -36,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1211.0,+276/-254,25.765,0.102,34.33,25.35,728.32,0.356,✔️,0.0,False,CPU -37,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1190.0,+175/-206,26.917,0.075,35.5,30.79,27108.48,0.066,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1190.0,+176/-212,25.981,0.088,35.5,30.85,19.2,6.922,✔️,0.0,False,GPU -39,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1181.0,+89/-95,24.783,0.0,36.0,35.1,4.93,0.671,✔️,0.0,False,CPU -40,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1181.0,+170/-206,27.166,0.035,36.0,34.18,11.78,0.099,✔️,0.0,False,CPU -41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1159.0,+197/-173,26.155,0.06,37.17,32.88,5.12,0.444,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1156.0,+276/-281,28.736,0.113,37.33,27.67,2721.87,1.334,✔️,0.0,False,CPU -43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1153.0,+228/-281,28.727,0.078,37.5,31.64,8.99,1.498,✔️,0.0,False,GPU -44,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1146.0,+149/-137,27.899,0.006,37.83,36.17,4370.77,0.22,✔️,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1137.0,+257/-308,27.785,0.097,38.33,25.06,729.17,0.337,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1120.0,+154/-209,29.623,0.01,39.17,36.91,17.24,0.42,✔️,0.0,False,GPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+143/-432,37.954,0.0,44.67,42.99,0.74,0.148,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),979.0,+86/-362,38.521,0.0,45.5,44.67,909.67,0.893,✔️,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),962.0,+100/-303,39.28,0.0,46.17,45.13,909.67,0.107,✔️,0.0,False,CPU -50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),962.0,+232/-508,34.491,0.032,46.17,41.47,8.37,0.685,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),939.0,+300/-564,41.716,0.037,47.0,40.48,155.52,1.033,✔️,0.0,False,CPU -52,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),934.0,+277/-430,39.387,0.004,47.17,43.75,0.76,0.153,✔️,0.0,False,CPU -53,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),924.0,+96/-253,35.184,0.0,47.5,47.01,14.57,0.363,✔️,0.0,False,CPU -54,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),748.0,+71/-303,44.164,0.0,52.17,51.98,4.67,0.855,➖,0.0,False,GPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),723.0,+130/-462,44.545,0.0,52.67,52.34,4.62,0.135,✔️,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),372.0,+132/-1205,55.52,0.0,56.33,56.29,155.52,0.12,✔️,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-548.0,+343/-1220,70.032,0.0,58.0,58.0,0.52,0.058,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1985.0,+819/-92,0.874,3.17,1.58,2.191,2793.65,8.93,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1838.0,+303/-74,0.726,5.58,4.21,12.495,2793.65,1.02,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1727.0,+326/-137,0.684,8.5,5.83,14.332,1295.16,14.354,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1699.0,+521/-146,0.63,9.42,5.11,11.083,9.88,0.825,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1668.0,+331/-107,0.592,10.5,5.33,14.698,3561.2,34.413,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1616.0,+349/-106,0.503,12.5,7.2,12.332,5.93,0.989,➖,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1604.0,+232/-93,0.529,13.0,4.49,16.24,4334.8,21.159,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),1604.0,+181/-40,0.508,13.0,11.38,15.748,206.59,4.778,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1567.0,+189/-61,0.465,14.67,13.1,16.59,4334.8,3.529,✔️,0.0,False,GPU +9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1506.0,+200/-100,0.376,17.67,14.64,17.857,3307.58,1.182,✔️,0.0,False,CPU +10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1489.0,+288/-107,0.369,18.5,12.65,16.716,3561.2,0.612,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1486.0,+335/-131,0.385,18.67,11.14,14.808,5.48,0.434,✔️,0.0,False,GPU +12,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1477.0,+342/-144,0.402,19.17,4.67,18.377,6700.69,3.268,✔️,0.0,False,CPU +13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1461.0,+178/-67,0.299,20.0,16.28,18.68,3307.58,0.108,✔️,0.0,False,CPU +14,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1433.0,+592/-306,0.488,21.5,6.49,22.111,21.84,1.394,➖,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1430.0,+312/-243,0.375,21.67,10.03,22.477,9411.86,433.416,✔️,0.0,False,GPU +16,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1422.0,+263/-141,0.275,22.17,15.51,21.223,5568.31,1.781,✔️,0.0,False,GPU +17,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1405.0,+164/-47,0.209,23.17,21.97,19.816,2029.77,1.073,✔️,0.0,False,CPU +18,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1402.0,+229/-82,0.213,23.33,16.53,19.85,1280.01,5.479,✔️,0.0,False,CPU +19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1399.0,+221/-145,0.263,23.5,15.15,21.671,5568.31,0.372,✔️,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1399.0,+112/-36,0.191,23.5,22.92,20.228,2029.77,6.631,✔️,0.0,False,CPU +21,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1394.0,+265/-96,0.216,23.83,15.08,19.946,1280.01,1.143,✔️,0.0,False,CPU +22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1387.0,+180/-132,0.213,24.25,20.29,21.447,6829.34,7.818,✔️,0.0,False,GPU +23,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1382.0,+259/-209,0.241,24.5,19.06,20.929,2059.45,3.769,✔️,0.0,False,GPU +24,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1369.0,+370/-235,0.293,25.33,6.95,22.95,13.87,2.152,✔️,0.0,False,GPU +25,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1367.0,+232/-176,0.168,25.42,20.34,21.793,6829.34,0.53,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1346.0,+255/-174,0.224,26.67,14.13,24.08,17.09,0.145,✔️,0.0,False,GPU +27,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1343.0,+323/-276,0.318,26.83,4.99,23.608,9411.86,42.924,✔️,0.0,False,GPU +28,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1309.0,+234/-162,0.189,28.83,21.41,23.403,410.08,4.104,➖,0.0,False,GPU +29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1292.0,+336/-351,0.223,29.83,19.05,26.238,2721.87,14.595,✔️,0.0,False,CPU +30,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1287.0,+129/-116,0.107,30.17,27.74,22.348,155.57,0.254,✔️,0.0,False,CPU +31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1270.0,+286/-229,0.143,31.17,22.23,22.852,2059.45,0.127,✔️,0.0,False,GPU +32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1264.0,+194/-149,0.115,31.5,23.88,25.058,27108.48,0.742,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1253.0,+196/-195,0.114,32.17,28.09,25.243,4370.77,2.162,✔️,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1238.0,+248/-283,0.117,33.0,25.98,26.005,728.32,2.548,✔️,0.0,False,CPU +35,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1227.0,+379/-376,0.285,33.67,8.76,27.727,42.75,45.46,✔️,0.0,False,GPU +36,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1210.0,+334/-355,0.146,34.67,20.47,25.893,729.17,2.154,✔️,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1205.0,+263/-259,0.1,35.0,26.53,25.765,728.32,0.356,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1182.0,+162/-209,0.085,36.33,31.71,25.981,19.2,6.922,✔️,0.0,False,GPU +39,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1182.0,+160/-204,0.075,36.33,31.9,26.917,27108.48,0.066,✔️,0.0,False,CPU +40,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1176.0,+71/-116,0.0,36.67,35.78,24.783,4.93,0.671,✔️,0.0,False,CPU +41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1173.0,+142/-204,0.038,36.83,34.87,27.166,11.78,0.099,✔️,0.0,False,CPU +42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1155.0,+186/-181,0.053,37.83,33.78,26.155,5.12,0.444,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1149.0,+268/-333,0.108,38.17,29.05,28.736,2721.87,1.334,✔️,0.0,False,CPU +44,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1149.0,+225/-269,0.074,38.17,32.2,28.727,8.99,1.498,✔️,0.0,False,GPU +45,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1139.0,+123/-136,0.009,38.67,36.91,27.899,4370.77,0.22,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1133.0,+254/-292,0.095,39.0,26.77,27.785,729.17,0.337,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1117.0,+145/-209,0.01,39.83,37.58,29.623,17.24,0.42,✔️,0.0,False,GPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+140/-390,0.0,45.33,43.69,37.954,0.74,0.148,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),976.0,+75/-309,0.0,46.33,45.57,38.521,909.67,0.893,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),958.0,+93/-290,0.0,47.0,46.03,39.28,909.67,0.107,✔️,0.0,False,CPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),958.0,+222/-468,0.029,47.0,42.52,34.491,8.37,0.685,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),940.0,+269/-584,0.037,47.67,40.86,41.716,155.52,1.033,✔️,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),936.0,+247/-400,0.004,47.83,44.23,39.387,0.76,0.153,✔️,0.0,False,CPU +54,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),922.0,+94/-257,0.0,48.33,47.86,35.184,14.57,0.363,✔️,0.0,False,CPU +55,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),753.0,+61/-325,0.0,53.0,52.79,44.164,4.67,0.855,➖,0.0,False,GPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),728.0,+119/-375,0.0,53.5,53.2,44.545,4.62,0.135,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),396.0,+125/-910,0.0,57.17,57.13,55.52,155.52,0.12,✔️,0.0,False,CPU +58,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-404.0,+186/-922,0.0,59.0,59.0,70.032,0.52,0.058,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip index 9af854fe4ceb891a3a8ef250157b196f1a219df3..75138ec71edd2ef12ab9aa5436a2862434705d60 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:76fb2bc92d4d8abf3c42704a0af86d4692dd925e1bdbe4ff037276396e4297d8 -size 1681180 +oid sha256:9bcfc3965462f313133faf3f3c2b3c17454fc10f282dc10ec8c5198d9b7a76ce +size 1570107 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index f4e454eb2f688088e86471df8f04b143af190d0d..31481aacbff0b859a269238a1d515f2a36fcd062 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:58947d7e8350c9eaa0f743fcff74de42d93efc9a6f4a3f18fefd1f2645d1cf37 -size 442857 +oid sha256:1d9bf155d746f309fafa0567b515adc41bc65a412fe5fd8e0ed8cad5a11be8d3 +size 458530 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip index 5f55cb416b45055fe81a3af016c4a13424cde45e..5f57fa8d2bd014e4d008227005742e08773bfb1d 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7d9458d0b7dbd62842671c733157b291fdc86b830fba5a0563af2635466baaa3 -size 999231 +oid sha256:34fc3db00abbcbb69bc3fd54b6be83164d7a5b0fd9569b9eaf5253e4d9f01c35 +size 1011376 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/tuning-impact-elo.png.zip index 33c9f88cd146e2a665527f3e03ac2b4b1571ddfa..05b611dcb803c4edade6c6deceda91d84b4f52c7 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:57850ccb00e4965ad63f6c192e1dc8a67fc6c98114c4ba4758e878b2a280cfc4 -size 147318 +oid sha256:6b7d8ec5af70c2e4c260a40ed94577679069ebd2e792822df3a01a3cafd3254d +size 152925 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/website_leaderboard.csv index 7b6ff2f7933900472b48320c2cee1eb29dc6fdf4..6a38b5b926da4d40d13724da454a365abd285c75 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/website_leaderboard.csv @@ -1,56 +1,57 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1878.0,+277/-162,1.248,0.798,5.46,2.46,2043.44,8.369,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1798.0,+204/-85,2.558,0.729,7.46,2.38,7125.58,336.907,✔️,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1798.0,+115/-60,2.649,0.652,7.46,4.37,616.43,4.041,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1761.0,+145/-59,3.061,0.677,8.54,4.62,7125.58,38.573,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1751.0,+194/-142,2.186,0.672,8.85,3.38,2043.44,0.781,✔️,0.0,False,GPU -5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1695.0,+96/-70,4.784,0.535,10.77,5.86,1489.94,5.79,✔️,0.0,False,CPU -6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1692.0,+106/-54,4.256,0.573,10.85,8.82,4057.12,10.478,✔️,0.0,False,GPU -7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1642.0,+178/-151,3.66,0.535,12.77,7.33,6.94,0.519,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1642.0,+150/-73,4.24,0.542,12.77,6.52,44.65,38.298,✔️,0.0,False,GPU -9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1608.0,+154/-138,6.753,0.501,14.15,5.57,3811.43,7.581,✔️,0.0,False,GPU -10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1566.0,+126/-126,5.413,0.43,16.0,11.47,4057.12,0.337,✔️,0.0,False,GPU -11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1560.0,+92/-63,7.383,0.332,16.31,13.64,686.46,5.477,✔️,0.0,False,CPU -12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1535.0,+82/-66,7.168,0.293,17.46,13.48,2895.38,1.317,✔️,0.0,False,CPU -13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1516.0,+109/-75,6.559,0.362,18.38,14.25,708.95,1.973,✔️,0.0,False,GPU -14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1506.0,+135/-95,5.831,0.314,18.85,15.04,4228.53,1.188,✔️,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1498.0,+109/-88,7.292,0.272,19.23,10.75,2895.38,0.073,✔️,0.0,False,CPU -16,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1481.0,+208/-182,13.045,0.407,20.08,5.06,5.49,0.754,➖,0.0,False,GPU -17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1475.0,+108/-87,8.18,0.243,20.38,16.02,686.46,0.743,✔️,0.0,False,CPU -18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1464.0,+95/-64,7.123,0.268,20.92,17.27,708.95,0.069,✔️,0.0,False,GPU -19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1439.0,+58/-51,8.102,0.18,22.15,20.62,848.99,0.475,✔️,0.0,False,CPU -20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1437.0,+53/-46,8.118,0.181,22.23,21.12,848.99,2.378,✔️,0.0,False,CPU -21,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1427.0,+106/-89,9.404,0.221,22.77,19.49,16.07,0.29,✔️,0.0,False,GPU -22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1412.0,+113/-82,6.664,0.201,23.54,20.78,4228.53,0.125,✔️,0.0,False,GPU -23,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1404.0,+126/-138,9.279,0.265,23.92,14.23,3811.43,0.452,✔️,0.0,False,GPU -24,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1403.0,+211/-142,6.396,0.322,23.96,5.75,3805.62,10.413,✔️,46.15,True,GPU -25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1398.0,+116/-125,9.254,0.188,24.23,19.58,8.35,0.087,✔️,0.0,False,CPU -26,🧠⚡,Foundation Model,[Mitra (default) [46.15% IMPUTED]](https://arxiv.org/abs/2510.21204),1350.0,+167/-112,8.579,0.214,26.69,12.3,70.97,1.59,✔️,46.15,True,GPU -27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1336.0,+123/-115,8.326,0.142,27.46,24.43,13.9,0.123,✔️,0.0,False,GPU -28,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1308.0,+180/-154,7.904,0.218,28.88,16.23,3805.62,0.262,✔️,46.15,True,GPU -29,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1289.0,+126/-104,10.165,0.085,29.85,26.13,9.17,1.416,✔️,0.0,False,GPU -30,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1281.0,+113/-112,12.346,0.103,30.23,25.29,161.73,0.784,✔️,0.0,False,CPU -31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1277.0,+89/-103,9.496,0.071,30.46,27.55,4452.11,0.852,✔️,0.0,False,CPU -32,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1261.0,+114/-110,12.658,0.116,31.23,25.13,161.73,0.12,✔️,0.0,False,CPU -33,🧠⚡,Foundation Model,[TabPFNv2 (default) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1256.0,+184/-178,9.06,0.19,31.46,19.12,2.78,0.315,✔️,46.15,True,GPU -34,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1234.0,+173/-152,12.992,0.084,32.54,14.85,2.47,0.755,✔️,0.0,False,GPU -35,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1221.0,+114/-126,10.114,0.054,33.15,29.53,4452.11,0.089,✔️,0.0,False,CPU -36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1220.0,+72/-95,10.993,0.031,33.23,31.93,2.03,0.296,✔️,0.0,False,CPU -37,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1191.0,+94/-114,11.709,0.062,34.62,31.76,2.15,0.18,✔️,0.0,False,CPU -38,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1186.0,+155/-179,13.499,0.118,34.85,23.77,3017.28,0.248,✔️,0.0,False,CPU -39,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1179.0,+69/-87,13.32,0.016,35.15,33.41,526.17,0.77,✔️,0.0,False,CPU -40,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1144.0,+148/-191,14.059,0.118,36.77,19.83,3017.28,0.027,✔️,0.0,False,CPU -41,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1134.0,+88/-107,14.071,0.015,37.19,35.35,526.17,0.119,✔️,0.0,False,CPU -42,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1113.0,+101/-107,14.254,0.023,38.08,35.85,0.42,0.061,✔️,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1069.0,+83/-106,14.361,0.004,39.85,38.7,527.21,2.831,✔️,0.0,False,CPU -44,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1042.0,+93/-142,14.199,0.0,40.85,39.11,20.5,0.081,✔️,0.0,False,CPU -45,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1030.0,+109/-172,15.156,0.05,41.31,38.59,10.04,0.03,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1026.0,+102/-118,14.845,0.0,41.46,40.27,527.21,0.311,✔️,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+66/-87,15.385,0.0,42.35,41.83,0.63,0.064,✔️,0.0,False,CPU -48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),881.0,+101/-164,18.896,0.0,45.85,45.08,3.08,0.293,✔️,0.0,False,CPU -49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),859.0,+134/-176,20.364,0.0,46.38,45.51,90.87,0.906,✔️,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),705.0,+107/-204,22.511,0.0,49.38,49.19,90.87,0.055,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),571.0,+216/-360,29.184,0.0,51.15,50.64,0.25,0.044,✔️,0.0,False,CPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),425.0,+111/-304,36.25,0.0,52.58,52.52,177.04,0.14,✔️,0.0,False,CPU -53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),338.0,+142/-400,36.435,0.0,53.27,53.2,177.04,0.059,✔️,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),187.0,+134/-333,38.891,0.0,54.23,54.22,0.88,0.109,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1867.0,+269/-162,0.799,5.54,2.46,1.248,2043.44,8.369,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1788.0,+197/-83,0.731,7.54,2.39,2.558,7125.58,336.907,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1786.0,+118/-58,0.65,7.62,4.39,2.649,616.43,4.041,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1752.0,+145/-59,0.677,8.62,4.65,3.061,7125.58,38.573,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1740.0,+192/-139,0.673,9.0,3.43,2.186,2043.44,0.781,✔️,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1683.0,+101/-71,0.535,11.0,5.9,4.784,1489.94,5.79,✔️,0.0,False,CPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1683.0,+103/-55,0.575,11.0,8.92,4.256,4057.12,10.478,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1634.0,+175/-149,0.537,12.92,7.36,3.66,6.94,0.519,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1632.0,+149/-71,0.543,13.0,6.56,4.24,44.65,38.298,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1599.0,+154/-139,0.503,14.38,5.58,6.753,3811.43,7.581,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1558.0,+126/-126,0.43,16.23,11.55,5.413,4057.12,0.337,✔️,0.0,False,GPU +11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1550.0,+98/-64,0.332,16.62,13.77,7.383,686.46,5.477,✔️,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1527.0,+82/-65,0.292,17.69,13.58,7.168,2895.38,1.317,✔️,0.0,False,CPU +13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1507.0,+108/-72,0.361,18.69,14.39,6.559,708.95,1.973,✔️,0.0,False,GPU +14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1498.0,+135/-94,0.315,19.15,15.21,5.831,4228.53,1.188,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1490.0,+110/-88,0.273,19.54,10.81,7.292,2895.38,0.073,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1476.0,+205/-177,0.41,20.23,5.11,13.045,5.49,0.754,➖,0.0,False,GPU +17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1466.0,+110/-87,0.247,20.77,16.15,8.18,686.46,0.743,✔️,0.0,False,CPU +18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1456.0,+97/-60,0.268,21.23,17.48,7.123,708.95,0.069,✔️,0.0,False,GPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1431.0,+57/-51,0.18,22.54,20.94,8.102,848.99,0.475,✔️,0.0,False,CPU +20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1428.0,+54/-46,0.182,22.69,21.49,8.118,848.99,2.378,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1418.0,+105/-89,0.223,23.23,19.88,9.404,16.07,0.29,✔️,0.0,False,GPU +22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1404.0,+111/-83,0.204,23.92,21.05,6.664,4228.53,0.125,✔️,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1399.0,+208/-138,0.319,24.19,5.76,6.396,3805.62,10.413,✔️,46.15,True,GPU +24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1397.0,+123/-136,0.267,24.31,14.42,9.279,3811.43,0.452,✔️,0.0,False,GPU +25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1388.0,+115/-127,0.19,24.77,19.81,9.254,8.35,0.087,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[Mitra (default) [46.15% IMPUTED]](https://arxiv.org/abs/2510.21204),1344.0,+162/-107,0.211,27.15,12.56,8.579,70.97,1.59,✔️,46.15,True,GPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1331.0,+119/-112,0.146,27.85,24.69,8.326,13.9,0.123,✔️,0.0,False,GPU +28,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1304.0,+176/-155,0.216,29.27,16.33,7.904,3805.62,0.262,✔️,46.15,True,GPU +29,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1297.0,+180/-162,0.179,29.62,17.41,9.647,18.76,6.542,➖,0.0,False,GPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1283.0,+119/-105,0.084,30.38,26.79,10.165,9.17,1.416,✔️,0.0,False,GPU +31,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1278.0,+111/-112,0.103,30.62,25.56,12.346,161.73,0.784,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1272.0,+84/-101,0.075,30.92,27.89,9.496,4452.11,0.852,✔️,0.0,False,CPU +33,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1259.0,+114/-110,0.116,31.62,25.41,12.658,161.73,0.12,✔️,0.0,False,CPU +34,🧠⚡,Foundation Model,[TabPFNv2 (default) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1254.0,+180/-178,0.187,31.85,19.31,9.06,2.78,0.315,✔️,46.15,True,GPU +35,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1231.0,+170/-152,0.085,33.0,14.91,12.992,2.47,0.755,✔️,0.0,False,GPU +36,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1219.0,+111/-122,0.055,33.62,29.95,10.114,4452.11,0.089,✔️,0.0,False,CPU +37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1214.0,+70/-95,0.032,33.85,32.42,10.993,2.03,0.296,✔️,0.0,False,CPU +38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1184.0,+93/-116,0.061,35.31,32.28,11.709,2.15,0.18,✔️,0.0,False,CPU +39,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1181.0,+151/-182,0.118,35.46,24.02,13.499,3017.28,0.248,✔️,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1175.0,+66/-83,0.013,35.77,34.08,13.32,526.17,0.77,✔️,0.0,False,CPU +41,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1137.0,+145/-188,0.118,37.54,20.03,14.059,3017.28,0.027,✔️,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1131.0,+87/-106,0.012,37.81,36.0,14.071,526.17,0.119,✔️,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1108.0,+102/-110,0.021,38.85,36.58,14.254,0.42,0.061,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1065.0,+82/-107,0.004,40.62,39.37,14.361,527.21,2.831,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1039.0,+90/-143,0.002,41.62,39.79,14.199,20.5,0.081,✔️,0.0,False,CPU +46,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1027.0,+108/-169,0.05,42.08,39.26,15.156,10.04,0.03,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1023.0,+102/-119,0.0,42.23,40.98,14.845,527.21,0.311,✔️,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+65/-90,0.0,43.04,42.5,15.385,0.63,0.064,✔️,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),880.0,+103/-164,0.0,46.69,45.87,18.896,3.08,0.293,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),858.0,+126/-174,0.0,47.23,46.37,20.364,90.87,0.906,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),699.0,+105/-201,0.0,50.38,50.19,22.511,90.87,0.055,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),565.0,+214/-357,0.0,52.15,51.65,29.184,0.25,0.044,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),419.0,+111/-304,0.0,53.58,53.52,36.25,177.04,0.14,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),332.0,+140/-396,0.0,54.27,54.2,36.435,177.04,0.059,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),182.0,+134/-329,0.0,55.23,55.22,38.891,0.88,0.109,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/winrate_matrix.png.zip index 314a855f897cde724e35293cd027fc800a091b96..a1a2c02182eec4d63f5467f623dd4bbdf793fa1d 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d827b9fd6dc3715e44eccd4345416eeb09dae7aa6d65061db10a527e9017d1d7 -size 1701640 +oid sha256:fcc1156e620a6ebba017f34843fc10fdfdec66dc83fd5d559ea81bdbe50a36be +size 1836209 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index 721961c74828795949b74a1db9199f9e133ec4f6..6af6ec1b003d376c99deb4aba358ce1674234038 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:eca5e40122891bdaebd55352bab16cac3a713dbf3dbcea3a35e62ba124e87033 -size 430652 +oid sha256:30dd630add01f158857d7535a53661173f35f5b81092a7d5d0509fb2b67c1bb6 +size 446846 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip index 8c78f812b874b0965ccd16761fd7a6189caf5b28..fd98aee1d17436534ee616c9dfd24e1504019e0e 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:79e9ab80ab665190263dcfce02068ccad00f1173f418eb09277c70384228b1f8 -size 917540 +oid sha256:ad08f57ee314c96ee2588256364ea62b2efb342e0dfc5007430246cf3350e3b2 +size 926558 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/tuning-impact-elo.png.zip index 371a533611da751d5b0376b202773135094e8a50..022ab18b6ed99153d0715daa1824322d42f42f3e 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2352a4f60b7396cae96db8c068b1cf67e91ee92354640f18cf713f1d8e25af2c -size 119314 +oid sha256:4985a030f4aab3e01583891d2c177c20ec4046f71ccb8469ae92f6f777c18ffb +size 123795 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/website_leaderboard.csv index 46a7e63e483c8172dd9737f1097129e0df9c843a..f9b0f8f9009f7ede0577dbfc4e85a81676205597 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/website_leaderboard.csv @@ -1,52 +1,53 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),2061.0,+1014/-71,1.163,0.832,4.4,3.24,739.1,11.65,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),2027.0,+1086/-121,1.262,0.835,5.0,2.74,739.1,0.689,✔️,0.0,False,GPU -2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1950.0,+850/-36,2.284,0.699,6.6,5.71,2552.28,4.522,✔️,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1916.0,+658/-8,3.209,0.655,7.4,3.4,474.94,11.837,✔️,0.0,False,CPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1885.0,+615/-13,3.513,0.612,8.2,6.09,400.37,2.16,✔️,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1870.0,+930/-101,2.851,0.637,8.6,6.91,2068.17,1.355,✔️,0.0,False,GPU -6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1822.0,+1355/-177,2.083,0.687,10.0,2.25,4529.88,357.209,✔️,0.0,False,GPU -7,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1790.0,+1289/-174,2.39,0.65,11.0,4.14,4529.88,39.544,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1783.0,+1058/-258,2.625,0.548,11.2,6.42,1.59,0.506,✔️,0.0,False,GPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1688.0,+928/-198,4.112,0.485,14.4,10.08,2068.17,0.068,✔️,0.0,False,GPU -10,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1677.0,+780/-243,4.058,0.471,14.8,11.42,2552.28,0.201,✔️,0.0,False,GPU -11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1677.0,+503/-139,5.836,0.377,14.8,11.88,223.05,9.707,✔️,0.0,False,CPU -12,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1671.0,+1088/-218,3.203,0.526,15.0,6.19,41.5,38.298,✔️,0.0,False,GPU -13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1660.0,+548/-97,6.007,0.349,15.4,12.03,223.05,0.793,✔️,0.0,False,CPU -14,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1649.0,+524/-91,4.481,0.418,15.8,13.49,438.55,0.52,✔️,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1644.0,+510/-102,6.235,0.3,16.0,10.62,1696.82,0.538,✔️,0.0,False,CPU -16,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1617.0,+754/-226,5.187,0.336,17.0,12.43,3456.17,0.611,✔️,0.0,False,GPU -17,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1590.0,+500/-169,6.475,0.28,18.0,7.25,1696.82,0.055,✔️,0.0,False,CPU -18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1559.0,+428/-97,5.477,0.253,19.2,17.73,438.55,0.035,✔️,0.0,False,GPU -19,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1553.0,+728/-163,5.3,0.287,19.4,16.95,6.87,0.098,✔️,0.0,False,GPU -20,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1537.0,+750/-208,4.7,0.306,20.0,4.21,2.07,0.754,➖,0.0,False,GPU -21,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1521.0,+561/-275,7.061,0.27,20.6,16.66,4.99,0.047,✔️,0.0,False,CPU -22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1505.0,+528/-187,6.188,0.183,21.2,19.63,3456.17,0.037,✔️,0.0,False,GPU -23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1494.0,+356/-58,6.853,0.189,21.6,20.89,289.35,1.798,✔️,0.0,False,CPU -24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1484.0,+386/-70,6.921,0.181,22.0,21.09,289.35,0.36,✔️,0.0,False,CPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1435.0,+648/-216,7.402,0.129,23.8,21.42,5.37,0.025,✔️,0.0,False,GPU -26,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1353.0,+354/-228,8.025,0.137,26.8,24.63,0.57,0.074,✔️,0.0,False,CPU -27,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1302.0,+357/-139,8.909,0.002,28.6,27.72,2.32,0.093,✔️,0.0,False,GPU -28,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1297.0,+375/-186,8.225,0.018,28.8,28.08,0.66,0.213,✔️,0.0,False,CPU -29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1274.0,+225/-245,8.161,0.064,29.6,27.65,4108.24,0.479,✔️,0.0,False,CPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1250.0,+508/-284,9.603,0.028,30.4,28.51,6.54,0.284,✔️,0.0,False,GPU -31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1221.0,+204/-296,9.253,0.029,31.4,29.47,4108.24,0.037,✔️,0.0,False,CPU -32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1172.0,+353/-451,10.288,0.023,33.0,31.22,35.47,0.502,✔️,0.0,False,CPU -33,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1147.0,+417/-890,13.625,0.144,33.8,25.8,1820.58,0.133,✔️,0.0,False,CPU -34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1140.0,+314/-411,10.454,0.006,34.0,32.83,35.47,0.054,✔️,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1134.0,+107/-401,10.053,0.0,34.2,33.96,79.96,0.571,✔️,0.0,False,CPU -36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1124.0,+128/-390,10.217,0.0,34.5,34.09,79.96,0.053,✔️,0.0,False,CPU -37,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1061.0,+165/-350,13.469,0.0,36.4,35.28,14.05,0.029,✔️,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1033.0,+387/-674,12.12,0.006,37.2,35.12,0.18,0.054,✔️,0.0,False,CPU -39,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1026.0,+316/-953,14.242,0.1,37.4,34.8,1820.58,0.012,✔️,0.0,False,CPU -40,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+78/-318,11.35,0.0,38.1,37.96,0.42,0.053,✔️,0.0,False,CPU -41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),996.0,+363/-1057,14.681,0.12,38.2,34.58,3.47,0.01,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),871.0,+51/-752,14.269,0.0,41.2,40.95,317.48,0.719,✔️,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),842.0,+43/-769,14.827,0.0,41.8,41.54,317.48,0.127,✔️,0.0,False,CPU -44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),800.0,+94/-889,18.666,0.0,42.6,42.42,142.18,1.027,✔️,0.0,False,CPU -45,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),662.0,+74/-902,21.083,0.0,44.8,44.73,142.18,0.057,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),662.0,+107/-831,18.199,0.0,44.8,44.56,1.33,0.126,✔️,0.0,False,CPU -47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),361.0,+132/-1452,28.294,0.0,47.6,47.49,0.06,0.036,✔️,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),145.0,+92/-1782,39.482,0.0,48.8,48.8,48.35,0.054,✔️,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),-119.0,+197/-2129,39.829,0.0,50.0,49.99,48.35,0.013,✔️,0.0,False,CPU -50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),-276.0,+185/-2881,42.789,0.0,50.6,50.6,0.13,0.02,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),2034.0,+922/-55,0.839,4.4,3.24,1.163,739.1,11.65,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),2001.0,+923/-105,0.841,5.0,2.74,1.262,739.1,0.689,✔️,0.0,False,GPU +2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1924.0,+642/-41,0.707,6.6,5.71,2.284,2552.28,4.522,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1883.0,+581/-12,0.661,7.6,3.42,3.209,474.94,11.837,✔️,0.0,False,CPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1853.0,+654/-28,0.618,8.4,6.13,3.513,400.37,2.16,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1846.0,+767/-85,0.648,8.6,6.91,2.851,2068.17,1.355,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1797.0,+975/-163,0.69,10.0,2.25,2.083,4529.88,357.209,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1765.0,+911/-166,0.654,11.0,4.14,2.39,4529.88,39.544,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1759.0,+894/-242,0.556,11.2,6.42,2.625,1.59,0.506,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1666.0,+707/-190,0.489,14.4,10.08,4.112,2068.17,0.068,✔️,0.0,False,GPU +10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1649.0,+507/-132,0.386,15.0,11.94,5.836,223.05,9.707,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1649.0,+700/-222,0.474,15.0,11.66,4.058,2552.28,0.201,✔️,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1644.0,+794/-200,0.533,15.2,6.22,3.203,41.5,38.298,✔️,0.0,False,GPU +13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1633.0,+559/-100,0.358,15.6,12.08,6.007,223.05,0.793,✔️,0.0,False,CPU +14,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1622.0,+581/-74,0.433,16.0,13.61,4.481,438.55,0.52,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1617.0,+561/-93,0.312,16.2,10.68,6.235,1696.82,0.538,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1590.0,+730/-206,0.343,17.2,12.6,5.187,3456.17,0.611,✔️,0.0,False,GPU +17,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1564.0,+548/-151,0.29,18.2,7.27,6.475,1696.82,0.055,✔️,0.0,False,CPU +18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1533.0,+484/-92,0.272,19.4,17.89,5.477,438.55,0.035,✔️,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1528.0,+545/-151,0.295,19.6,17.32,5.3,6.87,0.098,✔️,0.0,False,GPU +20,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1518.0,+706/-196,0.316,20.0,4.21,4.7,2.07,0.754,➖,0.0,False,GPU +21,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1492.0,+542/-270,0.28,21.0,16.83,7.061,4.99,0.047,✔️,0.0,False,CPU +22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1482.0,+582/-167,0.196,21.4,19.92,6.188,3456.17,0.037,✔️,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1471.0,+322/-53,0.203,21.8,21.03,6.853,289.35,1.798,✔️,0.0,False,CPU +24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1461.0,+336/-66,0.196,22.2,21.22,6.921,289.35,0.36,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1415.0,+648/-201,0.137,24.0,21.66,7.402,5.37,0.025,✔️,0.0,False,GPU +26,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1331.0,+404/-207,0.14,27.2,24.88,8.025,0.57,0.074,✔️,0.0,False,CPU +27,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1288.0,+318/-106,0.006,28.8,27.84,8.909,2.32,0.093,✔️,0.0,False,GPU +28,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1283.0,+418/-166,0.03,29.0,28.27,8.225,0.66,0.213,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1261.0,+222/-185,0.067,29.8,27.82,8.161,4108.24,0.479,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1234.0,+491/-233,0.028,30.8,28.77,9.603,6.54,0.284,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1212.0,+198/-242,0.032,31.6,29.67,9.253,4108.24,0.037,✔️,0.0,False,CPU +32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1166.0,+346/-383,0.034,33.2,31.32,10.288,35.47,0.502,✔️,0.0,False,CPU +33,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1137.0,+305/-327,0.017,34.2,32.95,10.454,35.47,0.054,✔️,0.0,False,CPU +34,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1137.0,+446/-460,0.121,34.2,25.03,10.329,8.68,7.525,➖,0.0,False,GPU +35,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1131.0,+406/-883,0.147,34.4,26.06,13.625,1820.58,0.133,✔️,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1131.0,+95/-238,0.0,34.4,34.11,10.053,79.96,0.571,✔️,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1122.0,+109/-229,0.0,34.7,34.23,10.217,79.96,0.053,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1050.0,+152/-316,0.0,37.0,35.77,13.469,14.05,0.029,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1024.0,+376/-520,0.017,37.8,35.52,12.12,0.18,0.054,✔️,0.0,False,CPU +40,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1010.0,+309/-975,0.104,38.2,35.39,14.242,1820.58,0.012,✔️,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+63/-264,0.0,38.5,38.32,11.35,0.42,0.053,✔️,0.0,False,CPU +42,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),982.0,+354/-1020,0.124,39.0,35.12,14.681,3.47,0.01,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),864.0,+81/-736,0.0,42.0,41.71,14.269,317.48,0.719,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),836.0,+84/-773,0.0,42.6,42.32,14.827,317.48,0.127,✔️,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),807.0,+90/-732,0.0,43.2,42.98,18.666,142.18,1.027,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),664.0,+109/-823,0.0,45.6,45.3,18.199,1.33,0.126,✔️,0.0,False,CPU +47,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),649.0,+52/-837,0.0,45.8,45.73,21.083,142.18,0.057,✔️,0.0,False,CPU +48,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),344.0,+125/-1382,0.0,48.6,48.5,28.294,0.06,0.036,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),133.0,+78/-1763,0.0,49.8,49.8,39.482,48.35,0.054,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),-121.0,+166/-2493,0.0,51.0,50.99,39.829,48.35,0.013,✔️,0.0,False,CPU +51,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),-274.0,+170/-3339,0.0,51.6,51.6,42.789,0.13,0.02,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/winrate_matrix.png.zip index c9ee70930761bab92116f38e9bb15d5da129797a..3bc3533f3cc9ea74086951df178e94094dab3b05 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8b24bec781fb0ff276e1e91e681472c7916ffdd71ab904ecd6bfe910f0030436 -size 1244143 +oid sha256:66a9eab7cd3500137e2dc44a64e2f84e4fe62684e89e59c1a24c71359d506913 +size 1348875 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index a66a426a1ea2d2d43dbf52c8d58e39010aa8f693..76233ddaddadaad75d905c9a8f3920b19306b3c0 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:91438b089230ed8d01b23828976c184e416bd7cdc717743f5ac54356f4f1ba29 -size 449846 +oid sha256:270b4421a2c13eab7d6ca16b4fafb29cfe054d3f97c576571db0bd474f5ff7d9 +size 465094 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip index 961a5cdae84636659065451f3fd4094d71aa43d3..d13b52a278a682c16cef1d91f6fe07bbc6addcf6 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1b3f59902166219b8d17fc77a944a58e13574015e6db771f42e34d90d6736fa2 -size 1051427 +oid sha256:7d45ac5a641769e40401b125eef2221d9ddf0215415bf0d448636b4b037cc1c1 +size 1072530 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/tuning-impact-elo.png.zip index 77275335e1cb3573d6eb14b725254c870aaf735c..d9ea41203b9c9dead56ab3d1ba71d7725e70b2a0 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c40e5cd50d050b4fd0512f1b5ee53ddf9fd865cbceda45700df563f7976487d2 -size 159497 +oid sha256:0c2f292c7b50e9296fe7db5009f55f7dbe148d3f3c3533e5a5063d299303999f +size 163993 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/website_leaderboard.csv index 84c89880db752e30929ad764896b0f5f71dd6e7e..0fb5bc1ef5b78cad49821fe3eb6969a0438acd4b 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/website_leaderboard.csv @@ -1,56 +1,57 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1891.0,+265/-104,2.855,0.738,5.88,2.47,9830.47,232.266,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1880.0,+464/-211,1.301,0.767,6.12,2.14,2355.62,7.55,✔️,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1846.0,+282/-80,2.109,0.656,7.0,3.72,716.17,4.181,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1846.0,+180/-36,3.48,0.674,7.0,4.99,9830.47,22.221,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1713.0,+345/-190,2.764,0.561,11.25,3.97,2355.62,0.879,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1710.0,+158/-52,4.888,0.533,11.38,6.74,51.27,43.085,✔️,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1687.0,+130/-51,5.134,0.512,12.25,10.65,4918.07,35.607,✔️,0.0,False,GPU -7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1672.0,+142/-65,5.768,0.442,12.88,10.65,4919.72,5.372,✔️,0.0,False,CPU -8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1651.0,+309/-190,4.306,0.51,13.75,8.04,8.24,0.532,✔️,0.0,False,GPU -9,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1644.0,+544/-239,3.301,0.523,14.06,3.75,6437.34,56.116,✔️,12.5,True,GPU -10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1577.0,+165/-113,6.226,0.381,17.0,12.55,4918.07,0.949,✔️,0.0,False,GPU -11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1572.0,+116/-77,8.35,0.272,17.25,15.04,1206.7,4.573,✔️,0.0,False,CPU -12,🧠⚡,Foundation Model,[Mitra (default) [12.50% IMPUTED]](https://arxiv.org/abs/2510.21204),1552.0,+246/-167,6.847,0.348,18.19,8.58,94.13,2.176,✔️,12.5,True,GPU -13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1548.0,+124/-62,7.752,0.252,18.38,16.21,3495.99,1.462,✔️,0.0,False,CPU -14,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1538.0,+206/-133,9.546,0.358,18.88,5.49,9006.35,8.016,✔️,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1515.0,+188/-91,7.803,0.233,20.0,15.41,3495.99,0.148,✔️,0.0,False,CPU -16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1515.0,+141/-90,7.857,0.291,20.0,14.78,1672.82,2.447,✔️,0.0,False,GPU -17,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1515.0,+153/-109,6.233,0.277,20.0,17.31,4322.16,1.598,✔️,0.0,False,GPU -18,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1513.0,+408/-258,18.26,0.453,20.12,5.79,5.86,0.71,➖,0.0,False,GPU -19,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1476.0,+150/-74,8.152,0.231,22.0,16.98,1672.82,0.097,✔️,0.0,False,GPU -20,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1475.0,+382/-238,5.751,0.355,22.06,11.85,6437.34,0.557,✔️,12.5,True,GPU -21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1472.0,+94/-57,8.84,0.136,22.25,20.33,1277.27,0.648,✔️,0.0,False,CPU -22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1465.0,+89/-51,8.909,0.133,22.62,21.26,1277.27,3.746,✔️,0.0,False,CPU -23,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1448.0,+125/-102,9.538,0.144,23.5,20.2,1206.7,0.689,✔️,0.0,False,CPU -24,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1421.0,+141/-132,11.969,0.154,24.88,21.51,25.13,0.294,✔️,0.0,False,GPU -25,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1419.0,+162/-120,6.961,0.179,25.0,21.57,4322.16,0.198,✔️,0.0,False,GPU -26,🧠⚡,Foundation Model,[TabPFNv2 (default) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1400.0,+328/-281,7.63,0.308,25.94,14.4,8.68,0.726,✔️,12.5,True,GPU -27,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1389.0,+147/-121,10.625,0.102,26.5,21.98,10.03,0.115,✔️,0.0,False,CPU -28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1360.0,+148/-186,13.632,0.138,28.0,22.48,644.11,0.953,✔️,0.0,False,CPU -29,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1340.0,+168/-199,14.036,0.169,29.0,21.79,644.11,0.187,✔️,0.0,False,CPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1340.0,+150/-102,10.515,0.113,29.0,24.67,22.41,7.82,✔️,0.0,False,GPU -31,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1328.0,+188/-190,12.542,0.12,29.62,16.81,9006.35,0.527,✔️,0.0,False,GPU -32,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1326.0,+164/-137,8.903,0.12,29.75,26.79,25.53,0.154,✔️,0.0,False,GPU -33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1311.0,+132/-150,10.331,0.067,30.5,27.26,5194.73,1.75,✔️,0.0,False,CPU -34,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1257.0,+190/-194,10.652,0.066,33.25,29.07,5194.73,0.121,✔️,0.0,False,CPU -35,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1241.0,+217/-212,13.421,0.097,34.0,22.26,3991.28,0.402,✔️,0.0,False,CPU -36,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1222.0,+233/-228,13.945,0.122,34.88,15.45,3991.28,0.034,✔️,0.0,False,CPU -37,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1220.0,+214/-280,15.544,0.121,35.0,11.51,7.0,1.362,✔️,0.0,False,GPU -38,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1214.0,+100/-178,15.363,0.026,35.25,32.69,657.99,1.135,✔️,0.0,False,CPU -39,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1198.0,+92/-125,12.723,0.015,36.0,34.93,4.54,0.374,✔️,0.0,False,CPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1175.0,+76/-124,14.418,0.006,37.0,35.99,2150.84,7.995,✔️,0.0,False,CPU -41,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1160.0,+97/-168,15.588,0.018,37.62,35.75,0.46,0.117,✔️,0.0,False,CPU -42,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1148.0,+114/-214,16.48,0.025,38.12,35.51,657.99,0.147,✔️,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1121.0,+107/-158,14.856,0.0,39.25,38.02,2150.84,0.566,✔️,0.0,False,CPU -44,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1114.0,+86/-123,14.012,0.0,39.5,38.78,4.08,0.255,✔️,0.0,False,CPU -45,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1055.0,+114/-185,15.454,0.0,41.75,40.45,12.71,0.039,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1030.0,+146/-217,14.656,0.0,42.62,41.09,22.25,0.131,✔️,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+92/-204,17.907,0.0,43.62,42.96,0.69,0.125,✔️,0.0,False,CPU -48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),988.0,+106/-216,19.332,0.0,44.0,43.14,6.77,0.511,✔️,0.0,False,CPU -49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),907.0,+147/-333,21.426,0.0,46.25,44.95,88.79,0.572,✔️,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),734.0,+153/-330,23.404,0.0,49.75,49.48,88.79,0.052,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),658.0,+224/-609,29.74,0.0,50.88,50.11,0.43,0.047,✔️,0.0,False,CPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),523.0,+100/-474,34.23,0.0,52.44,52.35,742.99,0.364,✔️,0.0,False,CPU -53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),484.0,+130/-555,34.314,0.0,52.81,52.72,742.99,0.097,✔️,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),334.0,+99/-505,36.454,0.0,54.0,53.99,3.98,0.12,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1885.0,+252/-97,0.741,6.0,2.49,2.855,9830.47,232.266,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1874.0,+459/-210,0.766,6.25,2.14,1.301,2355.62,7.55,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1841.0,+272/-72,0.652,7.12,3.73,2.109,716.17,4.181,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1841.0,+167/-35,0.674,7.12,5.04,3.48,9830.47,22.221,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1708.0,+330/-190,0.561,11.5,4.07,2.764,2355.62,0.879,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1704.0,+155/-54,0.535,11.62,6.8,4.888,51.27,43.085,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1683.0,+130/-49,0.512,12.5,10.9,5.134,4918.07,35.607,✔️,0.0,False,GPU +7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1668.0,+138/-63,0.441,13.12,10.79,5.768,4919.72,5.372,✔️,0.0,False,CPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1647.0,+307/-190,0.512,14.0,8.1,4.306,8.24,0.532,✔️,0.0,False,GPU +9,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1643.0,+541/-234,0.519,14.19,3.75,3.301,6437.34,56.116,✔️,12.5,True,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1573.0,+171/-110,0.379,17.38,12.72,6.226,4918.07,0.949,✔️,0.0,False,GPU +11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1568.0,+112/-80,0.271,17.62,15.23,8.35,1206.7,4.573,✔️,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1547.0,+123/-61,0.248,18.62,16.36,7.752,3495.99,1.462,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[Mitra (default) [12.50% IMPUTED]](https://arxiv.org/abs/2510.21204),1546.0,+241/-163,0.342,18.69,8.77,6.847,94.13,2.176,✔️,12.5,True,GPU +14,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1534.0,+213/-134,0.361,19.25,5.5,9.546,9006.35,8.016,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1512.0,+186/-90,0.234,20.38,15.55,7.803,3495.99,0.148,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1512.0,+156/-108,0.278,20.38,17.48,6.233,4322.16,1.598,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1512.0,+404/-254,0.453,20.38,5.9,18.26,5.86,0.71,➖,0.0,False,GPU +18,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1512.0,+144/-90,0.289,20.38,14.92,7.857,1672.82,2.447,✔️,0.0,False,GPU +19,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1475.0,+152/-72,0.229,22.38,17.24,8.152,1672.82,0.097,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1474.0,+379/-234,0.351,22.44,11.92,5.751,6437.34,0.557,✔️,12.5,True,GPU +21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1468.0,+88/-55,0.134,22.75,20.76,8.84,1277.27,0.648,✔️,0.0,False,CPU +22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1459.0,+89/-51,0.131,23.25,21.79,8.909,1277.27,3.746,✔️,0.0,False,CPU +23,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1445.0,+126/-103,0.149,24.0,20.45,9.538,1206.7,0.689,✔️,0.0,False,CPU +24,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1421.0,+196/-173,0.212,25.25,14.46,9.22,25.28,0.843,➖,0.0,False,GPU +25,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1417.0,+140/-135,0.154,25.5,21.89,11.969,25.13,0.294,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1417.0,+163/-120,0.182,25.5,21.82,6.961,4322.16,0.198,✔️,0.0,False,GPU +27,🧠⚡,Foundation Model,[TabPFNv2 (default) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1401.0,+327/-277,0.305,26.31,14.55,7.63,8.68,0.726,✔️,12.5,True,GPU +28,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1386.0,+147/-125,0.107,27.12,22.28,10.625,10.03,0.115,✔️,0.0,False,CPU +29,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1359.0,+148/-189,0.136,28.5,22.81,13.632,644.11,0.953,✔️,0.0,False,CPU +30,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1340.0,+166/-203,0.167,29.5,22.11,14.036,644.11,0.187,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1338.0,+150/-100,0.111,29.62,25.53,10.515,22.41,7.82,✔️,0.0,False,GPU +32,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1328.0,+192/-190,0.122,30.12,16.91,12.542,9006.35,0.527,✔️,0.0,False,GPU +33,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1326.0,+164/-140,0.127,30.25,27.05,8.903,25.53,0.154,✔️,0.0,False,GPU +34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1310.0,+130/-156,0.071,31.12,27.69,10.331,5194.73,1.75,✔️,0.0,False,CPU +35,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1257.0,+188/-194,0.065,33.88,29.6,10.652,5194.73,0.121,✔️,0.0,False,CPU +36,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1242.0,+214/-201,0.097,34.62,22.5,13.421,3991.28,0.402,✔️,0.0,False,CPU +37,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1221.0,+212/-279,0.119,35.62,11.55,15.544,7.0,1.362,✔️,0.0,False,GPU +38,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1221.0,+230/-228,0.121,35.62,15.59,13.945,3991.28,0.034,✔️,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1210.0,+96/-175,0.021,36.12,33.67,15.363,657.99,1.135,✔️,0.0,False,CPU +40,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1194.0,+92/-128,0.015,36.88,35.69,12.723,4.54,0.374,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1174.0,+76/-129,0.007,37.75,36.62,14.418,2150.84,7.995,✔️,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1157.0,+92/-172,0.014,38.5,36.7,15.588,0.46,0.117,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1146.0,+114/-200,0.02,39.0,36.5,16.48,657.99,0.147,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1122.0,+109/-166,0.0,40.0,38.67,14.856,2150.84,0.566,✔️,0.0,False,CPU +45,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1112.0,+84/-128,0.0,40.38,39.65,14.012,4.08,0.255,✔️,0.0,False,CPU +46,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1058.0,+113/-175,0.0,42.5,41.24,15.454,12.71,0.039,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1030.0,+146/-221,0.0,43.5,41.94,14.656,22.25,0.131,✔️,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+96/-211,0.0,44.5,43.85,17.907,0.69,0.125,✔️,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),988.0,+105/-219,0.0,44.88,43.95,19.332,6.77,0.511,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),904.0,+144/-330,0.0,47.25,45.99,21.426,88.79,0.572,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),732.0,+150/-322,0.0,50.75,50.48,23.404,88.79,0.052,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),655.0,+221/-605,0.0,51.88,51.13,29.74,0.43,0.047,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),520.0,+99/-467,0.0,53.44,53.35,34.23,742.99,0.364,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),481.0,+132/-551,0.0,53.81,53.72,34.314,742.99,0.097,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),332.0,+100/-508,0.0,55.0,54.99,36.454,3.98,0.12,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/winrate_matrix.png.zip index 8a4d17083ed3f6a051d8403730395a4a8560224f..6d0f39f3db292181544df24d4e20809e00e52983 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:df72aa9435a6b7a363028bbccda17403a16e33722f0dec460b93afad13a684e6 -size 1493505 +oid sha256:df56f54c60bfeb90c949a336b40fb407b0cc864b9ea7db0fa42a55d5776c8530 +size 1607127 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 0b9b58587d1aeb0f4c0d6d4ce36dfe7646eb658a..634c0bcc3248364a61d8d86a9b5df2d079168a01 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:580ad436f857ed94dc52d20da597435a03e58278947a397ab620c1017a0fa4f8 -size 451328 +oid sha256:eeca3afc06c673279859a8649c12c9bf86717c15c4bdda3a25475c5cee3fa700 +size 464463 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip index 60217de8a022dd6ec77109eefd621ebe0bc8cd8b..7fb11b727c520b09451c4646fc9d3bfcc33ce22e 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ea8d34ceef272c73d66d5dc117c66624fc6a3bb9fba13bebbb1e2ff8d15bbd77 -size 1042022 +oid sha256:23a240643776a6bc1b3adc69eea19318badac33a2ed6176d76afc8c693259981 +size 1064576 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip index 79b0e9881b6ebd09bbc9ad472868c2d907baa04c..4ed279ce57c08e9ce59fa496b20e480551454ac7 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0b6dc2dcebc9972bf40387ee971a1b6b95e9d115f0819c86647081d67993eec3 -size 120889 +oid sha256:b49c53bd0a4dbb5669785b97260aa25c6b243613ba173676e5b9201cbc4edab5 +size 129465 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/website_leaderboard.csv index 601be121811987b8fb57ec8c68320a67aca39762..d28758bde1bc29286ff5ff9b8dda819634c69945 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/website_leaderboard.csv @@ -1,56 +1,57 @@ -#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1906.0,+304/-113,3.095,0.726,6.43,2.55,10054.02,249.652,✔️,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1900.0,+513/-229,1.229,0.772,6.57,2.05,2279.61,6.732,✔️,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1884.0,+400/-86,1.908,0.683,7.0,3.48,669.82,4.041,✔️,0.0,False,GPU -3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1873.0,+180/-50,3.69,0.671,7.29,4.98,10054.02,28.583,✔️,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1780.0,+499/-189,2.584,0.597,10.14,3.32,6601.21,73.561,✔️,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1772.0,+224/-44,4.944,0.565,10.43,6.19,47.81,37.998,✔️,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1756.0,+146/-30,5.167,0.551,11.0,9.95,4840.44,33.436,✔️,0.0,False,GPU -7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1729.0,+403/-218,2.847,0.545,12.0,3.79,2279.61,0.781,✔️,0.0,False,GPU -8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1714.0,+373/-213,4.191,0.554,12.57,7.37,7.82,0.519,✔️,0.0,False,GPU -9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1693.0,+192/-49,6.056,0.444,13.43,10.94,5390.56,5.054,✔️,0.0,False,CPU -10,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1658.0,+220/-118,6.637,0.398,14.86,7.71,105.28,2.433,✔️,0.0,False,GPU -11,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1624.0,+410/-299,19.429,0.518,16.29,5.15,5.73,0.638,➖,0.0,False,GPU -12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1605.0,+159/-48,7.956,0.287,17.14,15.34,3301.07,1.317,✔️,0.0,False,CPU -13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1605.0,+257/-127,6.481,0.39,17.14,12.18,4840.44,0.706,✔️,0.0,False,GPU -14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1574.0,+114/-57,9.035,0.246,18.57,17.2,997.12,3.787,✔️,0.0,False,CPU -15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1568.0,+226/-106,8.008,0.266,18.86,14.48,3301.07,0.125,✔️,0.0,False,CPU -16,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1558.0,+418/-227,5.384,0.406,19.29,10.76,6601.21,0.588,✔️,0.0,False,GPU -17,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1544.0,+165/-105,8.299,0.295,20.0,14.25,1709.03,2.724,✔️,0.0,False,GPU -18,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1529.0,+188/-118,6.513,0.267,20.71,17.7,4228.53,1.447,✔️,0.0,False,GPU -19,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1515.0,+180/-138,10.909,0.267,21.43,15.3,9360.97,7.658,✔️,0.0,False,GPU -20,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1512.0,+142/-77,8.482,0.251,21.57,16.24,1709.03,0.097,✔️,0.0,False,GPU -21,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1469.0,+405/-336,7.531,0.352,23.71,13.17,8.97,0.837,✔️,0.0,False,GPU -22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1466.0,+83/-44,9.543,0.098,23.86,23.13,911.22,0.527,✔️,0.0,False,CPU -23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1463.0,+78/-38,9.588,0.1,24.0,23.39,911.22,3.532,✔️,0.0,False,CPU -24,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1434.0,+126/-68,10.356,0.105,25.43,23.65,997.12,0.636,✔️,0.0,False,CPU -25,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1425.0,+196/-137,7.312,0.16,25.86,21.99,4228.53,0.173,✔️,0.0,False,GPU -26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1422.0,+220/-139,11.174,0.117,26.0,21.17,9.39,0.105,✔️,0.0,False,CPU -27,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1407.0,+139/-152,13.106,0.121,26.71,24.25,27.26,0.29,✔️,0.0,False,GPU -28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1392.0,+209/-236,14.555,0.158,27.43,21.57,613.95,0.952,✔️,0.0,False,CPU -29,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1384.0,+236/-250,14.881,0.193,27.86,20.58,613.95,0.135,✔️,0.0,False,CPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1366.0,+155/-135,11.015,0.129,28.71,23.97,22.96,8.006,✔️,0.0,False,GPU -31,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1325.0,+195/-187,9.406,0.114,30.71,27.44,17.97,0.145,✔️,0.0,False,GPU -32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1290.0,+275/-208,14.054,0.111,32.43,20.76,3778.28,0.323,✔️,0.0,False,CPU -33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1282.0,+124/-160,11.205,0.026,32.86,31.52,5937.36,1.406,✔️,0.0,False,CPU -34,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1276.0,+260/-286,16.185,0.138,33.14,10.38,7.12,1.138,✔️,0.0,False,GPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1272.0,+275/-228,14.636,0.14,33.29,14.11,3778.28,0.028,✔️,0.0,False,CPU -36,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1272.0,+146/-174,14.048,0.037,33.29,30.98,9360.97,0.522,✔️,0.0,False,GPU -37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1229.0,+155/-260,16.471,0.03,35.29,32.39,628.51,0.812,✔️,0.0,False,CPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1222.0,+185/-280,11.535,0.031,35.57,32.35,5937.36,0.119,✔️,0.0,False,CPU -39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1175.0,+138/-220,16.642,0.02,37.57,35.46,0.43,0.117,✔️,0.0,False,CPU -40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1173.0,+176/-289,17.646,0.029,37.64,34.8,628.51,0.149,✔️,0.0,False,CPU -41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1172.0,+59/-100,13.734,0.0,37.71,37.36,4.41,0.363,✔️,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1150.0,+55/-106,15.611,0.0,38.57,38.08,2163.39,7.622,✔️,0.0,False,CPU -43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1114.0,+109/-169,14.895,0.0,40.0,39.21,3.97,0.24,✔️,0.0,False,CPU -44,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1095.0,+89/-158,15.956,0.0,40.71,39.47,10.54,0.036,✔️,0.0,False,CPU -45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1095.0,+106/-175,16.062,0.0,40.71,39.78,2163.39,0.713,✔️,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1002.0,+135/-301,15.699,0.0,43.86,42.36,20.5,0.13,✔️,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+131/-217,19.276,0.0,43.93,43.18,0.67,0.118,✔️,0.0,False,CPU -48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),943.0,+77/-213,21.065,0.0,45.57,45.12,6.88,0.452,✔️,0.0,False,CPU -49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),937.0,+178/-381,22.177,0.0,45.71,44.31,86.71,0.53,✔️,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),757.0,+168/-358,24.152,0.0,49.57,49.27,86.71,0.05,✔️,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),712.0,+245/-524,30.287,0.0,50.29,49.48,0.4,0.044,✔️,0.0,False,CPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),533.0,+85/-893,36.2,0.0,52.5,52.4,753.48,0.227,✔️,0.0,False,CPU -53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),504.0,+117/-984,36.249,0.0,52.79,52.68,753.48,0.091,✔️,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),352.0,+97/-674,37.99,0.0,54.0,53.99,4.35,0.119,✔️,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1905.0,+308/-104,0.729,6.57,2.58,3.095,10054.02,249.652,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1899.0,+520/-230,0.771,6.71,2.05,1.229,2279.61,6.732,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1883.0,+380/-88,0.679,7.14,3.5,1.908,669.82,4.041,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1873.0,+173/-48,0.67,7.43,5.05,3.69,10054.02,28.583,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1782.0,+493/-185,0.593,10.29,3.32,2.584,6601.21,73.561,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1770.0,+209/-45,0.567,10.71,6.24,4.944,47.81,37.998,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1755.0,+135/-30,0.551,11.29,10.2,5.167,4840.44,33.436,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1729.0,+411/-216,0.545,12.29,3.89,2.847,2279.61,0.781,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1715.0,+376/-211,0.555,12.86,7.43,4.191,7.82,0.519,✔️,0.0,False,GPU +9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1694.0,+186/-51,0.443,13.71,11.1,6.056,5390.56,5.054,✔️,0.0,False,CPU +10,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),1653.0,+222/-120,0.391,15.43,7.88,6.637,105.28,2.433,✔️,0.0,False,GPU +11,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),1627.0,+414/-295,0.518,16.57,5.24,19.429,5.73,0.638,➖,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1608.0,+152/-48,0.282,17.43,15.49,7.956,3301.07,1.317,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1605.0,+238/-130,0.388,17.57,12.35,6.481,4840.44,0.706,✔️,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1575.0,+117/-58,0.243,19.0,17.49,9.035,997.12,3.787,✔️,0.0,False,CPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1569.0,+232/-107,0.267,19.29,14.63,8.008,3301.07,0.125,✔️,0.0,False,CPU +16,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1560.0,+421/-226,0.401,19.71,10.82,5.384,6601.21,0.588,✔️,0.0,False,GPU +17,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1546.0,+164/-104,0.292,20.43,14.4,8.299,1709.03,2.724,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1532.0,+189/-120,0.268,21.14,17.91,6.513,4228.53,1.447,✔️,0.0,False,GPU +19,🧠⚡,Foundation Model,[SAP-RPT-OSS (default)](https://arxiv.org/abs/2506.10707),1521.0,+202/-160,0.242,21.71,13.12,8.659,22.8,0.769,➖,0.0,False,GPU +20,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1518.0,+181/-140,0.27,21.86,15.39,10.909,9360.97,7.658,✔️,0.0,False,GPU +21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1516.0,+143/-78,0.248,22.0,16.51,8.482,1709.03,0.097,✔️,0.0,False,GPU +22,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1474.0,+406/-336,0.348,24.14,13.31,7.531,8.97,0.837,✔️,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1468.0,+79/-43,0.096,24.43,23.78,9.543,911.22,0.527,✔️,0.0,False,CPU +24,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1462.0,+77/-40,0.098,24.71,24.12,9.588,911.22,3.532,✔️,0.0,False,CPU +25,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1437.0,+130/-69,0.11,26.0,24.03,10.356,997.12,0.636,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1428.0,+197/-140,0.164,26.43,22.29,7.312,4228.53,0.173,✔️,0.0,False,GPU +27,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1422.0,+221/-142,0.122,26.71,21.49,11.174,9.39,0.105,✔️,0.0,False,CPU +28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1408.0,+144/-154,0.12,27.43,24.82,13.106,27.26,0.29,✔️,0.0,False,GPU +29,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1396.0,+209/-230,0.156,28.0,21.91,14.555,613.95,0.952,✔️,0.0,False,CPU +30,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1388.0,+235/-247,0.191,28.43,20.91,14.881,613.95,0.135,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1368.0,+154/-136,0.127,29.43,24.9,11.015,22.96,8.006,✔️,0.0,False,GPU +32,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1331.0,+195/-190,0.121,31.29,27.74,9.406,17.97,0.145,✔️,0.0,False,GPU +33,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1294.0,+275/-209,0.111,33.14,21.0,14.054,3778.28,0.323,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1285.0,+128/-166,0.03,33.57,32.19,11.205,5937.36,1.406,✔️,0.0,False,CPU +35,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1280.0,+261/-291,0.136,33.86,10.42,16.185,7.12,1.138,✔️,0.0,False,GPU +36,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1280.0,+151/-173,0.039,33.86,31.37,14.048,9360.97,0.522,✔️,0.0,False,GPU +37,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1274.0,+268/-232,0.138,34.14,14.24,14.636,3778.28,0.028,✔️,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1227.0,+153/-256,0.024,36.29,33.49,16.471,628.51,0.812,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1227.0,+179/-286,0.03,36.29,33.1,11.535,5937.36,0.119,✔️,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1174.0,+135/-216,0.016,38.57,36.52,16.642,0.43,0.117,✔️,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1172.0,+173/-290,0.023,38.64,35.88,17.646,628.51,0.149,✔️,0.0,False,CPU +42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1170.0,+58/-102,0.0,38.71,38.37,13.734,4.41,0.363,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1153.0,+60/-112,0.0,39.43,38.89,15.611,2163.39,7.622,✔️,0.0,False,CPU +44,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1113.0,+107/-172,0.0,41.0,40.23,14.895,3.97,0.24,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1098.0,+112/-178,0.0,41.57,40.6,16.062,2163.39,0.713,✔️,0.0,False,CPU +46,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1098.0,+94/-161,0.0,41.57,40.32,15.956,10.54,0.036,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1002.0,+133/-301,0.0,44.86,43.39,15.699,20.5,0.13,✔️,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+129/-212,0.0,44.93,44.2,19.276,0.67,0.118,✔️,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),943.0,+77/-214,0.0,46.57,46.13,21.065,6.88,0.452,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),937.0,+178/-378,0.0,46.71,45.35,22.177,86.71,0.53,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),757.0,+167/-356,0.0,50.57,50.27,24.152,86.71,0.05,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),712.0,+244/-528,0.0,51.29,50.5,30.287,0.4,0.044,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),533.0,+85/-888,0.0,53.5,53.4,36.2,753.48,0.227,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),504.0,+117/-993,0.0,53.79,53.68,36.249,753.48,0.091,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),352.0,+97/-818,0.0,55.0,54.99,37.99,4.35,0.119,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip index 9ccfcb7bee0cca2aabd77f85e287b03fcb5e8e95..28519a02e389c884d862f756c116bb8df238e796 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5e24ab0d5b1957d46f486d7858888173c4738f1556e1d664359b611e980849cb -size 1396925 +oid sha256:157b0f5672b99db7e98d9ce3e9cb9c39749e3420244f15d11ccdb93a3e8cf1d7 +size 1515219 diff --git a/website_texts.py b/website_texts.py index 9e64000a6fbdc91d8f27df4eb3da81321cd60998..e7ae2050355c21f4e2335eff1b08ffc150c8357c 100644 --- a/website_texts.py +++ b/website_texts.py @@ -145,7 +145,9 @@ VERSION_HISTORY_BUTTON_TEXT = """ The following details updates to the leaderboard (date format is YYYY/MM/DD): -* 2025/12/18-v0.1.2.1: +* 2025/12/11-v0.1.2.2: + * Add new unverified model: SAP-RPT-OSS (a.k.a.: ConTexTab, sap-rpt-1-oss) +* 2025/11/27-v0.1.2.1: * Make tuning trajectories start from the default configuration. * UI improvements and more user-friendly explanations. * 2025/11/22-v0.1.2: Add newest version of TabArena LB for NeurIPS 2025