diff --git a/data/imputation_no/splits_all/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index efe6f4ec63c3610400ef5457242e2156c4d02a0f..a10dc781e82927f8b05f2b1b3dd7103e393b84b4 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3ab74bcd98d52b0516d8ece16fba2bf62d17355c3709f92d1176997deec7bdf9 +oid sha256:1dd9ea2c826175640f8d2397b1335e732569371bbcb838c21696add572fef528 size 434350 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_all/pareto_n_configs_imp.png.zip index 0adb6fac03aa6f45e116483d94fa0784558cbbe6..8bb9e45cab3893e0cc86f612eadaa9936410f4e1 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:dc338f509a3342554992e0ce9ba450fe38ef5b4c0434287cf6aff23593584051 -size 987831 +oid sha256:bad65cc0285c1450996cf99e3a2cb903a9c18154770b93e76067fe7afc3c6ef9 +size 968055 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_all/tuning-impact-elo.png.zip index 4a7e2cb7761873c8444607e3545b2c8e2502d19f..8e98e8e2ef2c23e866239c0302b8fa7e4d2abfb0 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b282bc5c113ba1ae79f044a2ce90c306ab0900f5301b949d50b532750fb4930b +oid sha256:c2789a139f2ed0e703246604cd3517c8cad11d2f18e5e058a6195ec2b4603eb3 size 108478 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_all/datasets_all/website_leaderboard.csv index e7e4af8c567f04bb697103ff99d7625c5684a82c..cf673d8023d1873fa5cd219707769e44889d4319 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_all/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_all/datasets_all/website_leaderboard.csv @@ -1,51 +1,51 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1679.0,+64/-51,0.789,6.94,3.2,5.507,582.21,6.116,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1626.0,+89/-60,0.739,8.58,2.9,4.753,2040.22,8.92,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1580.0,+73/-54,0.68,10.27,3.97,5.726,2040.22,1.218,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1546.0,+72/-52,0.654,11.6,4.97,6.429,5.81,0.645,0.0,False,GPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1539.0,+55/-51,0.549,11.93,5.37,8.693,1735.72,2.559,0.0,False,CPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1536.0,+57/-48,0.556,12.03,6.93,8.739,2950.72,11.988,0.0,False,GPU -6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1479.0,+65/-53,0.523,14.61,4.53,8.827,4907.64,286.652,0.0,False,GPU -7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1466.0,+56/-43,0.428,15.26,8.29,10.223,3285.87,1.472,0.0,False,GPU -8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1450.0,+36/-31,0.378,16.06,11.15,11.452,416.98,2.639,0.0,False,CPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1445.0,+46/-48,0.379,16.28,8.65,10.227,2950.72,0.663,0.0,False,GPU -10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1432.0,+44/-43,0.36,16.98,10.14,10.958,1658.41,0.653,0.0,False,CPU -11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1420.0,+60/-58,0.419,17.58,6.04,10.229,4907.64,39.96,0.0,False,GPU -12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1419.0,+42/-44,0.345,17.6,9.69,11.142,1658.41,0.081,0.0,False,CPU -13,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1409.0,+82/-54,0.385,18.12,7.61,11.442,4621.67,8.149,0.0,False,GPU -14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1404.0,+56/-47,0.341,18.38,9.66,11.14,3285.87,0.173,0.0,False,GPU -15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1400.0,+31/-30,0.28,18.63,13.38,12.143,416.98,0.334,0.0,False,CPU -16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1389.0,+35/-36,0.296,19.21,12.62,12.22,693.49,1.69,0.0,False,CPU -17,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1378.0,+41/-42,0.315,19.76,11.31,12.023,6.83,0.08,0.0,False,CPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1371.0,+45/-40,0.243,20.17,10.17,11.984,4621.67,0.475,0.0,False,GPU -19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1366.0,+49/-41,0.267,20.45,11.91,12.584,846.88,2.552,0.0,False,GPU -20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1363.0,+31/-33,0.241,20.59,14.97,12.54,693.49,0.308,0.0,False,CPU -21,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1339.0,+70/-60,0.336,21.91,7.2,12.933,47.62,43.739,0.0,False,GPU -22,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1313.0,+46/-38,0.198,23.35,14.43,13.883,846.88,0.13,0.0,False,GPU -23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1305.0,+54/-47,0.236,23.79,14.16,13.655,10.49,0.132,0.0,False,GPU -24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1299.0,+50/-50,0.164,24.11,16.52,13.439,2874.67,1.952,0.0,False,CPU -25,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1282.0,+43/-40,0.171,25.07,16.13,15.191,2929.85,0.422,0.0,False,CPU -26,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1257.0,+44/-44,0.123,26.41,15.77,15.566,14.87,0.308,0.0,False,GPU -27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1243.0,+47/-43,0.121,27.22,17.59,15.868,2929.85,0.051,0.0,False,CPU -28,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1240.0,+36/-41,0.086,27.36,19.31,14.957,10.06,1.689,0.0,False,GPU -29,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1219.0,+39/-38,0.091,28.49,20.6,15.422,1.94,0.123,0.0,False,CPU -30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1219.0,+52/-61,0.134,28.51,17.29,16.343,593.24,4.467,0.0,False,CPU -31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1218.0,+41/-47,0.078,28.58,20.1,15.309,2874.67,0.131,0.0,False,CPU -32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1212.0,+46/-42,0.101,28.89,19.17,16.88,183.02,0.761,0.0,False,CPU -33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1208.0,+46/-48,0.115,29.06,19.52,17.03,7.32,0.05,0.0,False,CPU -34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1190.0,+35/-29,0.048,30.03,26.04,16.049,1.96,0.142,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1179.0,+55/-48,0.076,30.61,20.12,17.756,373.18,0.771,0.0,False,CPU -36,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1172.0,+47/-52,0.076,30.96,19.36,17.853,183.02,0.091,0.0,False,CPU -37,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1153.0,+52/-55,0.057,31.96,21.29,17.842,593.24,0.306,0.0,False,CPU -38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1141.0,+49/-46,0.052,32.55,19.95,18.528,373.18,0.085,0.0,False,CPU -39,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1073.0,+35/-48,0.012,35.84,28.74,19.694,9.99,0.126,0.0,False,CPU -40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1043.0,+62/-61,0.036,37.16,29.18,23.269,3.23,0.919,0.0,False,GPU -41,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1005.0,+59/-60,0.013,38.74,28.39,22.719,2.86,0.373,0.0,False,CPU -42,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+45/-50,0.006,38.93,34.17,23.485,0.43,0.054,0.0,False,CPU -43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,998.0,+63/-88,0.016,39.0,27.81,25.352,129.01,1.802,0.0,False,CPU -44,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,980.0,+59/-71,0.011,39.69,34.68,24.998,0.25,0.05,0.0,False,CPU -45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,970.0,+64/-97,0.032,40.07,30.33,31.269,237.58,0.424,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,940.0,+71/-111,0.023,41.15,31.63,31.88,237.58,0.085,0.0,False,CPU -47,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,888.0,+68/-108,0.003,42.8,32.9,30.599,129.01,0.18,0.0,False,CPU -48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,853.0,+76/-133,0.014,43.79,34.99,34.618,1.19,0.12,0.0,False,CPU -49,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,636.0,+89/-113,0.0,47.95,43.9,44.269,0.19,0.036,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1679.0,+64/-51,5.507,0.789,6.94,3.2,582.21,6.116,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1626.0,+89/-60,4.753,0.739,8.58,2.9,2040.22,8.92,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1580.0,+73/-54,5.726,0.68,10.27,3.97,2040.22,1.218,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1546.0,+72/-52,6.429,0.654,11.6,4.97,5.81,0.645,✔️,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1539.0,+55/-51,8.693,0.549,11.93,5.37,1735.72,2.559,✔️,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1536.0,+57/-48,8.739,0.556,12.03,6.93,2950.72,11.988,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1479.0,+65/-53,8.827,0.523,14.61,4.53,4907.64,286.652,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1466.0,+56/-43,10.223,0.428,15.26,8.29,3285.87,1.472,✔️,0.0,False,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1450.0,+36/-31,11.452,0.378,16.06,11.15,416.98,2.639,✔️,0.0,False,CPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1445.0,+46/-48,10.227,0.379,16.28,8.65,2950.72,0.663,✔️,0.0,False,GPU +10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1432.0,+44/-43,10.958,0.36,16.98,10.14,1658.41,0.653,✔️,0.0,False,CPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1420.0,+60/-58,10.229,0.419,17.58,6.04,4907.64,39.96,✔️,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1419.0,+42/-44,11.142,0.345,17.6,9.69,1658.41,0.081,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1409.0,+82/-54,11.442,0.385,18.12,7.61,4621.67,8.149,✔️,0.0,False,GPU +14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1404.0,+56/-47,11.14,0.341,18.38,9.66,3285.87,0.173,✔️,0.0,False,GPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1400.0,+31/-30,12.143,0.28,18.63,13.38,416.98,0.334,✔️,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1389.0,+35/-36,12.22,0.296,19.21,12.62,693.49,1.69,✔️,0.0,False,CPU +17,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1378.0,+41/-42,12.023,0.315,19.76,11.31,6.83,0.08,✔️,0.0,False,CPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1371.0,+45/-40,11.984,0.243,20.17,10.17,4621.67,0.475,✔️,0.0,False,GPU +19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1366.0,+49/-41,12.584,0.267,20.45,11.91,846.88,2.552,✔️,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1363.0,+31/-33,12.54,0.241,20.59,14.97,693.49,0.308,✔️,0.0,False,CPU +21,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1339.0,+70/-60,12.933,0.336,21.91,7.2,47.62,43.739,✔️,0.0,False,GPU +22,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1313.0,+46/-38,13.883,0.198,23.35,14.43,846.88,0.13,✔️,0.0,False,GPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1305.0,+54/-47,13.655,0.236,23.79,14.16,10.49,0.132,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1299.0,+50/-50,13.439,0.164,24.11,16.52,2874.67,1.952,✔️,0.0,False,CPU +25,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1282.0,+43/-40,15.191,0.171,25.07,16.13,2929.85,0.422,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1257.0,+44/-44,15.566,0.123,26.41,15.77,14.87,0.308,✔️,0.0,False,GPU +27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1243.0,+47/-43,15.868,0.121,27.22,17.59,2929.85,0.051,✔️,0.0,False,CPU +28,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1240.0,+36/-41,14.957,0.086,27.36,19.31,10.06,1.689,✔️,0.0,False,GPU +29,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1219.0,+39/-38,15.422,0.091,28.49,20.6,1.94,0.123,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1219.0,+52/-61,16.343,0.134,28.51,17.29,593.24,4.467,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1218.0,+41/-47,15.309,0.078,28.58,20.1,2874.67,0.131,✔️,0.0,False,CPU +32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1212.0,+46/-42,16.88,0.101,28.89,19.17,183.02,0.761,✔️,0.0,False,CPU +33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1208.0,+46/-48,17.03,0.115,29.06,19.52,7.32,0.05,✔️,0.0,False,CPU +34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1190.0,+35/-29,16.049,0.048,30.03,26.04,1.96,0.142,✔️,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1179.0,+55/-48,17.756,0.076,30.61,20.12,373.18,0.771,✔️,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1172.0,+47/-52,17.853,0.076,30.96,19.36,183.02,0.091,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1153.0,+52/-55,17.842,0.057,31.96,21.29,593.24,0.306,✔️,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1141.0,+49/-46,18.528,0.052,32.55,19.95,373.18,0.085,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1073.0,+35/-48,19.694,0.012,35.84,28.74,9.99,0.126,✔️,0.0,False,CPU +40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1043.0,+62/-61,23.269,0.036,37.16,29.18,3.23,0.919,✔️,0.0,False,GPU +41,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1005.0,+59/-60,22.719,0.013,38.74,28.39,2.86,0.373,✔️,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+45/-50,23.485,0.006,38.93,34.17,0.43,0.054,✔️,0.0,False,CPU +43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),998.0,+63/-88,25.352,0.016,39.0,27.81,129.01,1.802,✔️,0.0,False,CPU +44,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),980.0,+59/-71,24.998,0.011,39.69,34.68,0.25,0.05,✔️,0.0,False,CPU +45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),970.0,+64/-97,31.269,0.032,40.07,30.33,237.58,0.424,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),940.0,+71/-111,31.88,0.023,41.15,31.63,237.58,0.085,✔️,0.0,False,CPU +47,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),888.0,+68/-108,30.599,0.003,42.8,32.9,129.01,0.18,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),853.0,+76/-133,34.618,0.014,43.79,34.99,1.19,0.12,✔️,0.0,False,CPU +49,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),636.0,+89/-113,44.269,0.0,47.95,43.9,0.19,0.036,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_all/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_all/winrate_matrix.png.zip index b0fa597a376b42f522482b1694e0eba25d95eee6..f3f7aef273c47476fdd25857778e2bbc6184bbed 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:348dfb8ae7ee428690bd12aed9771b32bcb4dfee97c8fc80db5696433d7b0ff4 +oid sha256:d7f0256cccbc9d360e60c248ee0d2afee7b3fbb0eb13ce8ded31b2f523126680 size 1739374 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index 08de6f023ce8dc5c97e8a598ca1e99507b0dabef..e364386bc5c70bc6e6d9d6f9df5a510b93bec420 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fd03f6dff15fd9fcda1d0dbc2cc3f5bcc08cb41a0600762210985e0171c15373 +oid sha256:2e82722ede5d45ca41b062fa94c487123bbf01997c66b06cd2cf06e4281142c5 size 399354 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip index 807bebde5664d9cabee72b2663a8bf2808878dad..8889dbfb5ad66e2e185502ae38fc02ab6b0fffc8 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b97ad68cbfd79a48e72703bb4d4b7d080493f752b2c1774faf646dac00a7c0d9 -size 981194 +oid sha256:b819ee8c30207b57868a68efcc7ec9b1d27ef0ff5aeeda722d05d4d59568bac1 +size 989200 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_medium/tuning-impact-elo.png.zip index 90197764f449b15e26a8aed93ca8955ded78bad2..9b9aea74c66d806917036138659a06b106202c62 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:701c5872555086215e2e04852b6a4f27bc9b7004114bf8d52735f904a99e98a6 +oid sha256:84be5741ee4ed6304624f803d704de577234f3a7a9827a3673a42bee30f83d2f size 112567 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_all/datasets_medium/website_leaderboard.csv index b3b083c9ccee1c5ac6f18fe8d056525ec6194ca8..3f0632fed8cdb49aa4245b9f5033d1684c70e668 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_medium/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_all/datasets_medium/website_leaderboard.csv @@ -1,51 +1,51 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1825.0,+86/-46,0.836,5.78,2.85,2.566,289.53,3.187,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1804.0,+82/-49,0.808,6.29,3.08,2.669,432.35,4.085,0.0,False,CPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1676.0,+164/-124,0.66,10.21,3.09,5.382,735.58,11.736,0.0,False,GPU -3,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1649.0,+109/-94,0.581,11.21,6.96,5.684,1719.82,1.675,0.0,False,GPU -4,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1635.0,+125/-116,0.598,11.74,6.46,5.75,2526.28,6.013,0.0,False,GPU -5,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1621.0,+86/-63,0.539,12.33,6.64,5.585,777.59,0.25,0.0,False,CPU -6,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1620.0,+145/-112,0.591,12.34,5.04,6.097,735.58,1.386,0.0,False,GPU -7,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1610.0,+94/-69,0.541,12.74,5.67,5.783,777.59,0.052,0.0,False,CPU -8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1602.0,+76/-56,0.493,13.08,8.87,7.351,131.56,2.639,0.0,False,CPU -9,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1567.0,+105/-96,0.524,14.58,7.14,5.97,3.24,0.031,0.0,False,CPU -10,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1560.0,+87/-61,0.447,14.9,8.9,7.478,282.13,0.563,0.0,False,CPU -11,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1549.0,+116/-94,0.405,15.39,8.76,7.016,1719.82,0.081,0.0,False,GPU -12,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1546.0,+155/-99,0.478,15.51,7.63,6.906,1.88,0.645,0.0,False,GPU -13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1542.0,+155/-95,0.426,15.7,7.33,7.41,1993.14,0.619,0.0,False,GPU -14,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1534.0,+141/-125,0.48,16.05,4.17,7.024,4786.55,444.544,0.0,False,GPU -15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1512.0,+80/-65,0.342,17.07,11.95,8.574,131.56,0.132,0.0,False,CPU -16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1510.0,+69/-51,0.356,17.18,13.18,8.062,282.13,0.066,0.0,False,CPU -17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1490.0,+152/-101,0.36,18.13,8.42,8.168,1993.14,0.061,0.0,False,GPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1481.0,+107/-91,0.32,18.51,12.5,8.151,2526.28,0.349,0.0,False,GPU -19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1438.0,+108/-100,0.271,20.59,12.75,9.676,566.74,0.951,0.0,False,GPU -20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1412.0,+122/-124,0.296,21.86,8.61,9.655,4786.55,42.641,0.0,False,GPU -21,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1387.0,+71/-71,0.179,23.12,16.95,9.424,1770.56,1.069,0.0,False,CPU -22,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1382.0,+97/-96,0.191,23.34,18.62,11.377,566.74,0.06,0.0,False,GPU -23,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1371.0,+104/-120,0.234,23.89,14.51,9.928,0.49,0.053,0.0,False,CPU -24,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1339.0,+130/-140,0.221,25.48,10.01,12.451,46.62,43.739,0.0,False,GPU -25,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1334.0,+119/-133,0.194,25.72,17.81,12.922,1517.65,0.134,0.0,False,CPU -26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1328.0,+132/-111,0.177,26.01,17.36,11.007,5.16,0.074,0.0,False,GPU -27,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1302.0,+77/-82,0.096,27.27,23.36,11.463,8.06,0.25,0.0,False,GPU -28,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1280.0,+84/-70,0.046,28.34,25.52,12.214,6.75,0.235,0.0,False,GPU -29,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1275.0,+78/-77,0.055,28.58,24.36,11.211,1770.56,0.051,0.0,False,CPU -30,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1271.0,+110/-134,0.125,28.79,22.32,13.812,1517.65,0.015,0.0,False,CPU -31,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1258.0,+62/-73,0.051,29.37,27.19,11.863,0.29,0.045,0.0,False,CPU -32,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1249.0,+86/-79,0.039,29.81,24.11,12.418,66.0,0.341,0.0,False,CPU -33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1220.0,+97/-140,0.085,31.14,26.84,15.399,3.12,0.016,0.0,False,CPU -34,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1216.0,+97/-87,0.04,31.35,25.48,13.131,66.0,0.024,0.0,False,CPU -35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1214.0,+108/-138,0.071,31.42,25.0,14.369,351.94,1.658,0.0,False,CPU -36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1206.0,+71/-86,0.021,31.77,29.62,12.542,36.48,0.329,0.0,False,CPU -37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1158.0,+63/-76,0.006,33.91,32.33,13.771,36.48,0.037,0.0,False,CPU -38,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1118.0,+67/-72,0.0,35.56,34.12,15.534,2.18,0.039,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1102.0,+85/-108,0.021,36.21,32.28,16.646,351.94,0.114,0.0,False,CPU -40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1046.0,+145/-191,0.018,38.34,33.71,23.683,2.49,0.145,0.0,False,GPU -41,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1022.0,+138/-176,0.024,39.18,35.21,20.34,1.51,0.111,0.0,False,CPU -42,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+60/-87,0.0,39.92,39.15,18.698,0.2,0.021,0.0,False,CPU -43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,998.0,+87/-126,0.006,39.99,34.2,22.252,150.3,10.071,0.0,False,CPU -44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,950.0,+152/-228,0.005,41.48,38.36,31.109,43.64,0.081,0.0,False,CPU -45,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,940.0,+106/-141,0.002,41.8,40.36,21.813,0.11,0.028,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,897.0,+156/-249,0.003,42.99,40.6,32.052,43.64,0.018,0.0,False,CPU -47,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,813.0,+58/-95,0.0,44.96,44.7,28.248,150.3,0.482,0.0,False,CPU -48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,785.0,+139/-247,0.0,45.52,44.41,34.158,0.19,0.019,0.0,False,CPU -49,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,566.0,+107/-159,0.0,48.53,48.44,42.214,0.1,0.072,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1825.0,+86/-46,2.566,0.836,5.78,2.85,289.53,3.187,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1804.0,+82/-49,2.669,0.808,6.29,3.08,432.35,4.085,✔️,0.0,False,CPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1676.0,+164/-124,5.382,0.66,10.21,3.09,735.58,11.736,✔️,0.0,False,GPU +3,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1649.0,+109/-94,5.684,0.581,11.21,6.96,1719.82,1.675,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1635.0,+125/-116,5.75,0.598,11.74,6.46,2526.28,6.013,✔️,0.0,False,GPU +5,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1621.0,+86/-63,5.585,0.539,12.33,6.64,777.59,0.25,✔️,0.0,False,CPU +6,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1620.0,+145/-112,6.097,0.591,12.34,5.04,735.58,1.386,✔️,0.0,False,GPU +7,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1610.0,+94/-69,5.783,0.541,12.74,5.67,777.59,0.052,✔️,0.0,False,CPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1602.0,+76/-56,7.351,0.493,13.08,8.87,131.56,2.639,✔️,0.0,False,CPU +9,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1567.0,+105/-96,5.97,0.524,14.58,7.14,3.24,0.031,✔️,0.0,False,CPU +10,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1560.0,+87/-61,7.478,0.447,14.9,8.9,282.13,0.563,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1549.0,+116/-94,7.016,0.405,15.39,8.76,1719.82,0.081,✔️,0.0,False,GPU +12,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1546.0,+155/-99,6.906,0.478,15.51,7.63,1.88,0.645,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1542.0,+155/-95,7.41,0.426,15.7,7.33,1993.14,0.619,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1534.0,+141/-125,7.024,0.48,16.05,4.17,4786.55,444.544,✔️,0.0,False,GPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1512.0,+80/-65,8.574,0.342,17.07,11.95,131.56,0.132,✔️,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1510.0,+69/-51,8.062,0.356,17.18,13.18,282.13,0.066,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1490.0,+152/-101,8.168,0.36,18.13,8.42,1993.14,0.061,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1481.0,+107/-91,8.151,0.32,18.51,12.5,2526.28,0.349,✔️,0.0,False,GPU +19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1438.0,+108/-100,9.676,0.271,20.59,12.75,566.74,0.951,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1412.0,+122/-124,9.655,0.296,21.86,8.61,4786.55,42.641,✔️,0.0,False,GPU +21,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1387.0,+71/-71,9.424,0.179,23.12,16.95,1770.56,1.069,✔️,0.0,False,CPU +22,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1382.0,+97/-96,11.377,0.191,23.34,18.62,566.74,0.06,✔️,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1371.0,+104/-120,9.928,0.234,23.89,14.51,0.49,0.053,✔️,0.0,False,CPU +24,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1339.0,+130/-140,12.451,0.221,25.48,10.01,46.62,43.739,✔️,0.0,False,GPU +25,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1334.0,+119/-133,12.922,0.194,25.72,17.81,1517.65,0.134,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1328.0,+132/-111,11.007,0.177,26.01,17.36,5.16,0.074,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1302.0,+77/-82,11.463,0.096,27.27,23.36,8.06,0.25,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1280.0,+84/-70,12.214,0.046,28.34,25.52,6.75,0.235,✔️,0.0,False,GPU +29,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1275.0,+78/-77,11.211,0.055,28.58,24.36,1770.56,0.051,✔️,0.0,False,CPU +30,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1271.0,+110/-134,13.812,0.125,28.79,22.32,1517.65,0.015,✔️,0.0,False,CPU +31,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1258.0,+62/-73,11.863,0.051,29.37,27.19,0.29,0.045,✔️,0.0,False,CPU +32,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1249.0,+86/-79,12.418,0.039,29.81,24.11,66.0,0.341,✔️,0.0,False,CPU +33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1220.0,+97/-140,15.399,0.085,31.14,26.84,3.12,0.016,✔️,0.0,False,CPU +34,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1216.0,+97/-87,13.131,0.04,31.35,25.48,66.0,0.024,✔️,0.0,False,CPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1214.0,+108/-138,14.369,0.071,31.42,25.0,351.94,1.658,✔️,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1206.0,+71/-86,12.542,0.021,31.77,29.62,36.48,0.329,✔️,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1158.0,+63/-76,13.771,0.006,33.91,32.33,36.48,0.037,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1118.0,+67/-72,15.534,0.0,35.56,34.12,2.18,0.039,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1102.0,+85/-108,16.646,0.021,36.21,32.28,351.94,0.114,✔️,0.0,False,CPU +40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1046.0,+145/-191,23.683,0.018,38.34,33.71,2.49,0.145,✔️,0.0,False,GPU +41,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1022.0,+138/-176,20.34,0.024,39.18,35.21,1.51,0.111,✔️,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+60/-87,18.698,0.0,39.92,39.15,0.2,0.021,✔️,0.0,False,CPU +43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),998.0,+87/-126,22.252,0.006,39.99,34.2,150.3,10.071,✔️,0.0,False,CPU +44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),950.0,+152/-228,31.109,0.005,41.48,38.36,43.64,0.081,✔️,0.0,False,CPU +45,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),940.0,+106/-141,21.813,0.002,41.8,40.36,0.11,0.028,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),897.0,+156/-249,32.052,0.003,42.99,40.6,43.64,0.018,✔️,0.0,False,CPU +47,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),813.0,+58/-95,28.248,0.0,44.96,44.7,150.3,0.482,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),785.0,+139/-247,34.158,0.0,45.52,44.41,0.19,0.019,✔️,0.0,False,CPU +49,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),566.0,+107/-159,42.214,0.0,48.53,48.44,0.1,0.072,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_all/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_medium/winrate_matrix.png.zip index a2e79b1bdf2c2bcf9b96883dd445ff5925494c65..a4d5fbcdcc11f6f61bede594c4a663c6a8f3ff31 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4b456446d2244486892edc8aadc235111c5ebd909560b2395fecbcc0dfc26278 +oid sha256:36be9d3ce0af98477238f9519e6a9a06ec46e0c5afe2bc615cc08bf456fb64a5 size 1634342 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 4ea02c88840a3a5ab3998b7dd826bda451461bb5..fa36ee16e6517350c9c52d474c9a39de7c45f00f 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fc2daec9ff7c8fec95e11c76a7892200ba5b61fea726f612cf992675ade9ff04 +oid sha256:a47eadc792672c3f41eff8c373824e252e6928b11cfe2a8e6324a4c6f55ef05b size 397284 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_small/pareto_n_configs_imp.png.zip index cf47d9342444ac075c435373b6d92a6f03bb4717..2aeed6bab7d379e9842cace94374418231320724 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:de7549430fdf91e2226031fe4d67a85f0153bc784a52430fd298040c8d6f8136 -size 981954 +oid sha256:b9e72f35f0f94d4d7734e2cce25b19ad20f0e57b566c38b06c41176826f7e12c +size 955425 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_small/tuning-impact-elo.png.zip index 75d46ef2823987833d8af4dc44664957531dd3a3..c8cae8036327bd14cfc1dbb79090d4a9335c721d 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8f30a940c53f43045c7a16b6bf14ce9c528cef2529410f3e61744ae4b604fdaf +oid sha256:23200a9c9d5f32d204e821ebd0df849df50e338db30ee7af5a4ecd319eecdaf2 size 107979 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_all/datasets_small/website_leaderboard.csv index b4c4bbe3b79f567fcdc13f3b692e84f3177cd697..2ea236aca7e999f24d82722bdfcab2a540ed1a99 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_small/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_all/datasets_small/website_leaderboard.csv @@ -1,51 +1,51 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1640.0,+68/-54,0.769,7.42,3.38,6.733,659.63,7.444,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1625.0,+97/-63,0.772,7.9,2.83,4.491,2289.05,8.049,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1580.0,+79/-58,0.717,9.4,3.65,5.571,2289.05,1.143,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1565.0,+86/-62,0.728,9.98,4.34,6.23,6.76,0.636,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1506.0,+68/-49,0.546,12.38,6.91,10.012,3770.75,21.902,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1470.0,+83/-61,0.54,14.01,4.71,9.579,5119.36,218.711,0.0,False,GPU -6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1464.0,+54/-48,0.441,14.28,7.78,11.203,3716.07,2.547,0.0,False,CPU -7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1448.0,+59/-48,0.429,15.08,8.77,11.395,3553.12,1.741,0.0,False,GPU -8,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1433.0,+76/-59,0.471,15.79,5.38,10.468,5119.36,28.351,0.0,False,GPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1416.0,+56/-42,0.369,16.66,8.6,11.565,3770.75,1.778,0.0,False,GPU -10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1404.0,+35/-36,0.33,17.29,12.49,13.16,892.41,2.574,0.0,False,CPU -11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1381.0,+56/-51,0.333,18.49,10.3,12.378,3553.12,0.241,0.0,False,GPU -12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1374.0,+43/-44,0.286,18.91,13.0,13.197,2476.51,0.814,0.0,False,CPU -13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1367.0,+35/-28,0.254,19.29,14.08,13.63,892.41,0.346,0.0,False,CPU -14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1360.0,+46/-36,0.263,19.62,13.76,13.375,2476.51,0.101,0.0,False,CPU -15,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1347.0,+47/-42,0.265,20.39,11.59,13.796,1126.71,2.941,0.0,False,GPU -16,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1346.0,+72/-67,0.384,20.43,6.44,13.133,50.32,43.709,0.0,False,GPU -17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1340.0,+64/-61,0.296,20.78,8.22,13.813,6521.96,8.396,0.0,False,GPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1338.0,+46/-42,0.211,20.86,9.44,13.58,6521.96,0.506,0.0,False,GPU -19,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1335.0,+36/-38,0.233,21.01,15.28,14.196,884.18,2.368,0.0,False,CPU -20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1319.0,+37/-41,0.227,21.92,14.94,14.545,9.64,0.128,0.0,False,CPU -21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1317.0,+34/-37,0.193,22.02,15.86,14.406,884.18,0.393,0.0,False,CPU -22,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1302.0,+56/-46,0.261,22.86,13.15,14.758,13.18,0.17,0.0,False,GPU -23,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1294.0,+40/-44,0.201,23.35,13.19,14.927,1126.71,0.163,0.0,False,GPU -24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1273.0,+47/-50,0.158,24.52,16.34,15.112,3865.95,2.158,0.0,False,CPU -25,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1268.0,+47/-45,0.162,24.79,15.52,16.137,3351.28,0.47,0.0,False,CPU -26,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1246.0,+48/-54,0.135,26.05,13.88,17.276,16.27,0.321,0.0,False,GPU -27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1237.0,+48/-47,0.119,26.56,16.17,16.725,3351.28,0.056,0.0,False,CPU -28,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1230.0,+39/-42,0.103,26.95,17.53,16.1,15.69,4.691,0.0,False,GPU -29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1224.0,+58/-62,0.16,27.29,15.32,17.165,1678.92,8.069,0.0,False,CPU -30,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1217.0,+70/-64,0.135,27.68,16.72,18.687,499.14,1.466,0.0,False,CPU -31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1208.0,+51/-51,0.128,28.2,17.53,17.71,9.08,0.057,0.0,False,CPU -32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1201.0,+40/-47,0.087,28.58,18.73,17.016,3865.95,0.148,0.0,False,CPU -33,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1180.0,+64/-62,0.105,29.74,16.59,19.553,499.14,0.207,0.0,False,CPU -34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1172.0,+54/-51,0.072,30.18,18.64,18.34,1678.92,0.826,0.0,False,CPU -35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1169.0,+40/-39,0.046,30.3,25.59,17.793,3.61,0.17,0.0,False,CPU -36,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1168.0,+34/-37,0.031,30.4,24.97,17.712,3.29,0.253,0.0,False,CPU -37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1158.0,+58/-70,0.091,30.94,18.82,19.98,590.04,1.461,0.0,False,CPU -38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1117.0,+56/-70,0.057,33.05,18.3,20.776,590.04,0.15,0.0,False,CPU -39,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1058.0,+48/-47,0.017,35.95,26.97,21.427,13.69,0.155,0.0,False,CPU -40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1042.0,+69/-90,0.043,36.67,27.64,23.096,5.44,1.098,0.0,False,GPU -41,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+56/-75,0.008,38.52,32.45,25.48,0.51,0.089,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,999.0,+50/-60,0.008,38.56,26.27,23.71,5.48,0.685,0.0,False,CPU -43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,998.0,+73/-96,0.019,38.58,25.8,26.644,111.84,1.215,0.0,False,CPU -44,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,993.0,+72/-72,0.015,38.81,32.76,26.326,0.47,0.092,0.0,False,CPU -45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,976.0,+86/-100,0.043,39.48,27.89,31.336,520.95,0.647,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,953.0,+84/-107,0.031,40.38,28.96,31.808,520.95,0.113,0.0,False,CPU -47,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,910.0,+73/-92,0.004,41.9,29.64,31.579,111.84,0.111,0.0,False,CPU -48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,872.0,+99/-124,0.02,43.07,32.15,34.809,2.59,0.138,0.0,False,CPU -49,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,654.0,+110/-151,0.0,47.7,42.25,45.125,0.37,0.033,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1640.0,+68/-54,6.733,0.769,7.42,3.38,659.63,7.444,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1625.0,+97/-63,4.491,0.772,7.9,2.83,2289.05,8.049,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1580.0,+79/-58,5.571,0.717,9.4,3.65,2289.05,1.143,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1565.0,+86/-62,6.23,0.728,9.98,4.34,6.76,0.636,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1506.0,+68/-49,10.012,0.546,12.38,6.91,3770.75,21.902,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1470.0,+83/-61,9.579,0.54,14.01,4.71,5119.36,218.711,✔️,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1464.0,+54/-48,11.203,0.441,14.28,7.78,3716.07,2.547,✔️,0.0,False,CPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1448.0,+59/-48,11.395,0.429,15.08,8.77,3553.12,1.741,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1433.0,+76/-59,10.468,0.471,15.79,5.38,5119.36,28.351,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1416.0,+56/-42,11.565,0.369,16.66,8.6,3770.75,1.778,✔️,0.0,False,GPU +10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1404.0,+35/-36,13.16,0.33,17.29,12.49,892.41,2.574,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1381.0,+56/-51,12.378,0.333,18.49,10.3,3553.12,0.241,✔️,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1374.0,+43/-44,13.197,0.286,18.91,13.0,2476.51,0.814,✔️,0.0,False,CPU +13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1367.0,+35/-28,13.63,0.254,19.29,14.08,892.41,0.346,✔️,0.0,False,CPU +14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1360.0,+46/-36,13.375,0.263,19.62,13.76,2476.51,0.101,✔️,0.0,False,CPU +15,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1347.0,+47/-42,13.796,0.265,20.39,11.59,1126.71,2.941,✔️,0.0,False,GPU +16,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1346.0,+72/-67,13.133,0.384,20.43,6.44,50.32,43.709,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1340.0,+64/-61,13.813,0.296,20.78,8.22,6521.96,8.396,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1338.0,+46/-42,13.58,0.211,20.86,9.44,6521.96,0.506,✔️,0.0,False,GPU +19,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1335.0,+36/-38,14.196,0.233,21.01,15.28,884.18,2.368,✔️,0.0,False,CPU +20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1319.0,+37/-41,14.545,0.227,21.92,14.94,9.64,0.128,✔️,0.0,False,CPU +21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1317.0,+34/-37,14.406,0.193,22.02,15.86,884.18,0.393,✔️,0.0,False,CPU +22,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1302.0,+56/-46,14.758,0.261,22.86,13.15,13.18,0.17,✔️,0.0,False,GPU +23,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1294.0,+40/-44,14.927,0.201,23.35,13.19,1126.71,0.163,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1273.0,+47/-50,15.112,0.158,24.52,16.34,3865.95,2.158,✔️,0.0,False,CPU +25,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1268.0,+47/-45,16.137,0.162,24.79,15.52,3351.28,0.47,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1246.0,+48/-54,17.276,0.135,26.05,13.88,16.27,0.321,✔️,0.0,False,GPU +27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1237.0,+48/-47,16.725,0.119,26.56,16.17,3351.28,0.056,✔️,0.0,False,CPU +28,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1230.0,+39/-42,16.1,0.103,26.95,17.53,15.69,4.691,✔️,0.0,False,GPU +29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1224.0,+58/-62,17.165,0.16,27.29,15.32,1678.92,8.069,✔️,0.0,False,CPU +30,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1217.0,+70/-64,18.687,0.135,27.68,16.72,499.14,1.466,✔️,0.0,False,CPU +31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1208.0,+51/-51,17.71,0.128,28.2,17.53,9.08,0.057,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1201.0,+40/-47,17.016,0.087,28.58,18.73,3865.95,0.148,✔️,0.0,False,CPU +33,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1180.0,+64/-62,19.553,0.105,29.74,16.59,499.14,0.207,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1172.0,+54/-51,18.34,0.072,30.18,18.64,1678.92,0.826,✔️,0.0,False,CPU +35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1169.0,+40/-39,17.793,0.046,30.3,25.59,3.61,0.17,✔️,0.0,False,CPU +36,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1168.0,+34/-37,17.712,0.031,30.4,24.97,3.29,0.253,✔️,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1158.0,+58/-70,19.98,0.091,30.94,18.82,590.04,1.461,✔️,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1117.0,+56/-70,20.776,0.057,33.05,18.3,590.04,0.15,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1058.0,+48/-47,21.427,0.017,35.95,26.97,13.69,0.155,✔️,0.0,False,CPU +40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1042.0,+69/-90,23.096,0.043,36.67,27.64,5.44,1.098,✔️,0.0,False,GPU +41,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+56/-75,25.48,0.008,38.52,32.45,0.51,0.089,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),999.0,+50/-60,23.71,0.008,38.56,26.27,5.48,0.685,✔️,0.0,False,CPU +43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),998.0,+73/-96,26.644,0.019,38.58,25.8,111.84,1.215,✔️,0.0,False,CPU +44,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),993.0,+72/-72,26.326,0.015,38.81,32.76,0.47,0.092,✔️,0.0,False,CPU +45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),976.0,+86/-100,31.336,0.043,39.48,27.89,520.95,0.647,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),953.0,+84/-107,31.808,0.031,40.38,28.96,520.95,0.113,✔️,0.0,False,CPU +47,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),910.0,+73/-92,31.579,0.004,41.9,29.64,111.84,0.111,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),872.0,+99/-124,34.809,0.02,43.07,32.15,2.59,0.138,✔️,0.0,False,CPU +49,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),654.0,+110/-151,45.125,0.0,47.7,42.25,0.37,0.033,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_all/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_small/winrate_matrix.png.zip index 26e4df3291be85aa6833c266e09956111b2694ea..0b169a54f56bb98f1eeb49a7b94f750fd6057b8e 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cd4f3b83906d55fa12c1647af4573b7c9816fc5a608ab11bfe4d0a5d5f10f5ce +oid sha256:10c40e7138ab23aa1e2ad6038a50adbe7e3c7f7cfe5d376584fb9fd5f401f89c size 1729917 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 0e5d9ee29ccb1bc5ac6a7fabeabc5b6b6a39f8ae..04ce9977869e956f4fb4c12cd3b539305d44dc43 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7745b6b4051e6c121203dddf9d74d5c8fd9f0f919e6eb82d895e1d89c25ab899 +oid sha256:814976a6f141963f3c1e07b42ce6e6474b20be3bdcaa25fde5755f544f3c8152 size 428426 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip index 37a4cb77bb990d9a450ebc59474aad1c5d673b02..f42caa4d774570e00081799ea163f5f29ae56f34 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:12fc29ef59fbcd6e44a02905d150fd9379bd71a8c220b77a16e800c257acf463 -size 1048964 +oid sha256:a80d25054695934b924850f798a7d9b8ccfe0122e6b6c7279ac9107116285c62 +size 1031798 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip index ebd19d0e801a9bafb8b9ec68dc493b1ee141776e..8d0c8de42a42b3a886affccfc5c8939e4c3e87d4 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:98d67d02c424a2064c561b33cd726834309a1e67f88ff45806d70266bb4fef6b +oid sha256:10a5b863795e2f720c603c8ebd1977e82aa7685b1d9e4d6a96453a5aea59f906 size 115758 diff --git a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/website_leaderboard.csv index 0c25af8e6c53948d5f8fd5b83bab734bdf7c3cca..d457704ce3fca7b09a4721e225ec12cf52e14791 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/website_leaderboard.csv @@ -1,56 +1,56 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1640.0,+95/-71,0.744,9.05,3.4,5.186,2166.18,7.945,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1637.0,+78/-54,0.72,9.17,4.23,7.606,611.55,7.01,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1602.0,+66/-63,0.723,10.54,5.18,6.822,6.14,0.586,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1597.0,+72/-62,0.693,10.73,4.24,6.284,2166.18,0.993,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1546.0,+74/-50,0.591,12.97,5.42,9.458,3899.42,55.833,0.0,False,GPU -5,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1516.0,+74/-47,0.559,14.46,5.84,10.623,136.59,3.391,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1514.0,+54/-45,0.504,14.56,8.05,11.014,3547.04,20.963,0.0,False,GPU -7,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1506.0,+74/-71,0.586,14.97,5.38,13.034,4.85,0.629,0.0,False,GPU -8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1468.0,+75/-61,0.481,16.95,5.43,10.616,4980.83,207.372,0.0,False,GPU -9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1459.0,+46/-52,0.384,17.46,9.83,12.287,3376.59,2.54,0.0,False,CPU -10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1446.0,+55/-50,0.373,18.19,10.24,12.487,3372.56,1.658,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1442.0,+80/-69,0.445,18.39,5.94,11.415,4980.83,23.583,0.0,False,GPU -12,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1439.0,+82/-69,0.434,18.6,6.82,12.024,3899.42,0.983,0.0,False,GPU -13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1422.0,+56/-47,0.337,19.55,10.33,12.593,3547.04,1.57,0.0,False,GPU -14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1398.0,+36/-36,0.258,20.98,15.0,14.456,771.57,2.49,0.0,False,CPU -15,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1390.0,+71/-69,0.399,21.43,9.2,13.505,4.22,0.554,0.0,False,GPU -16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1384.0,+58/-51,0.284,21.83,11.77,13.499,3372.56,0.208,0.0,False,GPU -17,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1382.0,+42/-37,0.24,21.91,15.08,14.384,2034.85,0.805,0.0,False,CPU -18,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1368.0,+37/-33,0.219,22.76,16.26,14.581,2034.85,0.097,0.0,False,CPU -19,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1366.0,+90/-66,0.381,22.88,6.96,14.141,48.9,43.804,0.0,False,GPU -20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1363.0,+36/-30,0.189,23.09,16.97,14.932,771.57,0.322,0.0,False,CPU -21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1342.0,+50/-45,0.205,24.36,14.6,15.104,1105.59,2.552,0.0,False,GPU -22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1338.0,+36/-36,0.181,24.6,18.37,15.506,828.74,2.312,0.0,False,CPU -23,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1338.0,+73/-67,0.25,24.64,11.28,15.155,6147.69,8.149,0.0,False,GPU -24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1338.0,+46/-49,0.17,24.64,11.71,14.737,6147.69,0.485,0.0,False,GPU -25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1327.0,+48/-45,0.184,25.31,17.24,15.809,8.51,0.123,0.0,False,CPU -26,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1320.0,+32/-34,0.142,25.71,18.6,15.718,828.74,0.336,0.0,False,CPU -27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1313.0,+54/-53,0.226,26.15,14.69,15.881,12.24,0.154,0.0,False,GPU -28,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1296.0,+45/-40,0.151,27.25,16.29,16.243,1105.59,0.13,0.0,False,GPU -29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1282.0,+48/-45,0.12,28.07,18.33,17.283,2929.85,0.433,0.0,False,CPU -30,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1275.0,+48/-54,0.124,28.54,20.3,16.303,3704.3,2.074,0.0,False,CPU -31,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1267.0,+47/-42,0.118,28.99,15.86,17.906,16.16,0.307,0.0,False,GPU -32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1252.0,+52/-49,0.084,29.95,18.24,17.885,2929.85,0.054,0.0,False,CPU -33,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1245.0,+42/-38,0.082,30.35,19.8,17.002,15.48,4.535,0.0,False,GPU -34,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1236.0,+51/-62,0.142,30.9,16.77,18.355,1459.62,8.056,0.0,False,CPU -35,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1219.0,+49/-53,0.078,31.95,19.88,18.772,8.47,0.054,0.0,False,CPU -36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1213.0,+60/-60,0.101,32.33,21.51,20.26,416.39,1.393,0.0,False,CPU -37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1203.0,+47/-48,0.065,32.91,21.43,18.22,3704.3,0.143,0.0,False,CPU -38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1181.0,+56/-61,0.063,34.18,20.76,19.544,1459.62,0.89,0.0,False,CPU -39,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1177.0,+57/-61,0.069,34.43,22.54,21.094,416.39,0.179,0.0,False,CPU -40,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1172.0,+37/-38,0.017,34.71,29.77,19.131,3.05,0.241,0.0,False,CPU -41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1158.0,+33/-41,0.021,35.5,31.12,19.359,3.39,0.158,0.0,False,CPU -42,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1142.0,+56/-58,0.047,36.41,24.9,21.744,572.67,1.421,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1103.0,+47/-48,0.019,38.56,25.42,22.52,572.67,0.143,0.0,False,CPU -44,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1067.0,+50/-60,0.012,40.39,30.45,22.688,11.82,0.147,0.0,False,CPU -45,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1053.0,+71/-83,0.03,41.1,31.46,24.705,5.39,1.034,0.0,False,GPU -46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1010.0,+55/-66,0.0,43.07,28.88,24.827,5.18,0.652,0.0,False,CPU -47,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1004.0,+72/-90,0.012,43.34,31.16,28.179,100.58,1.106,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1003.0,+71/-108,0.044,43.36,30.05,31.292,504.17,0.629,0.0,False,CPU -49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1002.0,+83/-87,0.008,43.4,36.68,27.374,0.42,0.078,0.0,False,CPU -50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+55/-70,0.002,43.49,37.64,26.617,0.47,0.069,0.0,False,CPU -51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,980.0,+74/-108,0.03,44.33,31.71,31.725,504.17,0.103,0.0,False,CPU -52,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,908.0,+78/-103,0.0,47.0,33.68,33.337,100.58,0.101,0.0,False,CPU -53,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,903.0,+82/-129,0.019,47.19,34.88,34.211,2.51,0.137,0.0,False,CPU -54,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,680.0,+97/-169,0.0,52.44,45.72,45.686,0.29,0.028,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1640.0,+95/-71,5.186,0.744,9.05,3.4,2166.18,7.945,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1637.0,+78/-54,7.606,0.72,9.17,4.23,611.55,7.01,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1602.0,+66/-63,6.822,0.723,10.54,5.18,6.14,0.586,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1597.0,+72/-62,6.284,0.693,10.73,4.24,2166.18,0.993,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1546.0,+74/-50,9.458,0.591,12.97,5.42,3899.42,55.833,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1516.0,+74/-47,10.623,0.559,14.46,5.84,136.59,3.391,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1514.0,+54/-45,11.014,0.504,14.56,8.05,3547.04,20.963,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1506.0,+74/-71,13.034,0.586,14.97,5.38,4.85,0.629,➖,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1468.0,+75/-61,10.616,0.481,16.95,5.43,4980.83,207.372,✔️,0.0,False,GPU +9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1459.0,+46/-52,12.287,0.384,17.46,9.83,3376.59,2.54,✔️,0.0,False,CPU +10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1446.0,+55/-50,12.487,0.373,18.19,10.24,3372.56,1.658,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1442.0,+80/-69,11.415,0.445,18.39,5.94,4980.83,23.583,✔️,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1439.0,+82/-69,12.024,0.434,18.6,6.82,3899.42,0.983,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1422.0,+56/-47,12.593,0.337,19.55,10.33,3547.04,1.57,✔️,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1398.0,+36/-36,14.456,0.258,20.98,15.0,771.57,2.49,✔️,0.0,False,CPU +15,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1390.0,+71/-69,13.505,0.399,21.43,9.2,4.22,0.554,✔️,0.0,False,GPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1384.0,+58/-51,13.499,0.284,21.83,11.77,3372.56,0.208,✔️,0.0,False,GPU +17,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1382.0,+42/-37,14.384,0.24,21.91,15.08,2034.85,0.805,✔️,0.0,False,CPU +18,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1368.0,+37/-33,14.581,0.219,22.76,16.26,2034.85,0.097,✔️,0.0,False,CPU +19,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1366.0,+90/-66,14.141,0.381,22.88,6.96,48.9,43.804,✔️,0.0,False,GPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1363.0,+36/-30,14.932,0.189,23.09,16.97,771.57,0.322,✔️,0.0,False,CPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1342.0,+50/-45,15.104,0.205,24.36,14.6,1105.59,2.552,✔️,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1338.0,+36/-36,15.506,0.181,24.6,18.37,828.74,2.312,✔️,0.0,False,CPU +23,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1338.0,+73/-67,15.155,0.25,24.64,11.28,6147.69,8.149,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1338.0,+46/-49,14.737,0.17,24.64,11.71,6147.69,0.485,✔️,0.0,False,GPU +25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1327.0,+48/-45,15.809,0.184,25.31,17.24,8.51,0.123,✔️,0.0,False,CPU +26,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1320.0,+32/-34,15.718,0.142,25.71,18.6,828.74,0.336,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1313.0,+54/-53,15.881,0.226,26.15,14.69,12.24,0.154,✔️,0.0,False,GPU +28,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1296.0,+45/-40,16.243,0.151,27.25,16.29,1105.59,0.13,✔️,0.0,False,GPU +29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1282.0,+48/-45,17.283,0.12,28.07,18.33,2929.85,0.433,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1275.0,+48/-54,16.303,0.124,28.54,20.3,3704.3,2.074,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1267.0,+47/-42,17.906,0.118,28.99,15.86,16.16,0.307,✔️,0.0,False,GPU +32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1252.0,+52/-49,17.885,0.084,29.95,18.24,2929.85,0.054,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1245.0,+42/-38,17.002,0.082,30.35,19.8,15.48,4.535,✔️,0.0,False,GPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1236.0,+51/-62,18.355,0.142,30.9,16.77,1459.62,8.056,✔️,0.0,False,CPU +35,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1219.0,+49/-53,18.772,0.078,31.95,19.88,8.47,0.054,✔️,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1213.0,+60/-60,20.26,0.101,32.33,21.51,416.39,1.393,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1203.0,+47/-48,18.22,0.065,32.91,21.43,3704.3,0.143,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1181.0,+56/-61,19.544,0.063,34.18,20.76,1459.62,0.89,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1177.0,+57/-61,21.094,0.069,34.43,22.54,416.39,0.179,✔️,0.0,False,CPU +40,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1172.0,+37/-38,19.131,0.017,34.71,29.77,3.05,0.241,✔️,0.0,False,CPU +41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1158.0,+33/-41,19.359,0.021,35.5,31.12,3.39,0.158,✔️,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1142.0,+56/-58,21.744,0.047,36.41,24.9,572.67,1.421,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1103.0,+47/-48,22.52,0.019,38.56,25.42,572.67,0.143,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1067.0,+50/-60,22.688,0.012,40.39,30.45,11.82,0.147,✔️,0.0,False,CPU +45,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1053.0,+71/-83,24.705,0.03,41.1,31.46,5.39,1.034,✔️,0.0,False,GPU +46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1010.0,+55/-66,24.827,0.0,43.07,28.88,5.18,0.652,✔️,0.0,False,CPU +47,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1004.0,+72/-90,28.179,0.012,43.34,31.16,100.58,1.106,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1003.0,+71/-108,31.292,0.044,43.36,30.05,504.17,0.629,✔️,0.0,False,CPU +49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1002.0,+83/-87,27.374,0.008,43.4,36.68,0.42,0.078,✔️,0.0,False,CPU +50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+55/-70,26.617,0.002,43.49,37.64,0.47,0.069,✔️,0.0,False,CPU +51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),980.0,+74/-108,31.725,0.03,44.33,31.71,504.17,0.103,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),908.0,+78/-103,33.337,0.0,47.0,33.68,100.58,0.101,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),903.0,+82/-129,34.211,0.019,47.19,34.88,2.51,0.137,✔️,0.0,False,CPU +54,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),680.0,+97/-169,45.686,0.0,52.44,45.72,0.29,0.028,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/winrate_matrix.png.zip index ab5fde715bb23f5b76c2dc50218d746567d19d6c..03d6d0cf1d70379371b4e24e09cc179487c64bfc 100644 --- a/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_all/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a4eb4cf5b22ffe61c7126a39e9142cefcf53241d59bc19a5ba0daef6274f7377 +oid sha256:6f6d351346ebb8ff8cb6cdd49a2cb2fa092dddbf6561bbaaaecbbbe2af43173a size 2212024 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index 77a091b2d02c1b733b84291b78a4c726927a61d9..f6786727aec371c9d5ce331ccb676b29dc688b5a 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8979e608672d5a0f32a339d07ff373d69dc10b01e1d2fa5cdbc9808720dd16c2 +oid sha256:f1e3a277f7edcdee89d89e95d1b5480b0d72b7271e7a90bbfa3944f443df0c9e size 429276 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip index d17d3d36fed03539daa2611d3312eba6d5ecd48e..ce98a89faab97a1068586de90bee92e2d1fad4c5 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:92fb1048a148173627e64602c7e3334ab49dcdc5eda005fd84595d81d8f26a17 -size 985265 +oid sha256:e34bbe80e56bc3cda9aa62b92aeab5758bd1ddb6d7c51a51d9c3f1f753eb6f0c +size 958750 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_all/tuning-impact-elo.png.zip index e4cb3ac95b85658a2268ec28fca38856280e424a..cb6ee0b1ea88565f9712615a49014dc30a6ee1bb 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2185545f1af5db251cf8b3acfd34ea001ca9e9e4c816eaf000981ccf7b09a8c0 +oid sha256:71fd051104b26359c822debf594b3a093680f7a2d753ec01e6f2dd257bdcc831 size 113807 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_binary/datasets_all/website_leaderboard.csv index e840217cfd8bf4440b36d8485b8238fa6ca0ae4b..00ce45e6dcdcad6f26f56051778f91962a81e830 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_all/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_binary/datasets_all/website_leaderboard.csv @@ -1,53 +1,53 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1657.0,+98/-71,0.803,7.38,2.99,5.749,442.09,6.381,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1568.0,+110/-63,0.681,10.55,3.6,6.02,1923.88,9.103,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1545.0,+101/-57,0.687,11.52,4.71,7.098,4.91,0.694,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1535.0,+101/-59,0.632,11.94,4.35,7.13,1923.88,1.329,0.0,False,GPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1515.0,+73/-65,0.555,12.85,5.23,9.038,1152.95,1.784,0.0,False,CPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1500.0,+53/-56,0.508,13.58,8.76,10.144,2775.44,8.849,0.0,False,GPU -6,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1475.0,+76/-57,0.463,14.8,7.23,10.795,2229.7,1.387,0.0,False,GPU -7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1438.0,+55/-38,0.382,16.76,10.85,12.3,328.96,1.102,0.0,False,CPU -8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1422.0,+72/-62,0.436,17.64,5.97,9.134,4982.66,358.753,0.0,False,GPU -9,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1416.0,+81/-59,0.371,17.95,8.6,11.711,2229.7,0.133,0.0,False,GPU -10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1408.0,+61/-52,0.353,18.37,9.9,12.08,1055.6,0.482,0.0,False,CPU -11,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1403.0,+59/-53,0.353,18.64,9.24,12.174,1055.6,0.063,0.0,False,CPU -12,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1403.0,+56/-50,0.374,18.67,9.35,12.048,4.27,0.064,0.0,False,CPU -13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1400.0,+52/-56,0.298,18.83,9.7,11.944,2775.44,0.415,0.0,False,GPU -14,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1385.0,+62/-53,0.314,19.7,11.17,13.208,472.99,0.621,0.0,False,CPU -15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1384.0,+49/-31,0.268,19.73,13.42,13.159,328.96,0.126,0.0,False,CPU -16,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1380.0,+68/-68,0.371,19.99,9.43,14.06,250.36,1.708,0.0,False,GPU -17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1367.0,+94/-94,0.374,20.73,8.42,12.88,3383.91,8.396,0.0,False,GPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1366.0,+58/-59,0.248,20.75,9.56,12.838,3383.91,0.51,0.0,False,GPU -19,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1355.0,+65/-58,0.301,21.39,7.03,10.825,4982.66,42.199,0.0,False,GPU -20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1350.0,+49/-43,0.236,21.73,14.36,13.724,472.99,0.13,0.0,False,CPU -21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1331.0,+58/-56,0.21,22.85,12.96,13.837,642.33,2.767,0.0,False,GPU -22,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1318.0,+49/-41,0.197,23.61,15.72,13.457,2241.65,2.415,0.0,False,CPU -23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1312.0,+69/-65,0.259,23.95,12.93,14.413,8.72,0.126,0.0,False,GPU -24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1311.0,+52/-48,0.18,24.03,15.4,15.332,1560.02,0.404,0.0,False,CPU -25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1302.0,+47/-58,0.178,24.57,13.95,15.116,642.33,0.146,0.0,False,GPU -26,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1283.0,+64/-72,0.243,25.68,9.36,14.197,49.67,43.824,0.0,False,GPU -27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1274.0,+51/-46,0.121,26.2,16.49,16.045,1560.02,0.052,0.0,False,CPU -28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1267.0,+50/-42,0.15,26.62,13.82,15.891,13.5,0.311,0.0,False,GPU -29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1267.0,+60/-64,0.152,26.62,16.59,15.482,559.38,4.08,0.0,False,CPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1265.0,+44/-44,0.11,26.75,17.68,14.803,9.33,1.116,0.0,False,GPU -31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1248.0,+49/-52,0.123,27.77,17.29,17.1,4.98,0.049,0.0,False,CPU -32,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1231.0,+66/-56,0.117,28.74,18.33,16.365,1.24,0.11,0.0,False,CPU -33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1226.0,+46/-39,0.077,29.08,19.82,15.668,2241.65,0.136,0.0,False,CPU -34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1191.0,+57/-63,0.061,31.05,19.79,17.521,559.38,0.276,0.0,False,CPU -35,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1188.0,+51/-53,0.061,31.26,22.1,18.0,125.38,0.674,0.0,False,CPU -36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1179.0,+49/-34,0.047,31.73,27.14,17.148,0.81,0.093,0.0,False,CPU -37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1154.0,+51/-57,0.035,33.12,22.93,19.362,169.75,0.603,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1146.0,+50/-53,0.038,33.56,23.89,19.162,125.38,0.069,0.0,False,CPU -39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1119.0,+43/-54,0.018,35.01,23.64,20.091,169.75,0.059,0.0,False,CPU -40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1093.0,+65/-48,0.019,36.39,27.59,20.071,4.95,0.091,0.0,False,CPU -41,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1057.0,+64/-82,0.021,38.16,24.5,21.46,2.56,0.267,0.0,False,CPU -42,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1051.0,+84/-107,0.054,38.43,26.2,26.808,162.68,0.405,0.0,False,CPU -43,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1045.0,+109/-116,0.099,38.71,22.36,25.52,0.85,0.129,0.0,False,GPU -44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1022.0,+61/-103,0.017,39.74,28.61,24.212,139.84,3.917,0.0,False,CPU -45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1022.0,+91/-118,0.039,39.75,27.56,27.502,162.68,0.054,0.0,False,CPU -46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+62/-79,0.013,40.7,33.84,24.287,0.25,0.029,0.0,False,CPU -47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,983.0,+73/-101,0.026,41.38,33.73,26.581,2.8,0.564,0.0,False,GPU -48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,959.0,+86/-119,0.024,42.33,30.48,29.448,0.84,0.094,0.0,False,CPU -49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,955.0,+75/-92,0.01,42.47,36.5,25.759,0.2,0.037,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,915.0,+82/-136,0.0,43.91,31.65,29.325,139.84,0.193,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,602.0,+105/-199,0.0,50.36,46.66,43.863,0.15,0.023,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1657.0,+98/-71,5.749,0.803,7.38,2.99,442.09,6.381,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1568.0,+110/-63,6.02,0.681,10.55,3.6,1923.88,9.103,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1545.0,+101/-57,7.098,0.687,11.52,4.71,4.91,0.694,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1535.0,+101/-59,7.13,0.632,11.94,4.35,1923.88,1.329,✔️,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1515.0,+73/-65,9.038,0.555,12.85,5.23,1152.95,1.784,✔️,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1500.0,+53/-56,10.144,0.508,13.58,8.76,2775.44,8.849,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1475.0,+76/-57,10.795,0.463,14.8,7.23,2229.7,1.387,✔️,0.0,False,GPU +7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1438.0,+55/-38,12.3,0.382,16.76,10.85,328.96,1.102,✔️,0.0,False,CPU +8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1422.0,+72/-62,9.134,0.436,17.64,5.97,4982.66,358.753,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1416.0,+81/-59,11.711,0.371,17.95,8.6,2229.7,0.133,✔️,0.0,False,GPU +10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1408.0,+61/-52,12.08,0.353,18.37,9.9,1055.6,0.482,✔️,0.0,False,CPU +11,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1403.0,+59/-53,12.174,0.353,18.64,9.24,1055.6,0.063,✔️,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1403.0,+56/-50,12.048,0.374,18.67,9.35,4.27,0.064,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1400.0,+52/-56,11.944,0.298,18.83,9.7,2775.44,0.415,✔️,0.0,False,GPU +14,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1385.0,+62/-53,13.208,0.314,19.7,11.17,472.99,0.621,✔️,0.0,False,CPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1384.0,+49/-31,13.159,0.268,19.73,13.42,328.96,0.126,✔️,0.0,False,CPU +16,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1380.0,+68/-68,14.06,0.371,19.99,9.43,250.36,1.708,➖,0.0,False,GPU +17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1367.0,+94/-94,12.88,0.374,20.73,8.42,3383.91,8.396,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1366.0,+58/-59,12.838,0.248,20.75,9.56,3383.91,0.51,✔️,0.0,False,GPU +19,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1355.0,+65/-58,10.825,0.301,21.39,7.03,4982.66,42.199,✔️,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1350.0,+49/-43,13.724,0.236,21.73,14.36,472.99,0.13,✔️,0.0,False,CPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1331.0,+58/-56,13.837,0.21,22.85,12.96,642.33,2.767,✔️,0.0,False,GPU +22,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1318.0,+49/-41,13.457,0.197,23.61,15.72,2241.65,2.415,✔️,0.0,False,CPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1312.0,+69/-65,14.413,0.259,23.95,12.93,8.72,0.126,✔️,0.0,False,GPU +24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1311.0,+52/-48,15.332,0.18,24.03,15.4,1560.02,0.404,✔️,0.0,False,CPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1302.0,+47/-58,15.116,0.178,24.57,13.95,642.33,0.146,✔️,0.0,False,GPU +26,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1283.0,+64/-72,14.197,0.243,25.68,9.36,49.67,43.824,✔️,0.0,False,GPU +27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1274.0,+51/-46,16.045,0.121,26.2,16.49,1560.02,0.052,✔️,0.0,False,CPU +28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1267.0,+50/-42,15.891,0.15,26.62,13.82,13.5,0.311,✔️,0.0,False,GPU +29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1267.0,+60/-64,15.482,0.152,26.62,16.59,559.38,4.08,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1265.0,+44/-44,14.803,0.11,26.75,17.68,9.33,1.116,✔️,0.0,False,GPU +31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1248.0,+49/-52,17.1,0.123,27.77,17.29,4.98,0.049,✔️,0.0,False,CPU +32,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1231.0,+66/-56,16.365,0.117,28.74,18.33,1.24,0.11,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1226.0,+46/-39,15.668,0.077,29.08,19.82,2241.65,0.136,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1191.0,+57/-63,17.521,0.061,31.05,19.79,559.38,0.276,✔️,0.0,False,CPU +35,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1188.0,+51/-53,18.0,0.061,31.26,22.1,125.38,0.674,✔️,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1179.0,+49/-34,17.148,0.047,31.73,27.14,0.81,0.093,✔️,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1154.0,+51/-57,19.362,0.035,33.12,22.93,169.75,0.603,✔️,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1146.0,+50/-53,19.162,0.038,33.56,23.89,125.38,0.069,✔️,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1119.0,+43/-54,20.091,0.018,35.01,23.64,169.75,0.059,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1093.0,+65/-48,20.071,0.019,36.39,27.59,4.95,0.091,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1057.0,+64/-82,21.46,0.021,38.16,24.5,2.56,0.267,✔️,0.0,False,CPU +42,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1051.0,+84/-107,26.808,0.054,38.43,26.2,162.68,0.405,✔️,0.0,False,CPU +43,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1045.0,+109/-116,25.52,0.099,38.71,22.36,0.85,0.129,➖,0.0,False,GPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1022.0,+61/-103,24.212,0.017,39.74,28.61,139.84,3.917,✔️,0.0,False,CPU +45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1022.0,+91/-118,27.502,0.039,39.75,27.56,162.68,0.054,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+62/-79,24.287,0.013,40.7,33.84,0.25,0.029,✔️,0.0,False,CPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),983.0,+73/-101,26.581,0.026,41.38,33.73,2.8,0.564,✔️,0.0,False,GPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),959.0,+86/-119,29.448,0.024,42.33,30.48,0.84,0.094,✔️,0.0,False,CPU +49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),955.0,+75/-92,25.759,0.01,42.47,36.5,0.2,0.037,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),915.0,+82/-136,29.325,0.0,43.91,31.65,139.84,0.193,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),602.0,+105/-199,43.863,0.0,50.36,46.66,0.15,0.023,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_all/winrate_matrix.png.zip index e09bac4ac78c6c9462ef01863fee3e21cd51228c..097c95c97446ac6867eb98e1e58a4b568bf80fbd 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:783344ba6c5f6c1d3bc94e50769f5330bbc21fb45bec22d445dd7f76d69ede86 +oid sha256:ccd8f2c2d9058269966c9ba26269dc384f77ba122a369a0a275ecfa62c24acdf size 2071274 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index ad74532ae0c43305b13ce94b72a2641049023bce..e2181cf98dd316af8a806593aa39bd16fc8ef05f 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7db63d502490ea6f171e4158f07599e360bcbf896034b80539013dc4cf2669f8 +oid sha256:37573f734e5396b487c7796eb775df4fbc8e00a9d50f06378ce92b8ae7ed5634 size 419894 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip index 4ad2c8ac9f7e4fe6df034454bbc6e30b00baa280..5a1f7d5d5d36e45e6f181b04442b21d9166a95e1 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:44ee0506dd2e7e004d3ad5fe55a48e72879dc48b4002bceb5592b4ebd3c996f5 -size 990457 +oid sha256:4a9b23b48d00a250039cfd2765632d6e5bb7cfe45d3bb6fd9de91cf1fd374183 +size 932514 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_medium/tuning-impact-elo.png.zip index 311239e3010d7de2ef154431c11385fcdb0a3f47..dd3ec9f360edc5513b4af9045cd919a550e75597 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f0b85edce9fc6e7fed4af0e845e4a7252b0e0c96601efb601290a81220d792ec +oid sha256:2249bd657be968fbaec47b182e32bd1eed209524c3c08f04c03eff4517edf112 size 118425 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_binary/datasets_medium/website_leaderboard.csv index 8cfc57ca22cb179cf53215cdc5a783b81af95939..8faf9eec1b35da574394391d25a2752f0e3a4754 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_medium/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_binary/datasets_medium/website_leaderboard.csv @@ -1,54 +1,54 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1966.0,+155/-60,0.904,4.95,2.42,2.212,275.07,2.448,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1904.0,+176/-64,0.851,6.35,2.7,2.49,325.91,2.156,0.0,False,CPU -2,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1759.0,+141/-88,0.631,10.98,5.87,4.725,619.15,0.198,0.0,False,CPU -3,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1754.0,+144/-81,0.636,11.17,4.87,5.018,619.15,0.016,0.0,False,CPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1751.0,+113/-38,0.575,11.28,8.0,5.44,1633.8,2.053,0.0,False,GPU -5,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1736.0,+126/-79,0.63,11.87,5.47,4.763,2.21,0.025,0.0,False,CPU -6,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1718.0,+152/-92,0.545,12.6,6.95,7.508,110.13,0.386,0.0,False,CPU -7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1701.0,+149/-83,0.524,13.3,8.51,7.923,86.85,0.668,0.0,False,CPU -8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1681.0,+192/-136,0.513,14.19,5.59,7.27,804.65,0.71,0.0,False,GPU -9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1679.0,+154/-138,0.544,14.26,7.1,6.907,2139.17,6.41,0.0,False,GPU -10,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1660.0,+210/-122,0.489,15.1,5.39,8.381,842.24,15.119,0.0,False,GPU -11,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1652.0,+124/-66,0.376,15.51,8.78,7.028,1633.8,0.088,0.0,False,GPU -12,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1640.0,+124/-80,0.421,16.04,11.5,8.376,110.13,0.061,0.0,False,CPU -13,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1625.0,+197/-156,0.458,16.73,6.29,7.978,804.65,0.063,0.0,False,GPU -14,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1610.0,+194/-117,0.419,17.44,6.83,9.033,842.24,1.465,0.0,False,GPU -15,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1582.0,+158/-128,0.416,18.81,7.95,8.63,2.3,0.896,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1580.0,+163/-89,0.323,18.95,12.1,9.74,86.85,0.078,0.0,False,CPU -17,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1564.0,+170/-166,0.42,19.72,8.41,9.211,8.53,3.578,0.0,False,GPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1552.0,+126/-84,0.253,20.35,12.42,9.782,2139.17,0.349,0.0,False,GPU -19,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1536.0,+178/-158,0.347,21.12,7.53,9.019,4860.88,475.791,0.0,False,GPU -20,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1527.0,+95/-112,0.248,21.62,14.48,9.12,1156.69,1.558,0.0,False,CPU -21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1490.0,+141/-106,0.196,23.53,15.19,10.678,572.82,1.935,0.0,False,GPU -22,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1478.0,+124/-111,0.174,24.2,17.96,12.87,270.2,0.192,0.0,False,CPU -23,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1461.0,+126/-146,0.228,25.07,13.69,10.861,0.49,0.053,0.0,False,CPU -24,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1454.0,+132/-118,0.163,25.44,19.32,12.698,572.82,0.109,0.0,False,GPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1453.0,+156/-178,0.241,25.48,14.6,10.49,5.5,0.074,0.0,False,GPU -26,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1431.0,+105/-129,0.123,26.67,20.62,11.432,371.59,2.386,0.0,False,CPU -27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1430.0,+81/-65,0.077,26.75,23.68,10.923,6.41,0.247,0.0,False,GPU -28,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1417.0,+121/-138,0.136,27.44,20.16,13.96,96.61,0.754,0.0,False,GPU -29,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1404.0,+129/-107,0.091,28.11,24.03,13.997,270.2,0.015,0.0,False,CPU -30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1380.0,+97/-126,0.069,29.38,23.47,11.531,1156.69,0.054,0.0,False,CPU -31,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1374.0,+134/-137,0.087,29.67,18.69,12.762,4860.88,42.641,0.0,False,GPU -32,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1361.0,+129/-110,0.05,30.35,26.41,15.831,1.07,0.018,0.0,False,CPU -33,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1343.0,+98/-137,0.069,31.22,28.09,12.461,0.29,0.033,0.0,False,CPU -34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1319.0,+52/-59,0.002,32.42,30.58,14.148,42.95,0.226,0.0,False,CPU -35,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1314.0,+88/-85,0.008,32.69,29.68,13.953,8.34,0.25,0.0,False,GPU -36,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1284.0,+174/-186,0.035,34.12,20.85,17.019,47.03,45.876,0.0,False,GPU -37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1280.0,+61/-90,0.0,34.32,32.02,13.457,40.36,0.318,0.0,False,CPU -38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1279.0,+74/-74,0.005,34.35,30.6,15.128,42.95,0.021,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1278.0,+84/-119,0.036,34.41,29.27,15.031,371.59,0.102,0.0,False,CPU -40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1224.0,+66/-92,0.0,36.86,35.23,15.173,1.86,0.046,0.0,False,CPU -41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1218.0,+64/-73,0.0,37.12,35.45,15.174,40.36,0.028,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1206.0,+134/-218,0.042,37.63,32.31,18.046,1.52,0.115,0.0,False,CPU -43,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1162.0,+147/-233,0.009,39.45,35.34,24.034,43.64,0.085,0.0,False,CPU -44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1140.0,+101/-190,0.011,40.28,31.54,20.729,513.06,78.02,0.0,False,CPU -45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1112.0,+166/-225,0.005,41.35,38.03,25.027,43.64,0.022,0.0,False,CPU -46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+88/-208,0.0,44.96,44.23,22.275,0.12,0.014,0.0,False,CPU -47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,982.0,+135/-219,0.0,45.44,43.9,26.73,0.22,0.024,0.0,False,CPU -48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,972.0,+202/-455,0.025,45.72,39.86,28.513,2.62,0.186,0.0,False,GPU -49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,918.0,+91/-188,0.0,47.05,46.26,30.352,0.24,0.036,0.0,False,GPU -50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,916.0,+100/-247,0.0,47.09,46.63,24.142,0.1,0.021,0.0,False,CPU -51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,883.0,+76/-126,0.0,47.81,47.49,27.794,513.06,2.568,0.0,False,CPU -52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,514.0,+107/-398,0.0,52.27,52.24,45.591,0.11,0.087,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1966.0,+155/-60,2.212,0.904,4.95,2.42,275.07,2.448,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1904.0,+176/-64,2.49,0.851,6.35,2.7,325.91,2.156,✔️,0.0,False,CPU +2,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1759.0,+141/-88,4.725,0.631,10.98,5.87,619.15,0.198,✔️,0.0,False,CPU +3,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1754.0,+144/-81,5.018,0.636,11.17,4.87,619.15,0.016,✔️,0.0,False,CPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1751.0,+113/-38,5.44,0.575,11.28,8.0,1633.8,2.053,✔️,0.0,False,GPU +5,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1736.0,+126/-79,4.763,0.63,11.87,5.47,2.21,0.025,✔️,0.0,False,CPU +6,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1718.0,+152/-92,7.508,0.545,12.6,6.95,110.13,0.386,✔️,0.0,False,CPU +7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1701.0,+149/-83,7.923,0.524,13.3,8.51,86.85,0.668,✔️,0.0,False,CPU +8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1681.0,+192/-136,7.27,0.513,14.19,5.59,804.65,0.71,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1679.0,+154/-138,6.907,0.544,14.26,7.1,2139.17,6.41,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1660.0,+210/-122,8.381,0.489,15.1,5.39,842.24,15.119,✔️,0.0,False,GPU +11,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1652.0,+124/-66,7.028,0.376,15.51,8.78,1633.8,0.088,✔️,0.0,False,GPU +12,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1640.0,+124/-80,8.376,0.421,16.04,11.5,110.13,0.061,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1625.0,+197/-156,7.978,0.458,16.73,6.29,804.65,0.063,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1610.0,+194/-117,9.033,0.419,17.44,6.83,842.24,1.465,✔️,0.0,False,GPU +15,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1582.0,+158/-128,8.63,0.416,18.81,7.95,2.3,0.896,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1580.0,+163/-89,9.74,0.323,18.95,12.1,86.85,0.078,✔️,0.0,False,CPU +17,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1564.0,+170/-166,9.211,0.42,19.72,8.41,8.53,3.578,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1552.0,+126/-84,9.782,0.253,20.35,12.42,2139.17,0.349,✔️,0.0,False,GPU +19,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1536.0,+178/-158,9.019,0.347,21.12,7.53,4860.88,475.791,✔️,0.0,False,GPU +20,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1527.0,+95/-112,9.12,0.248,21.62,14.48,1156.69,1.558,✔️,0.0,False,CPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1490.0,+141/-106,10.678,0.196,23.53,15.19,572.82,1.935,✔️,0.0,False,GPU +22,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1478.0,+124/-111,12.87,0.174,24.2,17.96,270.2,0.192,✔️,0.0,False,CPU +23,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1461.0,+126/-146,10.861,0.228,25.07,13.69,0.49,0.053,✔️,0.0,False,CPU +24,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1454.0,+132/-118,12.698,0.163,25.44,19.32,572.82,0.109,✔️,0.0,False,GPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1453.0,+156/-178,10.49,0.241,25.48,14.6,5.5,0.074,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1431.0,+105/-129,11.432,0.123,26.67,20.62,371.59,2.386,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1430.0,+81/-65,10.923,0.077,26.75,23.68,6.41,0.247,✔️,0.0,False,GPU +28,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1417.0,+121/-138,13.96,0.136,27.44,20.16,96.61,0.754,➖,0.0,False,GPU +29,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1404.0,+129/-107,13.997,0.091,28.11,24.03,270.2,0.015,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1380.0,+97/-126,11.531,0.069,29.38,23.47,1156.69,0.054,✔️,0.0,False,CPU +31,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1374.0,+134/-137,12.762,0.087,29.67,18.69,4860.88,42.641,✔️,0.0,False,GPU +32,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1361.0,+129/-110,15.831,0.05,30.35,26.41,1.07,0.018,✔️,0.0,False,CPU +33,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1343.0,+98/-137,12.461,0.069,31.22,28.09,0.29,0.033,✔️,0.0,False,CPU +34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1319.0,+52/-59,14.148,0.002,32.42,30.58,42.95,0.226,✔️,0.0,False,CPU +35,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1314.0,+88/-85,13.953,0.008,32.69,29.68,8.34,0.25,✔️,0.0,False,GPU +36,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1284.0,+174/-186,17.019,0.035,34.12,20.85,47.03,45.876,✔️,0.0,False,GPU +37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1280.0,+61/-90,13.457,0.0,34.32,32.02,40.36,0.318,✔️,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1279.0,+74/-74,15.128,0.005,34.35,30.6,42.95,0.021,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1278.0,+84/-119,15.031,0.036,34.41,29.27,371.59,0.102,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1224.0,+66/-92,15.173,0.0,36.86,35.23,1.86,0.046,✔️,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1218.0,+64/-73,15.174,0.0,37.12,35.45,40.36,0.028,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1206.0,+134/-218,18.046,0.042,37.63,32.31,1.52,0.115,✔️,0.0,False,CPU +43,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1162.0,+147/-233,24.034,0.009,39.45,35.34,43.64,0.085,✔️,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1140.0,+101/-190,20.729,0.011,40.28,31.54,513.06,78.02,✔️,0.0,False,CPU +45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1112.0,+166/-225,25.027,0.005,41.35,38.03,43.64,0.022,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+88/-208,22.275,0.0,44.96,44.23,0.12,0.014,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),982.0,+135/-219,26.73,0.0,45.44,43.9,0.22,0.024,✔️,0.0,False,CPU +48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),972.0,+202/-455,28.513,0.025,45.72,39.86,2.62,0.186,✔️,0.0,False,GPU +49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),918.0,+91/-188,30.352,0.0,47.05,46.26,0.24,0.036,➖,0.0,False,GPU +50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),916.0,+100/-247,24.142,0.0,47.09,46.63,0.1,0.021,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),883.0,+76/-126,27.794,0.0,47.81,47.49,513.06,2.568,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),514.0,+107/-398,45.591,0.0,52.27,52.24,0.11,0.087,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_medium/winrate_matrix.png.zip index 69b8832e2e72d5de7d29b42399c9760a8bb72b96..c4773ddbf4e42ad14311d24ea8b280aecc4165da 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0179b2e0997e900c319a3a5cf43b29f396a61a3a97b23e2d07c337dce82192e3 +oid sha256:58c56a938a79e3e723918c17ab82e3c3d3b8ff9cc40b8406dc8b50fc9e44b474 size 2102015 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 93cc3cbe3bcc99b383026c19fbdde2abd16f242e..e5be691b3307f0609e62f6e030307f1e72466738 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:09d6cd5b871ef3135cbae865a4a9965b387a85614651c1074b1a453df0ea4cbe +oid sha256:d40895726f44af8a1e29ac7e059ab41df14c611842441e5a9b78be9f3322053f size 430099 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip index 612fbd3d513e8028761884078a4a8fa8283bd1bf..d67bf57dd2f3de70218eecde530406006d933c15 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:58ec2732e734b0b9acc52e0f938e161f18521a103ec56468d54f0d93292b6060 -size 957912 +oid sha256:ab0ea4c367c7c476f7e280f5bf57a910fdc799f4cba65092cf45b083492ecbff +size 948765 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_small/tuning-impact-elo.png.zip index 428986fbf1cde7359601845a8bed08b80569218e..62a3f9b1cf07d4136d238b73fd4ebf12c1f89433 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c86e4d19ab70052c9bb6456343c526867fef05756fefd94833876c5013162365 +oid sha256:f64abe4468182633e34180de1e59952ebdfea38b4510064e0d19100c01498bdc size 124173 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_binary/datasets_small/website_leaderboard.csv index b97772301012df819761b01a357ab62e2af30869..f58e76d008b313ab70e5b241e779e59f560fd177 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_small/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_binary/datasets_small/website_leaderboard.csv @@ -1,53 +1,53 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1582.0,+110/-77,0.748,8.48,3.33,7.287,593.57,8.084,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1579.0,+98/-56,0.789,8.57,4.04,6.466,5.96,0.554,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1574.0,+142/-69,0.753,8.74,3.16,5.03,2052.28,8.252,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1547.0,+128/-57,0.713,9.74,3.78,6.337,2052.28,1.266,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1435.0,+81/-50,0.471,14.72,9.21,12.183,2950.72,15.595,0.0,False,GPU -5,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1425.0,+85/-56,0.434,15.23,8.35,12.33,2457.54,1.658,0.0,False,GPU -6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1415.0,+90/-69,0.417,15.75,8.92,11.866,1921.63,1.718,0.0,False,CPU -7,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1402.0,+82/-72,0.465,16.4,5.52,9.204,5351.68,268.579,0.0,False,GPU -8,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1390.0,+81/-66,0.467,17.07,7.72,14.126,317.91,1.983,0.0,False,GPU -9,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1370.0,+83/-73,0.385,18.2,5.58,10.014,5351.68,37.621,0.0,False,GPU -10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1366.0,+49/-38,0.31,18.4,12.57,14.198,512.57,1.18,0.0,False,CPU -11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1361.0,+85/-59,0.327,18.67,10.34,13.333,2457.54,0.178,0.0,False,GPU -12,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1333.0,+44/-34,0.237,20.3,14.26,14.644,512.57,0.243,0.0,False,CPU -13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1330.0,+80/-60,0.259,20.46,10.21,14.072,2950.72,1.225,0.0,False,GPU -14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1319.0,+88/-68,0.243,21.12,8.79,14.168,4806.04,0.512,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1310.0,+52/-43,0.224,21.68,14.73,15.257,1096.5,0.584,0.0,False,CPU -16,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1309.0,+64/-47,0.253,21.73,13.86,15.195,4.81,0.08,0.0,False,CPU -17,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1305.0,+53/-43,0.222,21.97,15.14,15.264,1096.5,0.074,0.0,False,CPU -18,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1297.0,+88/-76,0.324,22.45,7.6,13.006,51.83,43.615,0.0,False,GPU -19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1290.0,+51/-68,0.214,22.82,12.25,15.211,711.84,2.983,0.0,False,GPU -20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1289.0,+47/-43,0.203,22.93,15.54,15.673,693.49,1.398,0.0,False,CPU -21,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1278.0,+68/-61,0.262,23.55,12.39,16.117,10.08,0.152,0.0,False,GPU -22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1276.0,+114/-107,0.292,23.67,9.47,15.466,4806.04,8.461,0.0,False,GPU -23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1267.0,+48/-48,0.176,24.26,14.62,16.409,1634.54,0.422,0.0,False,CPU -24,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1265.0,+62/-65,0.21,24.38,11.3,16.741,14.37,0.334,0.0,False,GPU -25,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1264.0,+42/-33,0.149,24.39,16.25,16.038,693.49,0.194,0.0,False,CPU -26,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1263.0,+53/-61,0.182,24.48,12.54,16.17,711.84,0.162,0.0,False,GPU -27,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1259.0,+81/-56,0.167,24.72,16.57,15.336,2665.36,2.619,0.0,False,CPU -28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1243.0,+49/-50,0.134,25.68,14.64,16.945,1634.54,0.054,0.0,False,CPU -29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1224.0,+62/-72,0.163,26.88,15.4,17.235,644.55,5.974,0.0,False,CPU -30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1222.0,+52/-53,0.153,26.97,15.18,17.663,7.32,0.054,0.0,False,CPU -31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1221.0,+56/-50,0.123,27.02,16.05,16.483,10.89,2.249,0.0,False,GPU -32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1185.0,+72/-49,0.08,29.23,18.73,17.457,2665.36,0.143,0.0,False,CPU -33,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1174.0,+64/-77,0.071,29.9,17.46,18.602,644.55,0.398,0.0,False,CPU -34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1167.0,+63/-83,0.087,30.28,19.64,19.966,203.15,0.968,0.0,False,CPU -35,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1162.0,+61/-59,0.058,30.6,22.99,18.743,1.94,0.121,0.0,False,CPU -36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1132.0,+45/-51,0.033,32.32,27.15,19.177,2.44,0.121,0.0,False,CPU -37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1131.0,+59/-76,0.054,32.38,21.08,20.887,203.15,0.091,0.0,False,CPU -38,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1107.0,+54/-85,0.049,33.76,20.87,21.614,484.22,1.025,0.0,False,CPU -39,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1074.0,+108/-127,0.142,35.56,18.38,23.463,1.3,0.15,0.0,False,GPU -40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1073.0,+51/-82,0.024,35.64,21.78,22.235,484.22,0.086,0.0,False,CPU -41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1056.0,+61/-76,0.028,36.52,25.42,22.185,9.39,0.13,0.0,False,CPU -42,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1021.0,+89/-132,0.073,38.3,23.69,28.012,278.97,0.873,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1012.0,+76/-85,0.012,38.71,22.33,22.938,2.97,0.388,0.0,False,CPU -44,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+58/-111,0.018,39.3,30.97,25.166,0.42,0.041,0.0,False,CPU -45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,998.0,+96/-137,0.053,39.38,24.75,28.578,278.97,0.103,0.0,False,CPU -46,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,988.0,+75/-112,0.019,39.87,27.69,25.72,90.71,1.623,0.0,False,CPU -47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,986.0,+81/-105,0.027,39.92,31.85,25.768,3.71,0.919,0.0,False,GPU -48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,964.0,+79/-130,0.015,40.92,33.63,26.469,0.33,0.043,0.0,False,CPU -49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,952.0,+88/-162,0.034,41.4,27.1,30.627,1.48,0.139,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,921.0,+80/-144,0.001,42.66,27.85,29.993,90.71,0.125,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,616.0,+109/-176,0.0,49.97,44.93,43.126,0.22,0.02,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1582.0,+110/-77,7.287,0.748,8.48,3.33,593.57,8.084,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1579.0,+98/-56,6.466,0.789,8.57,4.04,5.96,0.554,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1574.0,+142/-69,5.03,0.753,8.74,3.16,2052.28,8.252,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1547.0,+128/-57,6.337,0.713,9.74,3.78,2052.28,1.266,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1435.0,+81/-50,12.183,0.471,14.72,9.21,2950.72,15.595,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1425.0,+85/-56,12.33,0.434,15.23,8.35,2457.54,1.658,✔️,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1415.0,+90/-69,11.866,0.417,15.75,8.92,1921.63,1.718,✔️,0.0,False,CPU +7,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1402.0,+82/-72,9.204,0.465,16.4,5.52,5351.68,268.579,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1390.0,+81/-66,14.126,0.467,17.07,7.72,317.91,1.983,➖,0.0,False,GPU +9,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1370.0,+83/-73,10.014,0.385,18.2,5.58,5351.68,37.621,✔️,0.0,False,GPU +10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1366.0,+49/-38,14.198,0.31,18.4,12.57,512.57,1.18,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1361.0,+85/-59,13.333,0.327,18.67,10.34,2457.54,0.178,✔️,0.0,False,GPU +12,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1333.0,+44/-34,14.644,0.237,20.3,14.26,512.57,0.243,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1330.0,+80/-60,14.072,0.259,20.46,10.21,2950.72,1.225,✔️,0.0,False,GPU +14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1319.0,+88/-68,14.168,0.243,21.12,8.79,4806.04,0.512,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1310.0,+52/-43,15.257,0.224,21.68,14.73,1096.5,0.584,✔️,0.0,False,CPU +16,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1309.0,+64/-47,15.195,0.253,21.73,13.86,4.81,0.08,✔️,0.0,False,CPU +17,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1305.0,+53/-43,15.264,0.222,21.97,15.14,1096.5,0.074,✔️,0.0,False,CPU +18,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1297.0,+88/-76,13.006,0.324,22.45,7.6,51.83,43.615,✔️,0.0,False,GPU +19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1290.0,+51/-68,15.211,0.214,22.82,12.25,711.84,2.983,✔️,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1289.0,+47/-43,15.673,0.203,22.93,15.54,693.49,1.398,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1278.0,+68/-61,16.117,0.262,23.55,12.39,10.08,0.152,✔️,0.0,False,GPU +22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1276.0,+114/-107,15.466,0.292,23.67,9.47,4806.04,8.461,✔️,0.0,False,GPU +23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1267.0,+48/-48,16.409,0.176,24.26,14.62,1634.54,0.422,✔️,0.0,False,CPU +24,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1265.0,+62/-65,16.741,0.21,24.38,11.3,14.37,0.334,✔️,0.0,False,GPU +25,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1264.0,+42/-33,16.038,0.149,24.39,16.25,693.49,0.194,✔️,0.0,False,CPU +26,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1263.0,+53/-61,16.17,0.182,24.48,12.54,711.84,0.162,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1259.0,+81/-56,15.336,0.167,24.72,16.57,2665.36,2.619,✔️,0.0,False,CPU +28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1243.0,+49/-50,16.945,0.134,25.68,14.64,1634.54,0.054,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1224.0,+62/-72,17.235,0.163,26.88,15.4,644.55,5.974,✔️,0.0,False,CPU +30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1222.0,+52/-53,17.663,0.153,26.97,15.18,7.32,0.054,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1221.0,+56/-50,16.483,0.123,27.02,16.05,10.89,2.249,✔️,0.0,False,GPU +32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1185.0,+72/-49,17.457,0.08,29.23,18.73,2665.36,0.143,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1174.0,+64/-77,18.602,0.071,29.9,17.46,644.55,0.398,✔️,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1167.0,+63/-83,19.966,0.087,30.28,19.64,203.15,0.968,✔️,0.0,False,CPU +35,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1162.0,+61/-59,18.743,0.058,30.6,22.99,1.94,0.121,✔️,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1132.0,+45/-51,19.177,0.033,32.32,27.15,2.44,0.121,✔️,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1131.0,+59/-76,20.887,0.054,32.38,21.08,203.15,0.091,✔️,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1107.0,+54/-85,21.614,0.049,33.76,20.87,484.22,1.025,✔️,0.0,False,CPU +39,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1074.0,+108/-127,23.463,0.142,35.56,18.38,1.3,0.15,➖,0.0,False,GPU +40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1073.0,+51/-82,22.235,0.024,35.64,21.78,484.22,0.086,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1056.0,+61/-76,22.185,0.028,36.52,25.42,9.39,0.13,✔️,0.0,False,CPU +42,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1021.0,+89/-132,28.012,0.073,38.3,23.69,278.97,0.873,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1012.0,+76/-85,22.938,0.012,38.71,22.33,2.97,0.388,✔️,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+58/-111,25.166,0.018,39.3,30.97,0.42,0.041,✔️,0.0,False,CPU +45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),998.0,+96/-137,28.578,0.053,39.38,24.75,278.97,0.103,✔️,0.0,False,CPU +46,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),988.0,+75/-112,25.72,0.019,39.87,27.69,90.71,1.623,✔️,0.0,False,CPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),986.0,+81/-105,25.768,0.027,39.92,31.85,3.71,0.919,✔️,0.0,False,GPU +48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),964.0,+79/-130,26.469,0.015,40.92,33.63,0.33,0.043,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),952.0,+88/-162,30.627,0.034,41.4,27.1,1.48,0.139,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),921.0,+80/-144,29.993,0.001,42.66,27.85,90.71,0.125,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),616.0,+109/-176,43.126,0.0,49.97,44.93,0.22,0.02,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_small/winrate_matrix.png.zip index fd87c7ee914de6db8defa5e91fba1e6ef61e1eb1..925697745b0a416fe30abcfef3e5ed7d44affdbe 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:78172565bb2582fdac1b2d466832f1a9afe26356d22bc415349ffaf92aca2f6f +oid sha256:4e57ff1a51e7be9c3d7151b16069c74a1f7f9e3f496c85bae2adec08d1e77971 size 2084457 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 6860dde1bdb4ae69c419d83bf083e0bf26c67ca7..decb6cd3cbfff460cf1890b30f9a8dcead7475ca 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:33786548e598a47b40a21281a191ef04bfcbc56650372b56c5483527bb4cced6 +oid sha256:fc63af627d7d502e12dd05fc5f18682e1b650cb18c1e0bb177b285928b9554b3 size 483576 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip index 871469bbcacbe460b2dbc9a3850e8c3ec089eb90..441ddcb6d8b81460ae65c8af6325fea082ee4d86 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:607718a2189859c597b085474b19b7604dd69b9214b9368fe8b78d4767e0c1e8 -size 1038997 +oid sha256:b8727986cd9b18c958c81b0a916b643bec00333f9951d32fa6a0f2dd174260bc +size 1037072 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip index 1a325393f01648fd856d7db66461049ba96d1939..535c74cbf4aba51654e4e5ee8fa87ffcd3383773 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a98a00b37dd6733cac15b4511aff08f6d130dc899b4406659209decbb2889a1c +oid sha256:81d0e1e01312078f508feaa69219e0a44ff430aa9dc161d2f04a568393e1a8a2 size 131179 diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/website_leaderboard.csv index f3f6c9e65b8ce5c20388b697463d899b2f1ab624..27e234f4851f3594c157236e6b1e39eca32dae81 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1600.0,+91/-54,0.729,10.36,5.33,7.184,5.52,0.529,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1588.0,+113/-69,0.685,10.87,4.46,5.826,2046.25,8.114,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1584.0,+88/-56,0.662,11.04,4.67,8.183,592.15,8.082,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1569.0,+104/-64,0.657,11.74,4.79,7.089,2046.25,1.254,0.0,False,GPU -4,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1510.0,+92/-60,0.585,14.67,5.88,10.733,144.27,3.772,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1497.0,+79/-54,0.512,15.33,6.23,10.68,3445.6,51.274,0.0,False,GPU -6,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1486.0,+98/-79,0.565,15.96,5.68,11.297,3.76,0.527,0.0,False,GPU -7,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1479.0,+79/-67,0.493,16.35,5.93,10.213,8.06,1.714,0.0,False,GPU -8,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1442.0,+65/-50,0.385,18.49,11.6,13.145,2879.46,14.626,0.0,False,GPU -9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1438.0,+75/-59,0.362,18.78,9.99,13.186,2379.18,1.595,0.0,False,GPU -10,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1430.0,+69/-59,0.406,19.25,9.32,14.423,316.0,1.931,0.0,False,GPU -11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1419.0,+76/-58,0.339,19.93,11.82,12.9,1828.68,1.648,0.0,False,CPU -12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1419.0,+84/-68,0.391,19.94,6.64,10.011,5319.2,249.314,0.0,False,GPU -13,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1415.0,+108/-75,0.407,20.2,6.42,13.416,3445.6,0.995,0.0,False,GPU -14,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1399.0,+104/-71,0.361,21.18,6.68,10.637,5319.2,29.488,0.0,False,GPU -15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1381.0,+86/-58,0.273,22.36,12.32,14.163,2379.18,0.176,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1376.0,+45/-44,0.233,22.67,15.32,15.222,449.53,1.149,0.0,False,CPU -17,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1372.0,+96/-85,0.405,22.91,8.63,15.503,3.73,0.488,0.0,False,GPU -18,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1352.0,+65/-52,0.22,24.25,12.67,14.896,2879.46,0.944,0.0,False,GPU -19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1347.0,+63/-58,0.225,24.59,10.35,14.824,4575.49,0.51,0.0,False,GPU -20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1347.0,+38/-37,0.17,24.6,17.27,15.649,449.53,0.215,0.0,False,CPU -21,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1326.0,+105/-79,0.303,25.96,8.98,13.65,51.44,43.709,0.0,False,GPU -22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1326.0,+61/-49,0.189,25.97,16.69,16.157,4.7,0.08,0.0,False,CPU -23,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1323.0,+56/-53,0.151,26.23,17.99,16.287,1081.93,0.556,0.0,False,CPU -24,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1316.0,+56/-45,0.159,26.64,19.09,16.312,1081.93,0.074,0.0,False,CPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1308.0,+67/-67,0.219,27.19,14.07,16.789,9.81,0.146,0.0,False,GPU -26,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1302.0,+53/-58,0.162,27.58,15.63,16.244,592.34,2.767,0.0,False,GPU -27,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1298.0,+52/-52,0.138,27.86,19.56,16.778,645.57,1.032,0.0,False,CPU -28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1295.0,+66/-71,0.174,28.07,13.53,17.352,14.36,0.311,0.0,False,GPU -29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1292.0,+50/-50,0.129,28.32,17.46,17.207,1618.46,0.415,0.0,False,CPU -30,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1290.0,+97/-103,0.254,28.39,11.16,16.484,4575.49,8.396,0.0,False,GPU -31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1286.0,+69/-58,0.14,28.68,19.64,15.992,2576.18,2.415,0.0,False,CPU -32,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1280.0,+54/-58,0.127,29.14,15.16,17.188,592.34,0.146,0.0,False,GPU -33,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1275.0,+42/-42,0.084,29.47,19.6,17.137,645.57,0.172,0.0,False,CPU -34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1271.0,+52/-55,0.1,29.74,16.79,17.71,1618.46,0.054,0.0,False,CPU -35,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1254.0,+54/-58,0.115,30.88,16.81,18.247,6.86,0.052,0.0,False,CPU -36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1253.0,+67/-78,0.143,30.93,17.0,17.887,618.9,5.468,0.0,False,CPU -37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1250.0,+55/-56,0.088,31.15,18.15,17.097,10.47,1.987,0.0,False,GPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1213.0,+63/-50,0.062,33.59,21.56,17.997,2576.18,0.136,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1206.0,+67/-74,0.058,34.09,19.4,19.212,618.9,0.352,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1183.0,+84/-72,0.067,35.56,23.19,21.088,199.83,0.865,0.0,False,CPU -41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1172.0,+64/-60,0.025,36.23,29.08,19.888,1.77,0.119,0.0,False,CPU -42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1150.0,+78/-78,0.04,37.6,25.09,21.933,199.83,0.086,0.0,False,CPU -43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1144.0,+40/-43,0.018,38.0,32.13,20.296,2.2,0.12,0.0,False,CPU -44,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1110.0,+95/-131,0.115,40.03,21.41,23.531,1.24,0.143,0.0,False,GPU -45,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1108.0,+65/-85,0.02,40.14,26.17,22.968,440.56,0.933,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1086.0,+75/-74,0.018,41.44,29.26,22.66,8.39,0.127,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1078.0,+52/-79,0.004,41.88,24.91,23.518,440.56,0.085,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1057.0,+98/-123,0.071,43.04,26.02,28.113,273.27,0.715,0.0,False,CPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1043.0,+78/-87,0.0,43.76,24.97,23.546,2.84,0.381,0.0,False,CPU -50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1034.0,+94/-126,0.047,44.19,27.79,28.584,273.27,0.098,0.0,False,CPU -51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1020.0,+83/-108,0.02,44.93,30.67,26.329,89.68,1.55,0.0,False,CPU -52,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1002.0,+98/-112,0.011,45.79,36.55,26.937,2.96,0.796,0.0,False,GPU -53,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+63/-105,0.003,45.87,37.86,26.578,0.42,0.04,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,989.0,+102/-139,0.03,46.39,30.04,30.404,1.44,0.138,0.0,False,CPU -55,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,976.0,+93/-114,0.012,46.96,38.72,27.729,0.31,0.042,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,953.0,+86/-138,0.001,47.95,30.7,30.595,89.68,0.12,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,639.0,+130/-215,0.0,55.9,49.79,43.912,0.21,0.019,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1600.0,+91/-54,7.184,0.729,10.36,5.33,5.52,0.529,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1588.0,+113/-69,5.826,0.685,10.87,4.46,2046.25,8.114,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1584.0,+88/-56,8.183,0.662,11.04,4.67,592.15,8.082,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1569.0,+104/-64,7.089,0.657,11.74,4.79,2046.25,1.254,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1510.0,+92/-60,10.733,0.585,14.67,5.88,144.27,3.772,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1497.0,+79/-54,10.68,0.512,15.33,6.23,3445.6,51.274,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1486.0,+98/-79,11.297,0.565,15.96,5.68,3.76,0.527,➖,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1479.0,+79/-67,10.213,0.493,16.35,5.93,8.06,1.714,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1442.0,+65/-50,13.145,0.385,18.49,11.6,2879.46,14.626,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1438.0,+75/-59,13.186,0.362,18.78,9.99,2379.18,1.595,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1430.0,+69/-59,14.423,0.406,19.25,9.32,316.0,1.931,➖,0.0,False,GPU +11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1419.0,+76/-58,12.9,0.339,19.93,11.82,1828.68,1.648,✔️,0.0,False,CPU +12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1419.0,+84/-68,10.011,0.391,19.94,6.64,5319.2,249.314,✔️,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1415.0,+108/-75,13.416,0.407,20.2,6.42,3445.6,0.995,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1399.0,+104/-71,10.637,0.361,21.18,6.68,5319.2,29.488,✔️,0.0,False,GPU +15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1381.0,+86/-58,14.163,0.273,22.36,12.32,2379.18,0.176,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1376.0,+45/-44,15.222,0.233,22.67,15.32,449.53,1.149,✔️,0.0,False,CPU +17,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1372.0,+96/-85,15.503,0.405,22.91,8.63,3.73,0.488,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1352.0,+65/-52,14.896,0.22,24.25,12.67,2879.46,0.944,✔️,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1347.0,+63/-58,14.824,0.225,24.59,10.35,4575.49,0.51,✔️,0.0,False,GPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1347.0,+38/-37,15.649,0.17,24.6,17.27,449.53,0.215,✔️,0.0,False,CPU +21,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1326.0,+105/-79,13.65,0.303,25.96,8.98,51.44,43.709,✔️,0.0,False,GPU +22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1326.0,+61/-49,16.157,0.189,25.97,16.69,4.7,0.08,✔️,0.0,False,CPU +23,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1323.0,+56/-53,16.287,0.151,26.23,17.99,1081.93,0.556,✔️,0.0,False,CPU +24,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1316.0,+56/-45,16.312,0.159,26.64,19.09,1081.93,0.074,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1308.0,+67/-67,16.789,0.219,27.19,14.07,9.81,0.146,✔️,0.0,False,GPU +26,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1302.0,+53/-58,16.244,0.162,27.58,15.63,592.34,2.767,✔️,0.0,False,GPU +27,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1298.0,+52/-52,16.778,0.138,27.86,19.56,645.57,1.032,✔️,0.0,False,CPU +28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1295.0,+66/-71,17.352,0.174,28.07,13.53,14.36,0.311,✔️,0.0,False,GPU +29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1292.0,+50/-50,17.207,0.129,28.32,17.46,1618.46,0.415,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1290.0,+97/-103,16.484,0.254,28.39,11.16,4575.49,8.396,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1286.0,+69/-58,15.992,0.14,28.68,19.64,2576.18,2.415,✔️,0.0,False,CPU +32,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1280.0,+54/-58,17.188,0.127,29.14,15.16,592.34,0.146,✔️,0.0,False,GPU +33,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1275.0,+42/-42,17.137,0.084,29.47,19.6,645.57,0.172,✔️,0.0,False,CPU +34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1271.0,+52/-55,17.71,0.1,29.74,16.79,1618.46,0.054,✔️,0.0,False,CPU +35,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1254.0,+54/-58,18.247,0.115,30.88,16.81,6.86,0.052,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1253.0,+67/-78,17.887,0.143,30.93,17.0,618.9,5.468,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1250.0,+55/-56,17.097,0.088,31.15,18.15,10.47,1.987,✔️,0.0,False,GPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1213.0,+63/-50,17.997,0.062,33.59,21.56,2576.18,0.136,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1206.0,+67/-74,19.212,0.058,34.09,19.4,618.9,0.352,✔️,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1183.0,+84/-72,21.088,0.067,35.56,23.19,199.83,0.865,✔️,0.0,False,CPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1172.0,+64/-60,19.888,0.025,36.23,29.08,1.77,0.119,✔️,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1150.0,+78/-78,21.933,0.04,37.6,25.09,199.83,0.086,✔️,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1144.0,+40/-43,20.296,0.018,38.0,32.13,2.2,0.12,✔️,0.0,False,CPU +44,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1110.0,+95/-131,23.531,0.115,40.03,21.41,1.24,0.143,➖,0.0,False,GPU +45,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1108.0,+65/-85,22.968,0.02,40.14,26.17,440.56,0.933,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1086.0,+75/-74,22.66,0.018,41.44,29.26,8.39,0.127,✔️,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1078.0,+52/-79,23.518,0.004,41.88,24.91,440.56,0.085,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1057.0,+98/-123,28.113,0.071,43.04,26.02,273.27,0.715,✔️,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1043.0,+78/-87,23.546,0.0,43.76,24.97,2.84,0.381,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1034.0,+94/-126,28.584,0.047,44.19,27.79,273.27,0.098,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1020.0,+83/-108,26.329,0.02,44.93,30.67,89.68,1.55,✔️,0.0,False,CPU +52,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1002.0,+98/-112,26.937,0.011,45.79,36.55,2.96,0.796,✔️,0.0,False,GPU +53,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+63/-105,26.578,0.003,45.87,37.86,0.42,0.04,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),989.0,+102/-139,30.404,0.03,46.39,30.04,1.44,0.138,✔️,0.0,False,CPU +55,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),976.0,+93/-114,27.729,0.012,46.96,38.72,0.31,0.042,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),953.0,+86/-138,30.595,0.001,47.95,30.7,89.68,0.12,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),639.0,+130/-215,43.912,0.0,55.9,49.79,0.21,0.019,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip index 638dd53bb30bfe8715f63867ac7b3878ec984073..b5366efe7d501801d000b176d670477926f0b280 100644 --- a/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f936028ebc0edda40cbeaa24cd8fa98995ab66cc1da7cc7a583e2ba79c96a503 +oid sha256:72ab1298a6f3165d254d76a2c5aa9e6ea2499200355cb437bfdcf786928aa9d8 size 2690375 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index d16e93ce4dee0ee950330748a3b999ba5486fe50..c47bf98234fa0788bb44e5fea02621e4587e1b94 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:41b2b68c08bf4c8b20b53c19cab3d3913cb96d16e8538fe0acd115cb8eae7eff +oid sha256:6bd7e126d5b12ad8d2b6f45ebf02c1452f74196f161c38c111ba598254da30cb size 439786 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip index b9cea6ba0bdd14469114dda09ac20e15282599a8..0e8129363e8b8a456e07db6b3aa3ad2563fcdca9 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:92a50fc0f408d31414b0fa5d7c12d492410c765c812b7fbae452aa4a323e4984 -size 1001503 +oid sha256:e55ea134c309bccc8dcd6084b333f316ab1c52773176fb266c6c7cb5e741b316 +size 954824 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_all/tuning-impact-elo.png.zip index 7c8e1b33e973f467b37404544980d59d4a6940ed..1543dc1c1f199869ee441354203147e76116fb15 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:caeba71b3ca8c6b5e75d780089f20ceb917a3d1c9e4a0c76fa86bc6b8e83c1e3 +oid sha256:7cdd342b55a7bf109dbe983c9c4f7bcc6c822faea1908ad566d313d241d44775 size 113343 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_classification/datasets_all/website_leaderboard.csv index af4dbd30f40bf40a7d9134fbf4ffef2bd341430a..0607ce1a12a2a33c77b04fd6a4152dc88e12db83 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_all/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_classification/datasets_all/website_leaderboard.csv @@ -1,53 +1,53 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1673.0,+76/-63,0.809,7.05,3.08,6.447,545.23,6.469,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1593.0,+81/-76,0.709,9.74,3.16,5.921,2046.25,8.976,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1566.0,+79/-70,0.671,10.79,3.98,6.967,2046.25,1.329,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1550.0,+72/-69,0.683,11.47,4.59,7.392,5.76,0.794,0.0,False,GPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1516.0,+78/-58,0.533,13.01,5.2,10.128,1684.49,1.97,0.0,False,CPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1500.0,+62/-54,0.508,13.74,8.2,10.684,2879.46,12.49,0.0,False,GPU -6,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1472.0,+76/-48,0.449,15.12,7.64,12.019,2466.21,1.502,0.0,False,GPU -7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1440.0,+48/-36,0.374,16.81,11.12,13.132,382.05,1.488,0.0,False,CPU -8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1421.0,+77/-56,0.428,17.83,6.22,11.076,4940.61,307.751,0.0,False,GPU -9,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1417.0,+73/-58,0.367,18.06,8.85,12.922,2466.21,0.176,0.0,False,GPU -10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1416.0,+58/-47,0.328,18.11,9.33,12.344,2879.46,0.598,0.0,False,GPU -11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1415.0,+54/-46,0.349,18.15,10.54,12.625,1372.94,0.556,0.0,False,CPU -12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1406.0,+58/-44,0.338,18.65,10.04,12.832,1372.94,0.074,0.0,False,CPU -13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1396.0,+41/-34,0.279,19.26,13.36,13.856,382.05,0.254,0.0,False,CPU -14,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1386.0,+53/-47,0.305,19.81,11.85,13.978,685.87,1.455,0.0,False,CPU -15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1383.0,+50/-46,0.325,19.97,10.59,13.472,5.72,0.076,0.0,False,CPU -16,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1376.0,+57/-54,0.354,20.4,10.32,15.277,282.72,1.886,0.0,False,GPU -17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1371.0,+89/-82,0.345,20.64,9.1,13.51,4879.89,8.744,0.0,False,GPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1370.0,+60/-41,0.238,20.74,9.81,13.555,4879.89,0.525,0.0,False,GPU -19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1359.0,+47/-44,0.245,21.35,14.58,14.358,685.87,0.205,0.0,False,CPU -20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1358.0,+67/-58,0.307,21.4,7.33,12.802,4940.61,41.606,0.0,False,GPU -21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1337.0,+44/-45,0.226,22.61,13.17,14.984,934.1,3.063,0.0,False,GPU -22,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1318.0,+43/-36,0.183,23.73,15.62,16.035,2686.11,0.47,0.0,False,CPU -23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1315.0,+61/-54,0.254,23.93,13.32,15.709,10.21,0.138,0.0,False,GPU -24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1314.0,+45/-43,0.184,23.95,16.27,14.908,2389.22,2.158,0.0,False,CPU -25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1294.0,+51/-54,0.175,25.17,14.33,16.495,934.1,0.169,0.0,False,GPU -26,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1280.0,+45/-38,0.126,25.99,16.88,16.759,2686.11,0.054,0.0,False,CPU -27,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1277.0,+67/-64,0.238,26.17,9.08,16.109,49.21,43.824,0.0,False,GPU -28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1266.0,+61/-60,0.169,26.82,15.31,17.277,618.9,4.766,0.0,False,CPU -29,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1252.0,+44/-46,0.128,27.62,18.04,17.998,6.86,0.057,0.0,False,CPU -30,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1252.0,+46/-45,0.125,27.62,15.26,17.837,14.78,0.346,0.0,False,GPU -31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1243.0,+40/-37,0.087,28.19,19.4,17.108,10.47,1.707,0.0,False,GPU -32,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1232.0,+48/-52,0.106,28.81,19.48,17.195,1.77,0.117,0.0,False,CPU -33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1228.0,+47/-37,0.081,29.06,20.0,17.147,2389.22,0.152,0.0,False,CPU -34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1214.0,+52/-52,0.1,29.84,19.19,18.724,189.76,0.743,0.0,False,CPU -35,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1195.0,+53/-59,0.074,30.91,19.12,19.123,618.9,0.298,0.0,False,CPU -36,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1188.0,+64/-57,0.09,31.32,18.57,19.635,323.74,0.743,0.0,False,CPU -37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1188.0,+48/-38,0.065,31.32,26.28,18.263,1.79,0.12,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1172.0,+54/-54,0.073,32.24,19.76,19.957,189.76,0.079,0.0,False,CPU -39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1150.0,+60/-56,0.062,33.44,18.14,20.504,323.74,0.076,0.0,False,CPU -40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1085.0,+45/-56,0.015,36.8,28.14,21.856,6.83,0.147,0.0,False,CPU -41,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1041.0,+86/-88,0.043,38.9,28.09,29.56,252.58,0.528,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1038.0,+63/-67,0.017,39.03,26.67,24.207,2.91,0.368,0.0,False,CPU -43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1019.0,+67/-93,0.02,39.87,26.93,27.374,139.84,3.615,0.0,False,CPU -44,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1013.0,+86/-103,0.079,40.12,25.02,28.483,1.11,0.189,0.0,False,GPU -45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1010.0,+87/-92,0.031,40.23,29.46,30.319,252.58,0.089,0.0,False,CPU -46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1005.0,+73/-85,0.032,40.44,33.02,27.121,3.61,0.939,0.0,False,GPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+58/-63,0.01,40.67,34.46,26.637,0.38,0.037,0.0,False,CPU -48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,947.0,+68/-72,0.008,42.72,37.34,28.882,0.25,0.041,0.0,False,CPU -49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,927.0,+94/-111,0.019,43.43,32.94,33.16,1.29,0.131,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,900.0,+84/-117,0.004,44.32,32.53,33.585,139.84,0.193,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,563.0,+111/-112,0.0,50.66,47.64,49.469,0.18,0.029,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1673.0,+76/-63,6.447,0.809,7.05,3.08,545.23,6.469,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1593.0,+81/-76,5.921,0.709,9.74,3.16,2046.25,8.976,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1566.0,+79/-70,6.967,0.671,10.79,3.98,2046.25,1.329,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1550.0,+72/-69,7.392,0.683,11.47,4.59,5.76,0.794,✔️,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1516.0,+78/-58,10.128,0.533,13.01,5.2,1684.49,1.97,✔️,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1500.0,+62/-54,10.684,0.508,13.74,8.2,2879.46,12.49,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1472.0,+76/-48,12.019,0.449,15.12,7.64,2466.21,1.502,✔️,0.0,False,GPU +7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1440.0,+48/-36,13.132,0.374,16.81,11.12,382.05,1.488,✔️,0.0,False,CPU +8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1421.0,+77/-56,11.076,0.428,17.83,6.22,4940.61,307.751,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1417.0,+73/-58,12.922,0.367,18.06,8.85,2466.21,0.176,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1416.0,+58/-47,12.344,0.328,18.11,9.33,2879.46,0.598,✔️,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1415.0,+54/-46,12.625,0.349,18.15,10.54,1372.94,0.556,✔️,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1406.0,+58/-44,12.832,0.338,18.65,10.04,1372.94,0.074,✔️,0.0,False,CPU +13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1396.0,+41/-34,13.856,0.279,19.26,13.36,382.05,0.254,✔️,0.0,False,CPU +14,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1386.0,+53/-47,13.978,0.305,19.81,11.85,685.87,1.455,✔️,0.0,False,CPU +15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1383.0,+50/-46,13.472,0.325,19.97,10.59,5.72,0.076,✔️,0.0,False,CPU +16,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1376.0,+57/-54,15.277,0.354,20.4,10.32,282.72,1.886,➖,0.0,False,GPU +17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1371.0,+89/-82,13.51,0.345,20.64,9.1,4879.89,8.744,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1370.0,+60/-41,13.555,0.238,20.74,9.81,4879.89,0.525,✔️,0.0,False,GPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1359.0,+47/-44,14.358,0.245,21.35,14.58,685.87,0.205,✔️,0.0,False,CPU +20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1358.0,+67/-58,12.802,0.307,21.4,7.33,4940.61,41.606,✔️,0.0,False,GPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1337.0,+44/-45,14.984,0.226,22.61,13.17,934.1,3.063,✔️,0.0,False,GPU +22,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1318.0,+43/-36,16.035,0.183,23.73,15.62,2686.11,0.47,✔️,0.0,False,CPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1315.0,+61/-54,15.709,0.254,23.93,13.32,10.21,0.138,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1314.0,+45/-43,14.908,0.184,23.95,16.27,2389.22,2.158,✔️,0.0,False,CPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1294.0,+51/-54,16.495,0.175,25.17,14.33,934.1,0.169,✔️,0.0,False,GPU +26,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1280.0,+45/-38,16.759,0.126,25.99,16.88,2686.11,0.054,✔️,0.0,False,CPU +27,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1277.0,+67/-64,16.109,0.238,26.17,9.08,49.21,43.824,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1266.0,+61/-60,17.277,0.169,26.82,15.31,618.9,4.766,✔️,0.0,False,CPU +29,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1252.0,+44/-46,17.998,0.128,27.62,18.04,6.86,0.057,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1252.0,+46/-45,17.837,0.125,27.62,15.26,14.78,0.346,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1243.0,+40/-37,17.108,0.087,28.19,19.4,10.47,1.707,✔️,0.0,False,GPU +32,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1232.0,+48/-52,17.195,0.106,28.81,19.48,1.77,0.117,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1228.0,+47/-37,17.147,0.081,29.06,20.0,2389.22,0.152,✔️,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1214.0,+52/-52,18.724,0.1,29.84,19.19,189.76,0.743,✔️,0.0,False,CPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1195.0,+53/-59,19.123,0.074,30.91,19.12,618.9,0.298,✔️,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1188.0,+64/-57,19.635,0.09,31.32,18.57,323.74,0.743,✔️,0.0,False,CPU +37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1188.0,+48/-38,18.263,0.065,31.32,26.28,1.79,0.12,✔️,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1172.0,+54/-54,19.957,0.073,32.24,19.76,189.76,0.079,✔️,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1150.0,+60/-56,20.504,0.062,33.44,18.14,323.74,0.076,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1085.0,+45/-56,21.856,0.015,36.8,28.14,6.83,0.147,✔️,0.0,False,CPU +41,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1041.0,+86/-88,29.56,0.043,38.9,28.09,252.58,0.528,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1038.0,+63/-67,24.207,0.017,39.03,26.67,2.91,0.368,✔️,0.0,False,CPU +43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1019.0,+67/-93,27.374,0.02,39.87,26.93,139.84,3.615,✔️,0.0,False,CPU +44,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1013.0,+86/-103,28.483,0.079,40.12,25.02,1.11,0.189,➖,0.0,False,GPU +45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1010.0,+87/-92,30.319,0.031,40.23,29.46,252.58,0.089,✔️,0.0,False,CPU +46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1005.0,+73/-85,27.121,0.032,40.44,33.02,3.61,0.939,✔️,0.0,False,GPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+58/-63,26.637,0.01,40.67,34.46,0.38,0.037,✔️,0.0,False,CPU +48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),947.0,+68/-72,28.882,0.008,42.72,37.34,0.25,0.041,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),927.0,+94/-111,33.16,0.019,43.43,32.94,1.29,0.131,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),900.0,+84/-117,33.585,0.004,44.32,32.53,139.84,0.193,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),563.0,+111/-112,49.469,0.0,50.66,47.64,0.18,0.029,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_all/winrate_matrix.png.zip index e5d0d2249c56bac1c47aca87a817601f75c263d3..9689db2383f16efea97dd4a44998dda025206909 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4412cc4e4acfe3d0b1f43a8c92229e9de6f20809b050633dee55777f88f1411e +oid sha256:597cf8a2debe07fef7194817772d424e9b90a8e5f88e729d87c6341999c84549 size 2041622 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index 0f594a75b803242960cb57b7b785cb7e85e60d62..0d69d1eb6040c27d5df279b04d4bb43269463529 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2a8347b99901bafd5d829f273bb4bf3c1c34b0e0eef183a023e5463dc2f256a9 +oid sha256:3de46251d70777fedf2009f664158f3a1cd46dacd611adc6a93e81b90ea28f65 size 420152 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip index 7a388d6ab1d2a7b031b6722990f9d15319650978..82d2c46b8e0567393e14501a1a72e241e9a880ae 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3d29fc7770ed6ace6d699448c49540fff92665fe6862340dbb2c44074380f024 -size 1007878 +oid sha256:1c48c1cc97b1dfd4312245ceb7067e74666b06c9186291e43ecc8d89924f64f4 +size 948975 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_medium/tuning-impact-elo.png.zip index 159822f78ae5431bc0c125d63164f795524581da..42d7e906f6eb60e18a313a886587491e44e981ca 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9706fe88435fe26d86b2b18584303159ab48ddcb5c3669d8a432028c24f3f583 +oid sha256:5be0ed1ffd3ea931c4655b13af19386480aeb6e9d5856f5840543c4b489bb95a size 118664 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_classification/datasets_medium/website_leaderboard.csv index dc0f99d32ca3dc3bcceb4ad363d94ff8522bb412..2286344fd351cbaf8cd052f34c36713535526adb 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_medium/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_classification/datasets_medium/website_leaderboard.csv @@ -1,54 +1,54 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1929.0,+143/-60,0.907,4.76,2.46,2.178,275.97,2.818,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1873.0,+172/-66,0.859,5.97,2.64,2.411,314.17,2.249,0.0,False,CPU -2,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1707.0,+129/-66,0.618,11.18,6.2,5.406,697.8,0.199,0.0,False,CPU -3,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1705.0,+141/-72,0.623,11.24,5.16,5.651,697.8,0.023,0.0,False,CPU -4,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1676.0,+124/-64,0.609,12.42,5.87,5.593,2.44,0.027,0.0,False,CPU -5,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1673.0,+138/-96,0.542,12.54,7.25,7.874,207.65,0.398,0.0,False,CPU -6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1669.0,+144/-74,0.529,12.7,8.34,8.132,93.21,0.758,0.0,False,CPU -7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1651.0,+120/-91,0.517,13.46,8.65,7.253,1624.54,1.763,0.0,False,GPU -8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1646.0,+218/-129,0.54,13.7,3.82,7.545,783.29,13.79,0.0,False,GPU -9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1626.0,+148/-132,0.53,14.59,7.55,7.556,2338.19,14.248,0.0,False,GPU -10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1604.0,+180/-118,0.462,15.6,6.08,8.633,1398.9,0.645,0.0,False,GPU -11,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1599.0,+111/-77,0.429,15.82,11.69,8.689,207.65,0.056,0.0,False,CPU -12,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1594.0,+205/-121,0.468,16.03,6.06,8.372,783.29,1.924,0.0,False,GPU -13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1567.0,+129/-90,0.338,17.34,9.49,8.727,1624.54,0.083,0.0,False,GPU -14,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1554.0,+167/-78,0.346,17.97,11.64,9.811,93.21,0.087,0.0,False,CPU -15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1554.0,+190/-148,0.412,17.98,6.83,9.345,1398.9,0.062,0.0,False,GPU -16,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1541.0,+154/-119,0.414,18.63,8.37,9.121,2.3,0.82,0.0,False,GPU -17,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1518.0,+166/-148,0.413,19.76,8.93,9.743,7.51,3.146,0.0,False,GPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1508.0,+114/-82,0.263,20.28,12.89,10.245,2338.19,0.54,0.0,False,GPU -19,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1486.0,+156/-141,0.326,21.43,8.09,9.963,4805.66,461.797,0.0,False,GPU -20,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1476.0,+89/-75,0.234,21.91,15.1,10.081,1169.3,1.442,0.0,False,CPU -21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1466.0,+146/-106,0.216,22.47,15.25,12.538,893.93,0.205,0.0,False,CPU -22,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1438.0,+156/-129,0.255,23.92,13.66,10.925,0.47,0.052,0.0,False,CPU -23,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1422.0,+131/-134,0.177,24.79,16.12,12.437,709.85,2.929,0.0,False,GPU -24,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1398.0,+160/-107,0.139,26.12,19.84,13.588,893.93,0.016,0.0,False,CPU -25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1388.0,+130/-132,0.146,26.66,20.31,14.508,709.85,0.19,0.0,False,GPU -26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1385.0,+149/-155,0.217,26.8,15.57,12.673,5.67,0.081,0.0,False,GPU -27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1359.0,+84/-82,0.069,28.18,24.73,13.533,6.98,0.233,0.0,False,GPU -28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1357.0,+96/-140,0.11,28.29,21.75,14.453,389.62,2.114,0.0,False,CPU -29,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1350.0,+138/-132,0.122,28.69,21.21,15.914,84.36,0.919,0.0,False,GPU -30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1345.0,+81/-84,0.073,28.91,23.58,12.262,1169.3,0.053,0.0,False,CPU -31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1342.0,+125/-102,0.084,29.11,25.2,15.622,2.09,0.019,0.0,False,CPU -32,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1329.0,+117/-121,0.078,29.75,19.44,13.696,4805.66,43.211,0.0,False,GPU -33,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1327.0,+118/-103,0.062,29.86,22.08,13.691,52.45,0.219,0.0,False,CPU -34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1294.0,+94/-111,0.062,31.52,28.6,13.707,0.28,0.033,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1289.0,+127/-116,0.063,31.78,23.78,14.607,52.45,0.021,0.0,False,CPU -36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1281.0,+79/-74,0.007,32.17,29.44,14.625,10.5,0.606,0.0,False,GPU -37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1262.0,+71/-88,0.025,33.1,30.65,13.747,38.42,0.32,0.0,False,CPU -38,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1249.0,+123/-152,0.032,33.75,21.52,17.564,46.9,45.623,0.0,False,GPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1218.0,+79/-144,0.032,35.18,30.19,17.632,389.62,0.112,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1203.0,+77/-86,0.011,35.87,33.94,15.534,38.42,0.028,0.0,False,CPU -41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1179.0,+62/-68,0.0,36.96,35.47,16.733,1.87,0.046,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1145.0,+110/-215,0.038,38.39,33.26,21.37,1.61,0.112,0.0,False,CPU -43,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1103.0,+142/-227,0.008,40.08,36.17,26.799,43.46,0.087,0.0,False,CPU -44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1070.0,+93/-195,0.01,41.31,32.77,24.389,331.68,44.947,0.0,False,CPU -45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1051.0,+151/-241,0.005,41.96,38.8,28.022,43.46,0.021,0.0,False,CPU -46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+102/-183,0.0,43.65,42.56,22.382,0.12,0.013,0.0,False,CPU -47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,927.0,+140/-232,0.0,45.73,44.31,29.594,0.2,0.023,0.0,False,CPU -48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,916.0,+205/-468,0.022,46.02,40.6,31.186,2.8,0.224,0.0,False,GPU -49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,884.0,+91/-212,0.0,46.79,46.36,26.748,0.09,0.019,0.0,False,CPU -50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,861.0,+107/-173,0.0,47.32,46.59,32.966,0.2,0.033,0.0,False,GPU -51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,816.0,+64/-150,0.0,48.23,47.9,32.095,331.68,1.626,0.0,False,CPU -52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,451.0,+130/-393,0.0,52.34,52.31,49.299,0.12,0.101,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1929.0,+143/-60,2.178,0.907,4.76,2.46,275.97,2.818,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1873.0,+172/-66,2.411,0.859,5.97,2.64,314.17,2.249,✔️,0.0,False,CPU +2,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1707.0,+129/-66,5.406,0.618,11.18,6.2,697.8,0.199,✔️,0.0,False,CPU +3,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1705.0,+141/-72,5.651,0.623,11.24,5.16,697.8,0.023,✔️,0.0,False,CPU +4,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1676.0,+124/-64,5.593,0.609,12.42,5.87,2.44,0.027,✔️,0.0,False,CPU +5,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1673.0,+138/-96,7.874,0.542,12.54,7.25,207.65,0.398,✔️,0.0,False,CPU +6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1669.0,+144/-74,8.132,0.529,12.7,8.34,93.21,0.758,✔️,0.0,False,CPU +7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1651.0,+120/-91,7.253,0.517,13.46,8.65,1624.54,1.763,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1646.0,+218/-129,7.545,0.54,13.7,3.82,783.29,13.79,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1626.0,+148/-132,7.556,0.53,14.59,7.55,2338.19,14.248,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1604.0,+180/-118,8.633,0.462,15.6,6.08,1398.9,0.645,✔️,0.0,False,GPU +11,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1599.0,+111/-77,8.689,0.429,15.82,11.69,207.65,0.056,✔️,0.0,False,CPU +12,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1594.0,+205/-121,8.372,0.468,16.03,6.06,783.29,1.924,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1567.0,+129/-90,8.727,0.338,17.34,9.49,1624.54,0.083,✔️,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1554.0,+167/-78,9.811,0.346,17.97,11.64,93.21,0.087,✔️,0.0,False,CPU +15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1554.0,+190/-148,9.345,0.412,17.98,6.83,1398.9,0.062,✔️,0.0,False,GPU +16,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1541.0,+154/-119,9.121,0.414,18.63,8.37,2.3,0.82,✔️,0.0,False,GPU +17,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1518.0,+166/-148,9.743,0.413,19.76,8.93,7.51,3.146,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1508.0,+114/-82,10.245,0.263,20.28,12.89,2338.19,0.54,✔️,0.0,False,GPU +19,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1486.0,+156/-141,9.963,0.326,21.43,8.09,4805.66,461.797,✔️,0.0,False,GPU +20,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1476.0,+89/-75,10.081,0.234,21.91,15.1,1169.3,1.442,✔️,0.0,False,CPU +21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1466.0,+146/-106,12.538,0.216,22.47,15.25,893.93,0.205,✔️,0.0,False,CPU +22,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1438.0,+156/-129,10.925,0.255,23.92,13.66,0.47,0.052,✔️,0.0,False,CPU +23,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1422.0,+131/-134,12.437,0.177,24.79,16.12,709.85,2.929,✔️,0.0,False,GPU +24,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1398.0,+160/-107,13.588,0.139,26.12,19.84,893.93,0.016,✔️,0.0,False,CPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1388.0,+130/-132,14.508,0.146,26.66,20.31,709.85,0.19,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1385.0,+149/-155,12.673,0.217,26.8,15.57,5.67,0.081,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1359.0,+84/-82,13.533,0.069,28.18,24.73,6.98,0.233,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1357.0,+96/-140,14.453,0.11,28.29,21.75,389.62,2.114,✔️,0.0,False,CPU +29,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1350.0,+138/-132,15.914,0.122,28.69,21.21,84.36,0.919,➖,0.0,False,GPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1345.0,+81/-84,12.262,0.073,28.91,23.58,1169.3,0.053,✔️,0.0,False,CPU +31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1342.0,+125/-102,15.622,0.084,29.11,25.2,2.09,0.019,✔️,0.0,False,CPU +32,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1329.0,+117/-121,13.696,0.078,29.75,19.44,4805.66,43.211,✔️,0.0,False,GPU +33,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1327.0,+118/-103,13.691,0.062,29.86,22.08,52.45,0.219,✔️,0.0,False,CPU +34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1294.0,+94/-111,13.707,0.062,31.52,28.6,0.28,0.033,✔️,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1289.0,+127/-116,14.607,0.063,31.78,23.78,52.45,0.021,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1281.0,+79/-74,14.625,0.007,32.17,29.44,10.5,0.606,✔️,0.0,False,GPU +37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1262.0,+71/-88,13.747,0.025,33.1,30.65,38.42,0.32,✔️,0.0,False,CPU +38,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1249.0,+123/-152,17.564,0.032,33.75,21.52,46.9,45.623,✔️,0.0,False,GPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1218.0,+79/-144,17.632,0.032,35.18,30.19,389.62,0.112,✔️,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1203.0,+77/-86,15.534,0.011,35.87,33.94,38.42,0.028,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1179.0,+62/-68,16.733,0.0,36.96,35.47,1.87,0.046,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1145.0,+110/-215,21.37,0.038,38.39,33.26,1.61,0.112,✔️,0.0,False,CPU +43,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1103.0,+142/-227,26.799,0.008,40.08,36.17,43.46,0.087,✔️,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1070.0,+93/-195,24.389,0.01,41.31,32.77,331.68,44.947,✔️,0.0,False,CPU +45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1051.0,+151/-241,28.022,0.005,41.96,38.8,43.46,0.021,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+102/-183,22.382,0.0,43.65,42.56,0.12,0.013,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),927.0,+140/-232,29.594,0.0,45.73,44.31,0.2,0.023,✔️,0.0,False,CPU +48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),916.0,+205/-468,31.186,0.022,46.02,40.6,2.8,0.224,✔️,0.0,False,GPU +49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),884.0,+91/-212,26.748,0.0,46.79,46.36,0.09,0.019,✔️,0.0,False,CPU +50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),861.0,+107/-173,32.966,0.0,47.32,46.59,0.2,0.033,➖,0.0,False,GPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),816.0,+64/-150,32.095,0.0,48.23,47.9,331.68,1.626,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),451.0,+130/-393,49.299,0.0,52.34,52.31,0.12,0.101,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_medium/winrate_matrix.png.zip index f0383e4fc917d94175f889dbb3c639c90230e776..d4409a41585bf3b0274c232c4c84c4b432c88c4e 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7844015ae3a527022ddb90b51b6317a42c30db6b9aae2d71a986c77a0d83615e +oid sha256:a2e75580dd3f3c6afecb6c21e7f64ae847ec30929e333433a88451173b51bd16 size 2104005 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 5b57854b9c0419d9b4769f60abf1543b55975dfe..5b385da2ef35154fa03f28ef132a93289089ae94 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5928d64f3b05875ae9a17f637423f6592cc4090a1b0830a5a42cbfa32fecf351 +oid sha256:b68ab421a3bda314adeded5f2b2fd1d2d3844a597aed2f6ba7647f3724f0562e size 420646 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip index cb8aabbcf7c5c2de2ff9d34fb6d3a77e043410eb..31a2558814ee137da8a22d407611ce57a5cfe093 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0ca44dd09f35857d8e852eccd21840b24db35c83c0f75ed5bed58c6704670317 -size 989010 +oid sha256:e5191ebaf888402c65abc94e978357b0102867a75dbf7539a0ef3ba4d1973cb8 +size 952213 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_small/tuning-impact-elo.png.zip index eeb589455f7b803ab12b337b164916bd189aecee..2c2741cc99cee98c8e40330b918a25b307feffa1 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:64d1d15e597f7ec965fcad544391dfad317dff5ff0db2937b3a5fd914c429b1b +oid sha256:851c168ac8aba376fa906cd6d598d0bd80af847498978961fb367476367d974f size 124086 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_classification/datasets_small/website_leaderboard.csv index 8daf7f37270cdc0ed63bb697ac3aa4295f93ce43..204a78d70f070d6eabfbb2f0575514b4a7a17aa4 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_small/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_classification/datasets_small/website_leaderboard.csv @@ -1,53 +1,53 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1621.0,+89/-68,0.765,7.91,3.4,7.988,610.76,8.082,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1605.0,+95/-83,0.761,8.43,2.98,5.358,2289.05,8.114,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1588.0,+82/-68,0.736,9.04,3.55,6.481,2289.05,1.254,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1587.0,+87/-84,0.769,9.05,3.97,6.793,6.07,0.762,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1470.0,+71/-56,0.499,13.99,8.09,11.927,3422.82,18.936,0.0,False,GPU -5,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1448.0,+70/-57,0.44,15.11,8.48,13.246,3170.35,1.694,0.0,False,GPU -6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1438.0,+81/-56,0.408,15.62,8.07,12.901,3143.8,1.884,0.0,False,CPU -7,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1416.0,+66/-62,0.457,16.78,5.78,11.489,5271.34,231.986,0.0,False,GPU -8,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1399.0,+74/-53,0.434,17.68,8.76,15.067,330.95,2.359,0.0,False,GPU -9,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1388.0,+62/-63,0.345,18.27,10.0,14.216,3170.35,0.241,0.0,False,GPU -10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1386.0,+46/-33,0.31,18.39,12.83,14.934,735.1,1.901,0.0,False,CPU -11,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1383.0,+68/-54,0.32,18.57,9.32,13.652,3422.82,1.534,0.0,False,GPU -12,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1380.0,+70/-62,0.384,18.72,6.02,12.497,5271.34,33.986,0.0,False,GPU -13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1360.0,+48/-29,0.249,19.89,14.24,15.316,735.1,0.311,0.0,False,CPU -14,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1345.0,+47/-39,0.246,20.75,14.59,15.222,1714.62,0.768,0.0,False,CPU -15,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1340.0,+55/-50,0.227,21.06,9.12,14.753,6047.72,0.525,0.0,False,GPU -16,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1334.0,+42/-38,0.228,21.39,15.26,15.413,1714.62,0.098,0.0,False,CPU -17,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1322.0,+58/-58,0.242,22.07,12.42,15.908,1053.84,3.063,0.0,False,GPU -18,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1314.0,+49/-44,0.21,22.55,15.66,16.175,797.9,2.228,0.0,False,CPU -19,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1310.0,+53/-40,0.216,22.77,15.24,16.305,7.19,0.118,0.0,False,CPU -20,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1308.0,+81/-81,0.273,22.93,10.09,15.656,6047.72,8.744,0.0,False,GPU -21,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1304.0,+66/-53,0.265,23.13,12.73,16.81,10.78,0.17,0.0,False,GPU -22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1298.0,+46/-42,0.173,23.49,16.12,16.399,797.9,0.353,0.0,False,CPU -23,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1293.0,+77/-78,0.306,23.79,7.55,15.602,51.44,43.709,0.0,False,GPU -24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1283.0,+48/-40,0.166,24.41,15.85,17.3,3351.28,0.544,0.0,False,CPU -25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1275.0,+57/-55,0.183,24.89,13.03,17.218,1053.84,0.169,0.0,False,GPU -26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1275.0,+57/-53,0.16,24.91,16.94,16.647,2977.49,2.415,0.0,False,CPU -27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1254.0,+47/-42,0.121,26.17,16.12,17.908,3351.28,0.064,0.0,False,CPU -28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1251.0,+61/-61,0.166,26.3,13.09,18.997,15.54,0.346,0.0,False,GPU -29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1248.0,+71/-62,0.189,26.54,13.9,18.299,1423.89,9.208,0.0,False,CPU -30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1234.0,+57/-48,0.142,27.33,16.49,18.862,8.03,0.059,0.0,False,CPU -31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1216.0,+46/-41,0.092,28.43,18.13,18.399,14.89,4.433,0.0,False,GPU -32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1207.0,+79/-69,0.126,28.96,17.02,20.517,399.55,1.466,0.0,False,CPU -33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1200.0,+46/-51,0.084,29.36,19.11,18.904,2977.49,0.156,0.0,False,CPU -34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1196.0,+61/-50,0.088,29.63,16.97,19.665,1423.89,0.749,0.0,False,CPU -35,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1176.0,+43/-48,0.044,30.77,24.27,19.449,2.79,0.238,0.0,False,CPU -36,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1168.0,+80/-76,0.094,31.24,17.28,21.549,399.55,0.174,0.0,False,CPU -37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1162.0,+48/-50,0.063,31.56,25.83,19.906,3.15,0.141,0.0,False,CPU -38,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1153.0,+71/-87,0.099,32.1,17.65,21.772,571.65,1.39,0.0,False,CPU -39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1114.0,+72/-74,0.062,34.3,16.83,22.622,571.65,0.129,0.0,False,CPU -40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1062.0,+62/-63,0.021,37.04,26.37,23.697,11.29,0.204,0.0,False,CPU -41,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1044.0,+97/-100,0.107,37.89,21.55,26.892,2.59,0.533,0.0,False,GPU -42,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1027.0,+75/-96,0.055,38.74,26.12,30.557,483.02,0.874,0.0,False,CPU -43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1026.0,+72/-102,0.035,38.78,31.12,25.68,4.95,1.012,0.0,False,GPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1009.0,+65/-65,0.009,39.55,25.04,25.23,5.04,0.646,0.0,False,CPU -45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1007.0,+84/-107,0.023,39.66,25.44,28.452,121.17,1.63,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1002.0,+76/-98,0.04,39.88,27.22,31.151,483.02,0.113,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+64/-71,0.014,39.96,32.48,28.169,0.46,0.068,0.0,False,CPU -48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,960.0,+77/-98,0.011,41.63,35.13,29.657,0.42,0.074,0.0,False,CPU -49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,926.0,+96/-133,0.026,42.94,30.32,34.444,2.39,0.143,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,917.0,+99/-112,0.005,43.28,29.33,34.126,121.17,0.136,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,582.0,+111/-168,0.0,50.41,46.46,49.533,0.28,0.023,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1621.0,+89/-68,7.988,0.765,7.91,3.4,610.76,8.082,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1605.0,+95/-83,5.358,0.761,8.43,2.98,2289.05,8.114,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1588.0,+82/-68,6.481,0.736,9.04,3.55,2289.05,1.254,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1587.0,+87/-84,6.793,0.769,9.05,3.97,6.07,0.762,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1470.0,+71/-56,11.927,0.499,13.99,8.09,3422.82,18.936,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1448.0,+70/-57,13.246,0.44,15.11,8.48,3170.35,1.694,✔️,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1438.0,+81/-56,12.901,0.408,15.62,8.07,3143.8,1.884,✔️,0.0,False,CPU +7,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1416.0,+66/-62,11.489,0.457,16.78,5.78,5271.34,231.986,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1399.0,+74/-53,15.067,0.434,17.68,8.76,330.95,2.359,➖,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1388.0,+62/-63,14.216,0.345,18.27,10.0,3170.35,0.241,✔️,0.0,False,GPU +10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1386.0,+46/-33,14.934,0.31,18.39,12.83,735.1,1.901,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1383.0,+68/-54,13.652,0.32,18.57,9.32,3422.82,1.534,✔️,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1380.0,+70/-62,12.497,0.384,18.72,6.02,5271.34,33.986,✔️,0.0,False,GPU +13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1360.0,+48/-29,15.316,0.249,19.89,14.24,735.1,0.311,✔️,0.0,False,CPU +14,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1345.0,+47/-39,15.222,0.246,20.75,14.59,1714.62,0.768,✔️,0.0,False,CPU +15,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1340.0,+55/-50,14.753,0.227,21.06,9.12,6047.72,0.525,✔️,0.0,False,GPU +16,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1334.0,+42/-38,15.413,0.228,21.39,15.26,1714.62,0.098,✔️,0.0,False,CPU +17,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1322.0,+58/-58,15.908,0.242,22.07,12.42,1053.84,3.063,✔️,0.0,False,GPU +18,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1314.0,+49/-44,16.175,0.21,22.55,15.66,797.9,2.228,✔️,0.0,False,CPU +19,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1310.0,+53/-40,16.305,0.216,22.77,15.24,7.19,0.118,✔️,0.0,False,CPU +20,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1308.0,+81/-81,15.656,0.273,22.93,10.09,6047.72,8.744,✔️,0.0,False,GPU +21,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1304.0,+66/-53,16.81,0.265,23.13,12.73,10.78,0.17,✔️,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1298.0,+46/-42,16.399,0.173,23.49,16.12,797.9,0.353,✔️,0.0,False,CPU +23,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1293.0,+77/-78,15.602,0.306,23.79,7.55,51.44,43.709,✔️,0.0,False,GPU +24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1283.0,+48/-40,17.3,0.166,24.41,15.85,3351.28,0.544,✔️,0.0,False,CPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1275.0,+57/-55,17.218,0.183,24.89,13.03,1053.84,0.169,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1275.0,+57/-53,16.647,0.16,24.91,16.94,2977.49,2.415,✔️,0.0,False,CPU +27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1254.0,+47/-42,17.908,0.121,26.17,16.12,3351.28,0.064,✔️,0.0,False,CPU +28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1251.0,+61/-61,18.997,0.166,26.3,13.09,15.54,0.346,✔️,0.0,False,GPU +29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1248.0,+71/-62,18.299,0.189,26.54,13.9,1423.89,9.208,✔️,0.0,False,CPU +30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1234.0,+57/-48,18.862,0.142,27.33,16.49,8.03,0.059,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1216.0,+46/-41,18.399,0.092,28.43,18.13,14.89,4.433,✔️,0.0,False,GPU +32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1207.0,+79/-69,20.517,0.126,28.96,17.02,399.55,1.466,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1200.0,+46/-51,18.904,0.084,29.36,19.11,2977.49,0.156,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1196.0,+61/-50,19.665,0.088,29.63,16.97,1423.89,0.749,✔️,0.0,False,CPU +35,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1176.0,+43/-48,19.449,0.044,30.77,24.27,2.79,0.238,✔️,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1168.0,+80/-76,21.549,0.094,31.24,17.28,399.55,0.174,✔️,0.0,False,CPU +37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1162.0,+48/-50,19.906,0.063,31.56,25.83,3.15,0.141,✔️,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1153.0,+71/-87,21.772,0.099,32.1,17.65,571.65,1.39,✔️,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1114.0,+72/-74,22.622,0.062,34.3,16.83,571.65,0.129,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1062.0,+62/-63,23.697,0.021,37.04,26.37,11.29,0.204,✔️,0.0,False,CPU +41,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1044.0,+97/-100,26.892,0.107,37.89,21.55,2.59,0.533,➖,0.0,False,GPU +42,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1027.0,+75/-96,30.557,0.055,38.74,26.12,483.02,0.874,✔️,0.0,False,CPU +43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1026.0,+72/-102,25.68,0.035,38.78,31.12,4.95,1.012,✔️,0.0,False,GPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1009.0,+65/-65,25.23,0.009,39.55,25.04,5.04,0.646,✔️,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1007.0,+84/-107,28.452,0.023,39.66,25.44,121.17,1.63,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1002.0,+76/-98,31.151,0.04,39.88,27.22,483.02,0.113,✔️,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+64/-71,28.169,0.014,39.96,32.48,0.46,0.068,✔️,0.0,False,CPU +48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),960.0,+77/-98,29.657,0.011,41.63,35.13,0.42,0.074,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),926.0,+96/-133,34.444,0.026,42.94,30.32,2.39,0.143,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),917.0,+99/-112,34.126,0.005,43.28,29.33,121.17,0.136,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),582.0,+111/-168,49.533,0.0,50.41,46.46,0.28,0.023,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_small/winrate_matrix.png.zip index 55eedf0832ea812f5edc807630d79f8a72ae7a74..a59f6bcb090e5855bbc57cf6b35d1650b79b3287 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c7ceac32f0de3543928c9f5c1555e6a60a16e6b7f59a838abddf6ac201992b40 +oid sha256:47b5ac64d47ef026c71a1cbec9e7ca65f2bab86b9c5b85ad7859e5202f98e093 size 2071503 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 68a77ff1dd9e37f5f021012f30655dfea8e35705..d0789551547daa806cd048d487b830c24347101d 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b349df431daea8a8bf5293741f622707a71953616c88725dd3816b4c0a32015a +oid sha256:c5da55dc35c03ecd57aa841f4bcf3f2ba689959fa1cc7422af2258e654132c31 size 471650 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip index e4b81761c96ed17bdba23ad15ae2d70c90b70f1b..d5488542d481e8975273be1732609a1bad753463 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9d4b0c21ddff96de33cf89933287f3d1cc6f7686e2ea2e0b5d529ef5a60a5bfc -size 1073311 +oid sha256:da4d76b4ddee7f40f2f4099a92c7a1c66f1430dafc58cc1df67047a2eae597ef +size 1051061 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip index 2908de4b01e9703168804fccf2f4a3a2b6bfb90a..3c944c29bc3eefa8f0efa4d9baae08dd77cb0c64 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:da2018312c5fd5a48044b93dc3e4613742d285aa3708c8de104bcdbd792e735a +oid sha256:5a57f5ff76c703004367a4e66acca6c3b122bf30dff44eca330154f76fd45acd size 120230 diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/website_leaderboard.csv index 3e24e20a24c4cda44ab4bc96fbb38217adfa91f4..9eb8e140ebb7cc10688c97ebfcf83baa97abecd1 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1634.0,+83/-79,0.728,9.78,3.8,6.234,2168.17,7.961,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1631.0,+70/-57,0.747,9.87,4.93,7.553,5.99,0.633,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1625.0,+85/-64,0.701,10.13,4.67,9.074,604.44,8.082,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1614.0,+81/-69,0.702,10.59,4.34,7.377,2168.17,1.23,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1544.0,+69/-56,0.576,13.78,5.67,11.21,3445.6,48.236,0.0,False,GPU -5,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1536.0,+80/-59,0.593,14.17,6.16,11.857,146.65,4.101,0.0,False,GPU -6,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1518.0,+87/-73,0.58,15.09,5.58,11.42,4.37,0.585,0.0,False,GPU -7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1486.0,+61/-44,0.447,16.86,9.64,13.101,3169.9,16.793,0.0,False,GPU -8,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1459.0,+71/-61,0.447,18.45,6.6,12.877,8.89,1.743,0.0,False,GPU -9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1454.0,+71/-46,0.373,18.73,10.01,14.494,2828.45,1.596,0.0,False,GPU -10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1452.0,+82/-72,0.442,18.89,6.45,13.931,3445.6,0.995,0.0,False,GPU -11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1444.0,+66/-55,0.356,19.32,10.43,14.093,2898.23,1.884,0.0,False,CPU -12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1425.0,+77/-70,0.388,20.47,6.97,12.671,5119.36,231.986,0.0,False,GPU -13,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1424.0,+60/-53,0.369,20.56,10.58,15.951,330.95,2.023,0.0,False,GPU -14,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1410.0,+84/-72,0.421,21.42,8.85,15.11,4.06,0.436,0.0,False,GPU -15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1400.0,+56/-54,0.297,22.04,11.42,14.802,3169.9,1.446,0.0,False,GPU -16,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1400.0,+82/-72,0.355,22.07,7.06,13.572,5119.36,26.967,0.0,False,GPU -17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1399.0,+72/-51,0.29,22.12,11.71,15.476,2828.45,0.216,0.0,False,GPU -18,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1388.0,+42/-35,0.23,22.83,15.77,16.363,647.56,1.72,0.0,False,CPU -19,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1362.0,+37/-30,0.175,24.46,17.6,16.741,647.56,0.284,0.0,False,CPU -20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1356.0,+50/-36,0.186,24.9,17.76,16.602,1465.86,0.692,0.0,False,CPU -21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1355.0,+56/-50,0.195,24.97,11.03,15.889,5944.88,0.516,0.0,False,GPU -22,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1342.0,+48/-35,0.173,25.77,18.98,16.823,1465.86,0.091,0.0,False,CPU -23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1326.0,+64/-56,0.23,26.84,14.3,18.007,10.42,0.155,0.0,False,GPU -24,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1325.0,+46/-41,0.157,26.9,19.33,17.617,766.06,1.917,0.0,False,CPU -25,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1323.0,+95/-83,0.242,27.06,11.78,17.015,5944.88,8.396,0.0,False,GPU -26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1320.0,+45/-44,0.159,27.23,18.32,17.726,5.72,0.11,0.0,False,CPU -27,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1320.0,+87/-95,0.3,27.25,8.62,16.755,50.32,43.824,0.0,False,GPU -28,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1320.0,+50/-53,0.172,27.25,16.63,17.396,934.1,2.767,0.0,False,GPU -29,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1309.0,+39/-36,0.119,27.97,19.24,17.841,766.06,0.278,0.0,False,CPU -30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1297.0,+46/-41,0.117,28.77,18.8,18.633,2686.11,0.47,0.0,False,CPU -31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1294.0,+56/-54,0.135,28.96,20.03,17.813,2862.05,2.158,0.0,False,CPU -32,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1283.0,+45/-58,0.14,29.68,15.39,19.49,14.8,0.337,0.0,False,GPU -33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1277.0,+50/-50,0.12,30.07,16.88,18.738,934.1,0.165,0.0,False,GPU -34,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1275.0,+77/-87,0.174,30.22,15.28,19.42,1358.63,8.067,0.0,False,CPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1269.0,+45/-48,0.083,30.63,18.46,19.24,2686.11,0.056,0.0,False,CPU -36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1244.0,+50/-47,0.089,32.25,19.07,20.124,7.4,0.057,0.0,False,CPU -37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1243.0,+44/-39,0.068,32.34,20.31,19.291,13.83,3.953,0.0,False,GPU -38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1219.0,+64/-78,0.076,33.88,19.08,20.796,1358.63,0.897,0.0,False,CPU -39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1213.0,+53/-43,0.061,34.25,22.19,20.109,2862.05,0.152,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1199.0,+60/-67,0.076,35.15,24.0,22.25,370.85,1.466,0.0,False,CPU -41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1184.0,+45/-43,0.019,36.07,30.03,21.0,2.4,0.218,0.0,False,CPU -42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1160.0,+62/-70,0.046,37.55,25.98,23.266,370.85,0.161,0.0,False,CPU -43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1153.0,+36/-44,0.027,37.95,32.6,21.599,2.9,0.133,0.0,False,CPU -44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1133.0,+70/-69,0.043,39.17,25.14,23.712,527.42,1.39,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1093.0,+61/-65,0.011,41.45,24.97,24.551,527.42,0.123,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1081.0,+56/-63,0.014,42.1,30.05,24.892,10.38,0.188,0.0,False,CPU -47,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1071.0,+83/-130,0.088,42.63,24.72,27.865,2.43,0.495,0.0,False,GPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1058.0,+100/-110,0.054,43.29,28.54,30.113,436.81,0.874,0.0,False,CPU -49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1034.0,+89/-103,0.024,44.5,35.91,27.438,4.4,0.974,0.0,False,GPU -50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1033.0,+101/-105,0.036,44.53,30.27,30.656,436.81,0.106,0.0,False,CPU -51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1032.0,+72/-81,0.0,44.59,27.79,26.107,4.73,0.623,0.0,False,CPU -52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1008.0,+77/-109,0.015,45.72,32.67,30.107,113.26,1.55,0.0,False,CPU -53,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+61/-83,0.003,46.1,38.93,29.212,0.45,0.066,0.0,False,CPU -54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,969.0,+75/-112,0.009,47.45,40.18,30.601,0.4,0.07,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,958.0,+91/-127,0.023,47.9,33.34,33.436,2.16,0.139,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,909.0,+87/-143,0.001,49.71,34.2,36.035,113.26,0.12,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,600.0,+121/-261,0.0,56.33,51.43,49.89,0.26,0.023,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1634.0,+83/-79,6.234,0.728,9.78,3.8,2168.17,7.961,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1631.0,+70/-57,7.553,0.747,9.87,4.93,5.99,0.633,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1625.0,+85/-64,9.074,0.701,10.13,4.67,604.44,8.082,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1614.0,+81/-69,7.377,0.702,10.59,4.34,2168.17,1.23,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1544.0,+69/-56,11.21,0.576,13.78,5.67,3445.6,48.236,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1536.0,+80/-59,11.857,0.593,14.17,6.16,146.65,4.101,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1518.0,+87/-73,11.42,0.58,15.09,5.58,4.37,0.585,➖,0.0,False,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1486.0,+61/-44,13.101,0.447,16.86,9.64,3169.9,16.793,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1459.0,+71/-61,12.877,0.447,18.45,6.6,8.89,1.743,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1454.0,+71/-46,14.494,0.373,18.73,10.01,2828.45,1.596,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1452.0,+82/-72,13.931,0.442,18.89,6.45,3445.6,0.995,✔️,0.0,False,GPU +11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1444.0,+66/-55,14.093,0.356,19.32,10.43,2898.23,1.884,✔️,0.0,False,CPU +12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1425.0,+77/-70,12.671,0.388,20.47,6.97,5119.36,231.986,✔️,0.0,False,GPU +13,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1424.0,+60/-53,15.951,0.369,20.56,10.58,330.95,2.023,➖,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1410.0,+84/-72,15.11,0.421,21.42,8.85,4.06,0.436,✔️,0.0,False,GPU +15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1400.0,+56/-54,14.802,0.297,22.04,11.42,3169.9,1.446,✔️,0.0,False,GPU +16,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1400.0,+82/-72,13.572,0.355,22.07,7.06,5119.36,26.967,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1399.0,+72/-51,15.476,0.29,22.12,11.71,2828.45,0.216,✔️,0.0,False,GPU +18,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1388.0,+42/-35,16.363,0.23,22.83,15.77,647.56,1.72,✔️,0.0,False,CPU +19,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1362.0,+37/-30,16.741,0.175,24.46,17.6,647.56,0.284,✔️,0.0,False,CPU +20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1356.0,+50/-36,16.602,0.186,24.9,17.76,1465.86,0.692,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1355.0,+56/-50,15.889,0.195,24.97,11.03,5944.88,0.516,✔️,0.0,False,GPU +22,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1342.0,+48/-35,16.823,0.173,25.77,18.98,1465.86,0.091,✔️,0.0,False,CPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1326.0,+64/-56,18.007,0.23,26.84,14.3,10.42,0.155,✔️,0.0,False,GPU +24,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1325.0,+46/-41,17.617,0.157,26.9,19.33,766.06,1.917,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1323.0,+95/-83,17.015,0.242,27.06,11.78,5944.88,8.396,✔️,0.0,False,GPU +26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1320.0,+45/-44,17.726,0.159,27.23,18.32,5.72,0.11,✔️,0.0,False,CPU +27,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1320.0,+87/-95,16.755,0.3,27.25,8.62,50.32,43.824,✔️,0.0,False,GPU +28,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1320.0,+50/-53,17.396,0.172,27.25,16.63,934.1,2.767,✔️,0.0,False,GPU +29,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1309.0,+39/-36,17.841,0.119,27.97,19.24,766.06,0.278,✔️,0.0,False,CPU +30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1297.0,+46/-41,18.633,0.117,28.77,18.8,2686.11,0.47,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1294.0,+56/-54,17.813,0.135,28.96,20.03,2862.05,2.158,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1283.0,+45/-58,19.49,0.14,29.68,15.39,14.8,0.337,✔️,0.0,False,GPU +33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1277.0,+50/-50,18.738,0.12,30.07,16.88,934.1,0.165,✔️,0.0,False,GPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1275.0,+77/-87,19.42,0.174,30.22,15.28,1358.63,8.067,✔️,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1269.0,+45/-48,19.24,0.083,30.63,18.46,2686.11,0.056,✔️,0.0,False,CPU +36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1244.0,+50/-47,20.124,0.089,32.25,19.07,7.4,0.057,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1243.0,+44/-39,19.291,0.068,32.34,20.31,13.83,3.953,✔️,0.0,False,GPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1219.0,+64/-78,20.796,0.076,33.88,19.08,1358.63,0.897,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1213.0,+53/-43,20.109,0.061,34.25,22.19,2862.05,0.152,✔️,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1199.0,+60/-67,22.25,0.076,35.15,24.0,370.85,1.466,✔️,0.0,False,CPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1184.0,+45/-43,21.0,0.019,36.07,30.03,2.4,0.218,✔️,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1160.0,+62/-70,23.266,0.046,37.55,25.98,370.85,0.161,✔️,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1153.0,+36/-44,21.599,0.027,37.95,32.6,2.9,0.133,✔️,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1133.0,+70/-69,23.712,0.043,39.17,25.14,527.42,1.39,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1093.0,+61/-65,24.551,0.011,41.45,24.97,527.42,0.123,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1081.0,+56/-63,24.892,0.014,42.1,30.05,10.38,0.188,✔️,0.0,False,CPU +47,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1071.0,+83/-130,27.865,0.088,42.63,24.72,2.43,0.495,➖,0.0,False,GPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1058.0,+100/-110,30.113,0.054,43.29,28.54,436.81,0.874,✔️,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1034.0,+89/-103,27.438,0.024,44.5,35.91,4.4,0.974,✔️,0.0,False,GPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1033.0,+101/-105,30.656,0.036,44.53,30.27,436.81,0.106,✔️,0.0,False,CPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1032.0,+72/-81,26.107,0.0,44.59,27.79,4.73,0.623,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1008.0,+77/-109,30.107,0.015,45.72,32.67,113.26,1.55,✔️,0.0,False,CPU +53,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+61/-83,29.212,0.003,46.1,38.93,0.45,0.066,✔️,0.0,False,CPU +54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),969.0,+75/-112,30.601,0.009,47.45,40.18,0.4,0.07,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),958.0,+91/-127,33.436,0.023,47.9,33.34,2.16,0.139,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),909.0,+87/-143,36.035,0.001,49.71,34.2,113.26,0.12,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),600.0,+121/-261,49.89,0.0,56.33,51.43,0.26,0.023,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip index a864fb99d67c662c46cd954f1500be6398f0eedb..9c86d79e7227d9f64f40a6c1cbf6f5b69bdb8da4 100644 --- a/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f2d108298b8807cf1f16cb60d3e435091a643eb653052340b0d92183ddad0857 +oid sha256:d76acd884961fc3c50d365e6564270d36b5266513ae03863ce4503e941ea2c89 size 2801926 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index 53257fc1da22bf448e33e8d3c7d08322225ebb81..81df7d8182fad1a644649ba7073479bbe7897d97 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3e4a8a2c38b55f2aea496a5da37419f9919fcb2f5b55b7e3e532da6cd2940e43 +oid sha256:528e3be3e90c0609f350f4d34275e13e6edacb5d78a5da5d8b3fa60747443560 size 419969 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip index 859cc1eca0e4b1c5c31b8861f15cd7d78dbaa6be..50d621f5bf87a8055167469f64b4635aba567364 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6f2888952e914138db7784f0d08eda250ba66bcdb33413a2ef8bb1faeeb72dd6 -size 951519 +oid sha256:136db68f3f27fdaf8d2ef5249f6063cfe5dd18fa3f8ee9aff193a091b5c2dd9b +size 924416 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip index e35d5e84adb8894dd9cb6da94f949b5f1265338c..2558ec6f2f3306a0e16a4868818c311bc63353f2 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:81f535dc60f72929211eca71035956558c460b88f62b97a1358897c410d74f93 +oid sha256:207a1db3d79883293969ace68570ac4fddc4d2da1db09628feb0d83e8926c865 size 117390 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/website_leaderboard.csv index 87063dd151ad25bc36895faee5218919b6448af2..06a285f296ecc9a24b2ca1a8abf29994fc92c071 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/website_leaderboard.csv @@ -1,53 +1,53 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1760.0,+178/-83,0.832,5.81,3.5,9.062,1045.67,7.086,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1734.0,+245/-102,0.819,6.48,3.01,6.353,2710.27,1.347,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1725.0,+327/-154,0.812,6.69,2.17,5.551,2710.27,8.099,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1588.0,+259/-161,0.668,11.28,4.18,8.494,10.23,0.844,0.0,False,GPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1534.0,+219/-124,0.45,13.62,5.1,14.217,6219.24,3.78,0.0,False,CPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1519.0,+190/-118,0.51,14.32,6.59,12.711,4150.25,19.131,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1496.0,+137/-88,0.441,15.41,8.18,13.842,4150.25,3.423,0.0,False,GPU -7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1478.0,+125/-87,0.398,16.31,9.73,16.609,5885.87,1.632,0.0,False,GPU -8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1465.0,+122/-75,0.344,17.0,12.25,16.251,1377.29,5.202,0.0,False,CPU -9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1458.0,+101/-59,0.335,17.34,13.96,14.67,3104.39,1.095,0.0,False,CPU -10,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1456.0,+100/-70,0.322,17.47,13.14,16.47,1377.29,1.259,0.0,False,CPU -11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1438.0,+123/-87,0.349,18.46,9.94,17.463,5885.87,0.299,0.0,False,GPU -12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1435.0,+170/-123,0.397,18.57,7.39,18.358,4635.14,212.285,0.0,False,GPU -13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1433.0,+102/-67,0.278,18.67,14.85,15.297,3104.39,0.212,0.0,False,CPU -14,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1410.0,+103/-98,0.276,19.91,15.48,16.733,2210.02,0.974,0.0,False,CPU -15,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1404.0,+101/-101,0.269,20.24,15.39,16.868,2210.02,3.985,0.0,False,CPU -16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1403.0,+113/-97,0.239,20.3,13.01,15.873,7127.77,9.214,0.0,False,GPU -17,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1396.0,+76/-45,0.201,20.68,10.91,16.245,7127.77,0.776,0.0,False,GPU -18,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1382.0,+179/-128,0.333,21.41,8.7,20.214,4635.14,34.77,0.0,False,GPU -19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1376.0,+153/-120,0.284,21.74,14.05,19.283,1980.6,3.605,0.0,False,GPU -20,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1373.0,+139/-108,0.293,21.93,15.97,19.839,507.73,4.428,0.0,False,GPU -21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1361.0,+115/-90,0.197,22.61,16.5,18.672,36484.23,0.827,0.0,False,CPU -22,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1339.0,+128/-120,0.238,23.84,15.04,20.568,17.83,0.197,0.0,False,GPU -23,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1327.0,+185/-126,0.245,24.53,12.84,21.441,717.36,1.612,0.0,False,CPU -24,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1326.0,+185/-166,0.293,24.56,10.84,20.662,737.91,1.864,0.0,False,CPU -25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1322.0,+77/-52,0.144,24.83,21.0,18.814,50.54,0.227,0.0,False,CPU -26,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1315.0,+106/-103,0.146,25.21,18.54,19.435,36484.23,0.127,0.0,False,CPU -27,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1314.0,+158/-113,0.136,25.25,18.72,20.348,3901.28,2.028,0.0,False,CPU -28,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1282.0,+125/-114,0.145,27.06,21.54,21.368,19.31,0.108,0.0,False,CPU -29,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1278.0,+182/-138,0.203,27.3,11.99,22.94,717.36,0.326,0.0,False,CPU -30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1276.0,+123/-114,0.163,27.43,15.98,21.665,1980.6,0.61,0.0,False,GPU -31,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1273.0,+194/-174,0.234,27.57,11.87,24.011,2644.99,12.294,0.0,False,CPU -32,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1273.0,+198/-187,0.227,27.57,9.69,22.052,737.91,0.316,0.0,False,CPU -33,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1265.0,+210/-138,0.221,28.02,8.16,23.278,43.39,42.575,0.0,False,GPU -34,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1247.0,+118/-131,0.098,28.99,20.71,22.692,3901.28,0.223,0.0,False,CPU -35,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1246.0,+115/-89,0.063,29.06,25.43,20.308,4.72,0.542,0.0,False,CPU -36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1232.0,+137/-118,0.133,29.78,23.49,22.446,5.04,0.429,0.0,False,CPU -37,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1221.0,+151/-140,0.122,30.37,16.99,25.129,2644.99,1.148,0.0,False,CPU -38,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1202.0,+94/-115,0.032,31.38,25.09,25.131,17.94,0.691,0.0,False,GPU -39,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1159.0,+68/-94,0.0,33.59,30.64,25.754,19.2,7.624,0.0,False,GPU -40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1089.0,+135/-197,0.054,36.92,30.62,29.146,9.42,1.675,0.0,False,GPU -41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1055.0,+75/-117,0.0,38.37,30.41,28.55,13.69,0.314,0.0,False,CPU -42,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1006.0,+229/-348,0.031,40.34,22.09,39.232,151.69,2.346,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+118/-185,0.0,40.55,36.98,35.45,0.74,0.137,0.0,False,CPU -44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,997.0,+100/-244,0.0,40.66,38.52,39.878,892.95,0.818,0.0,False,CPU -45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,959.0,+92/-261,0.0,42.01,39.7,40.883,892.95,0.146,0.0,False,CPU -46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,949.0,+131/-194,0.003,42.33,39.94,34.505,8.51,0.734,0.0,False,CPU -47,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,906.0,+139/-185,0.0,43.69,40.86,40.594,0.69,0.137,0.0,False,CPU -48,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,844.0,+66/-133,0.0,45.37,45.16,39.593,5.05,1.013,0.0,False,GPU -49,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,824.0,+265/-559,0.016,45.88,36.33,49.561,151.69,0.258,0.0,False,CPU -50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,746.0,+93/-250,0.0,47.53,47.17,47.08,4.53,0.164,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,198.0,+134/-662,0.0,51.77,51.76,70.493,0.52,0.076,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1760.0,+178/-83,9.062,0.832,5.81,3.5,1045.67,7.086,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1734.0,+245/-102,6.353,0.819,6.48,3.01,2710.27,1.347,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1725.0,+327/-154,5.551,0.812,6.69,2.17,2710.27,8.099,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1588.0,+259/-161,8.494,0.668,11.28,4.18,10.23,0.844,✔️,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1534.0,+219/-124,14.217,0.45,13.62,5.1,6219.24,3.78,✔️,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1519.0,+190/-118,12.711,0.51,14.32,6.59,4150.25,19.131,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1496.0,+137/-88,13.842,0.441,15.41,8.18,4150.25,3.423,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1478.0,+125/-87,16.609,0.398,16.31,9.73,5885.87,1.632,✔️,0.0,False,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1465.0,+122/-75,16.251,0.344,17.0,12.25,1377.29,5.202,✔️,0.0,False,CPU +9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1458.0,+101/-59,14.67,0.335,17.34,13.96,3104.39,1.095,✔️,0.0,False,CPU +10,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1456.0,+100/-70,16.47,0.322,17.47,13.14,1377.29,1.259,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1438.0,+123/-87,17.463,0.349,18.46,9.94,5885.87,0.299,✔️,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1435.0,+170/-123,18.358,0.397,18.57,7.39,4635.14,212.285,✔️,0.0,False,GPU +13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1433.0,+102/-67,15.297,0.278,18.67,14.85,3104.39,0.212,✔️,0.0,False,CPU +14,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1410.0,+103/-98,16.733,0.276,19.91,15.48,2210.02,0.974,✔️,0.0,False,CPU +15,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1404.0,+101/-101,16.868,0.269,20.24,15.39,2210.02,3.985,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1403.0,+113/-97,15.873,0.239,20.3,13.01,7127.77,9.214,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1396.0,+76/-45,16.245,0.201,20.68,10.91,7127.77,0.776,✔️,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1382.0,+179/-128,20.214,0.333,21.41,8.7,4635.14,34.77,✔️,0.0,False,GPU +19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1376.0,+153/-120,19.283,0.284,21.74,14.05,1980.6,3.605,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1373.0,+139/-108,19.839,0.293,21.93,15.97,507.73,4.428,➖,0.0,False,GPU +21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1361.0,+115/-90,18.672,0.197,22.61,16.5,36484.23,0.827,✔️,0.0,False,CPU +22,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1339.0,+128/-120,20.568,0.238,23.84,15.04,17.83,0.197,✔️,0.0,False,GPU +23,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1327.0,+185/-126,21.441,0.245,24.53,12.84,717.36,1.612,✔️,0.0,False,CPU +24,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1326.0,+185/-166,20.662,0.293,24.56,10.84,737.91,1.864,✔️,0.0,False,CPU +25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1322.0,+77/-52,18.814,0.144,24.83,21.0,50.54,0.227,✔️,0.0,False,CPU +26,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1315.0,+106/-103,19.435,0.146,25.21,18.54,36484.23,0.127,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1314.0,+158/-113,20.348,0.136,25.25,18.72,3901.28,2.028,✔️,0.0,False,CPU +28,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1282.0,+125/-114,21.368,0.145,27.06,21.54,19.31,0.108,✔️,0.0,False,CPU +29,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1278.0,+182/-138,22.94,0.203,27.3,11.99,717.36,0.326,✔️,0.0,False,CPU +30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1276.0,+123/-114,21.665,0.163,27.43,15.98,1980.6,0.61,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1273.0,+194/-174,24.011,0.234,27.57,11.87,2644.99,12.294,✔️,0.0,False,CPU +32,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1273.0,+198/-187,22.052,0.227,27.57,9.69,737.91,0.316,✔️,0.0,False,CPU +33,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1265.0,+210/-138,23.278,0.221,28.02,8.16,43.39,42.575,✔️,0.0,False,GPU +34,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1247.0,+118/-131,22.692,0.098,28.99,20.71,3901.28,0.223,✔️,0.0,False,CPU +35,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1246.0,+115/-89,20.308,0.063,29.06,25.43,4.72,0.542,✔️,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1232.0,+137/-118,22.446,0.133,29.78,23.49,5.04,0.429,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1221.0,+151/-140,25.129,0.122,30.37,16.99,2644.99,1.148,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1202.0,+94/-115,25.131,0.032,31.38,25.09,17.94,0.691,✔️,0.0,False,GPU +39,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1159.0,+68/-94,25.754,0.0,33.59,30.64,19.2,7.624,✔️,0.0,False,GPU +40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1089.0,+135/-197,29.146,0.054,36.92,30.62,9.42,1.675,✔️,0.0,False,GPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1055.0,+75/-117,28.55,0.0,38.37,30.41,13.69,0.314,✔️,0.0,False,CPU +42,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1006.0,+229/-348,39.232,0.031,40.34,22.09,151.69,2.346,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+118/-185,35.45,0.0,40.55,36.98,0.74,0.137,✔️,0.0,False,CPU +44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),997.0,+100/-244,39.878,0.0,40.66,38.52,892.95,0.818,✔️,0.0,False,CPU +45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),959.0,+92/-261,40.883,0.0,42.01,39.7,892.95,0.146,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),949.0,+131/-194,34.505,0.003,42.33,39.94,8.51,0.734,✔️,0.0,False,CPU +47,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),906.0,+139/-185,40.594,0.0,43.69,40.86,0.69,0.137,✔️,0.0,False,CPU +48,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),844.0,+66/-133,39.593,0.0,45.37,45.16,5.05,1.013,➖,0.0,False,GPU +49,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),824.0,+265/-559,49.561,0.016,45.88,36.33,151.69,0.258,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),746.0,+93/-250,47.08,0.0,47.53,47.17,4.53,0.164,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),198.0,+134/-662,70.493,0.0,51.77,51.76,0.52,0.076,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/winrate_matrix.png.zip index 5968b4b4d2d42727dcf627bbeee516b6dbfa91cf..0e92baa4364090afe172dadf9dbb69737eeb9b83 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7044cc937df41448be482c3b01aa846da805e260cb93976dc7be8aa5af306c41 +oid sha256:6c16c319c7936bbd481690c97e48d8d4a09c18dc3b09f8e304fde2b85e9689f7 size 1988199 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index 9f7559f2266d3ad347e58953f23210fefc944c4e..e0e57b9fb3dd95172c81d50e1a62f56ad3702fcc 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:40b62a18ea285b21e014dcd6fe5be66c90cb2cab020bc6ed54c8ed94c3e38986 +oid sha256:23051d990f26b358d1b0774145ea8528299d7870d2bb6b5b6aeb021b3c51fa2e size 444887 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip index aa4e688feba7e49be28c1e73b45f6fb9b991c7c5..1c26b24a6cffb5b1ba6e3130b498b0c3504e1386 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0c5430a3114c31d695eba5195595e770a2c6a7e86dd6069b5f4969effa691e4e -size 977457 +oid sha256:fc6db28c154ad18bed6811d344d3bf9c6ac2b5b2628235e0b92cecdfb4e40aed +size 951729 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip index 158486c5d429ddafe6f9209f53ecfb39f85147ac..ea04436cc646b011eb2e76e69a3c66af1df8a60f 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:33cb433f2ada5e02999e95e87c16fac3485e9c6cf71aac38735029012243231a +oid sha256:cee6ff6f9ea7e012f32526d1c3e1e2a27fd4068b6f9675f2a442b7a4c234cd33 size 106892 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/website_leaderboard.csv index c4f7f7abf85b38dcd22115ae20c737de87eb39da..a419ca2395ccc34bf4d1abe9827fdbf5cbfafd8c 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/website_leaderboard.csv @@ -1,54 +1,54 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,9270.0,+0/-0,1.0,1.11,1.06,0.019,133.98,8.253,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,8803.0,+0/-0,0.934,2.56,2.2,1.7,276.87,4.662,0.0,False,CPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,8700.0,+0/-0,0.927,3.0,2.84,1.878,276.86,3.44,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,8621.0,+0/-0,0.905,3.33,3.0,2.424,133.98,2.816,0.0,False,GPU -4,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,6195.0,+0/-0,0.594,6.78,6.31,9.581,61.95,0.17,0.0,False,CPU -5,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,6182.0,+0/-0,0.595,6.89,6.45,9.545,7606.4,0.826,0.0,False,CPU -6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,6134.0,+0/-0,0.573,7.33,7.13,10.011,157.72,3.649,0.0,False,CPU -7,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,6038.0,+0/-0,0.578,8.22,7.72,9.898,7606.4,0.048,0.0,False,CPU -8,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,5997.0,+0/-0,0.578,8.67,7.91,9.921,61.95,0.016,0.0,False,CPU -9,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,5959.0,+0/-0,0.552,9.11,8.66,10.441,157.72,0.374,0.0,False,CPU -10,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,5682.0,+0/-0,0.508,11.89,10.98,11.347,2688.02,0.24,0.0,False,CPU -11,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,5671.0,+0/-0,0.518,12.0,11.9,11.168,466.47,1.036,0.0,False,CPU -12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,5573.0,+0/-0,0.499,13.0,12.51,11.536,2688.02,0.726,0.0,False,CPU -13,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,5519.0,+0/-0,0.501,13.56,13.39,11.505,0.39,0.051,0.0,False,CPU -14,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,5486.0,+0/-0,0.501,13.89,13.77,11.505,466.47,0.051,0.0,False,CPU -15,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,5145.0,+0/-0,0.399,17.0,16.0,13.544,1.48,0.53,0.0,False,GPU -16,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,5108.0,+0/-0,0.423,17.33,17.24,13.063,7.54,0.047,0.0,False,CPU -17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,5082.0,+0/-0,0.407,17.56,17.52,13.396,4423.24,29.2,0.0,False,GPU -18,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,5032.0,+0/-0,0.389,18.0,17.84,13.739,17.7,0.086,0.0,False,CPU -19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,4812.0,+0/-0,0.354,19.67,19.62,14.412,4423.24,2.237,0.0,False,GPU -20,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,4739.0,+0/-0,0.348,20.11,20.02,14.523,2.82,1.073,0.0,False,GPU -21,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,3202.0,+0/-0,0.25,22.11,22.11,16.353,20.98,0.329,0.0,False,CPU -22,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,2532.0,+0/-0,0.132,24.22,24.17,18.457,4638.91,343.215,0.0,False,GPU -23,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,2475.0,+0/-0,0.115,24.56,24.47,18.732,2389.31,0.267,0.0,False,CPU -24,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,2475.0,+0/-0,0.113,24.56,24.51,18.777,20.98,0.027,0.0,False,CPU -25,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,2456.0,+0/-0,0.109,24.67,24.61,18.841,2389.31,0.023,0.0,False,CPU -26,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1738.0,+0/-0,0.0,27.44,27.43,20.674,14.87,1.855,0.0,False,GPU -27,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1554.0,+0/-0,0.0,28.33,28.3,20.901,1993.14,0.452,0.0,False,GPU -28,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1404.0,+0/-0,0.0,29.22,29.16,21.65,1993.14,0.059,0.0,False,GPU -29,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1218.0,+0/-0,0.0,30.44,30.4,22.097,4638.91,43.781,0.0,False,GPU -30,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1218.0,+0/-0,0.0,30.44,30.35,22.475,40.54,38.741,0.0,False,GPU -31,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+0/-0,0.0,31.89,31.77,23.341,0.18,0.008,0.0,False,CPU -32,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,821.0,+0/-0,0.0,33.0,32.98,23.568,1294.93,0.417,0.0,False,GPU -33,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,665.0,+0/-0,0.0,33.89,33.86,24.01,1294.93,0.06,0.0,False,GPU -34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,599.0,+0/-0,0.0,34.22,34.18,24.919,0.18,0.04,0.0,False,CPU -35,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,-215.0,+0/-0,0.0,36.11,36.11,28.274,1863.58,6.302,0.0,False,GPU -36,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,-734.0,+0/-0,0.0,37.56,37.54,30.79,1863.58,1.131,0.0,False,GPU -37,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,-788.0,+0/-0,0.0,37.78,37.75,30.768,3.96,0.025,0.0,False,CPU -38,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,-1010.0,+0/-0,0.0,38.67,38.66,32.321,6.85,0.088,0.0,False,GPU -39,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,-1462.0,+0/-0,0.0,39.89,39.89,33.496,66.06,1.792,0.0,False,GPU -40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,-2063.0,+0/-0,0.0,41.0,41.0,37.017,7.55,0.149,0.0,False,GPU -41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,-2631.0,+0/-0,0.0,42.11,42.11,41.044,569.0,0.17,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,-2955.0,+0/-0,0.0,42.89,42.89,41.644,569.0,1.658,0.0,False,CPU -43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,-3626.0,+0/-0,0.0,44.11,44.11,50.207,0.07,0.015,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,-4076.0,+0/-0,0.0,45.22,45.21,51.289,2.86,0.085,0.0,False,CPU -45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-4255.0,+0/-0,0.0,45.78,45.77,51.687,30.41,0.089,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-4823.0,+0/-0,0.0,47.44,47.44,54.976,30.41,0.015,0.0,False,CPU -47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-5018.0,+0/-0,0.0,48.33,48.32,55.365,0.13,0.014,0.0,False,CPU -48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,-5106.0,+0/-0,0.0,48.78,48.72,55.242,3.09,0.997,0.0,False,GPU -49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,-5308.0,+0/-0,0.0,49.78,49.77,56.49,0.11,0.023,0.0,False,GPU -50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-5500.0,+0/-0,0.0,50.56,50.54,57.332,76.27,3.054,0.0,False,CPU -51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-6498.0,+0/-0,0.0,52.0,52.0,70.796,76.27,0.37,0.0,False,CPU -52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-7418.0,+0/-0,0.0,53.0,53.0,82.671,0.14,0.136,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),9270.0,+0/-0,0.019,1.0,1.11,1.06,133.98,8.253,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",8803.0,+0/-0,1.7,0.934,2.56,2.2,276.87,4.662,✔️,0.0,False,CPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",8700.0,+0/-0,1.878,0.927,3.0,2.84,276.86,3.44,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),8621.0,+0/-0,2.424,0.905,3.33,3.0,133.98,2.816,✔️,0.0,False,GPU +4,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),6195.0,+0/-0,9.581,0.594,6.78,6.31,61.95,0.17,✔️,0.0,False,CPU +5,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),6182.0,+0/-0,9.545,0.595,6.89,6.45,7606.4,0.826,✔️,0.0,False,CPU +6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),6134.0,+0/-0,10.011,0.573,7.33,7.13,157.72,3.649,✔️,0.0,False,CPU +7,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),6038.0,+0/-0,9.898,0.578,8.22,7.72,7606.4,0.048,✔️,0.0,False,CPU +8,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),5997.0,+0/-0,9.921,0.578,8.67,7.91,61.95,0.016,✔️,0.0,False,CPU +9,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),5959.0,+0/-0,10.441,0.552,9.11,8.66,157.72,0.374,✔️,0.0,False,CPU +10,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),5682.0,+0/-0,11.347,0.508,11.89,10.98,2688.02,0.24,✔️,0.0,False,CPU +11,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),5671.0,+0/-0,11.168,0.518,12.0,11.9,466.47,1.036,✔️,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),5573.0,+0/-0,11.536,0.499,13.0,12.51,2688.02,0.726,✔️,0.0,False,CPU +13,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),5519.0,+0/-0,11.505,0.501,13.56,13.39,0.39,0.051,✔️,0.0,False,CPU +14,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),5486.0,+0/-0,11.505,0.501,13.89,13.77,466.47,0.051,✔️,0.0,False,CPU +15,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),5145.0,+0/-0,13.544,0.399,17.0,16.0,1.48,0.53,✔️,0.0,False,GPU +16,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),5108.0,+0/-0,13.063,0.423,17.33,17.24,7.54,0.047,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),5082.0,+0/-0,13.396,0.407,17.56,17.52,4423.24,29.2,✔️,0.0,False,GPU +18,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),5032.0,+0/-0,13.739,0.389,18.0,17.84,17.7,0.086,✔️,0.0,False,CPU +19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),4812.0,+0/-0,14.412,0.354,19.67,19.62,4423.24,2.237,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),4739.0,+0/-0,14.523,0.348,20.11,20.02,2.82,1.073,✔️,0.0,False,GPU +21,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),3202.0,+0/-0,16.353,0.25,22.11,22.11,20.98,0.329,✔️,0.0,False,CPU +22,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),2532.0,+0/-0,18.457,0.132,24.22,24.17,4638.91,343.215,✔️,0.0,False,GPU +23,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),2475.0,+0/-0,18.732,0.115,24.56,24.47,2389.31,0.267,✔️,0.0,False,CPU +24,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),2475.0,+0/-0,18.777,0.113,24.56,24.51,20.98,0.027,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),2456.0,+0/-0,18.841,0.109,24.67,24.61,2389.31,0.023,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1738.0,+0/-0,20.674,0.0,27.44,27.43,14.87,1.855,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1554.0,+0/-0,20.901,0.0,28.33,28.3,1993.14,0.452,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1404.0,+0/-0,21.65,0.0,29.22,29.16,1993.14,0.059,✔️,0.0,False,GPU +29,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1218.0,+0/-0,22.097,0.0,30.44,30.4,4638.91,43.781,✔️,0.0,False,GPU +30,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1218.0,+0/-0,22.475,0.0,30.44,30.35,40.54,38.741,✔️,0.0,False,GPU +31,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+0/-0,23.341,0.0,31.89,31.77,0.18,0.008,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),821.0,+0/-0,23.568,0.0,33.0,32.98,1294.93,0.417,✔️,0.0,False,GPU +33,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),665.0,+0/-0,24.01,0.0,33.89,33.86,1294.93,0.06,✔️,0.0,False,GPU +34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),599.0,+0/-0,24.919,0.0,34.22,34.18,0.18,0.04,✔️,0.0,False,CPU +35,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),-215.0,+0/-0,28.274,0.0,36.11,36.11,1863.58,6.302,✔️,0.0,False,GPU +36,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),-734.0,+0/-0,30.79,0.0,37.56,37.54,1863.58,1.131,✔️,0.0,False,GPU +37,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),-788.0,+0/-0,30.768,0.0,37.78,37.75,3.96,0.025,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),-1010.0,+0/-0,32.321,0.0,38.67,38.66,6.85,0.088,✔️,0.0,False,GPU +39,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),-1462.0,+0/-0,33.496,0.0,39.89,39.89,66.06,1.792,➖,0.0,False,GPU +40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),-2063.0,+0/-0,37.017,0.0,41.0,41.0,7.55,0.149,✔️,0.0,False,GPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),-2631.0,+0/-0,41.044,0.0,42.11,42.11,569.0,0.17,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),-2955.0,+0/-0,41.644,0.0,42.89,42.89,569.0,1.658,✔️,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),-3626.0,+0/-0,50.207,0.0,44.11,44.11,0.07,0.015,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),-4076.0,+0/-0,51.289,0.0,45.22,45.21,2.86,0.085,✔️,0.0,False,CPU +45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),-4255.0,+0/-0,51.687,0.0,45.78,45.77,30.41,0.089,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),-4823.0,+0/-0,54.976,0.0,47.44,47.44,30.41,0.015,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),-5018.0,+0/-0,55.365,0.0,48.33,48.32,0.13,0.014,✔️,0.0,False,CPU +48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),-5106.0,+0/-0,55.242,0.0,48.78,48.72,3.09,0.997,✔️,0.0,False,GPU +49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),-5308.0,+0/-0,56.49,0.0,49.78,49.77,0.11,0.023,➖,0.0,False,GPU +50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),-5500.0,+0/-0,57.332,0.0,50.56,50.54,76.27,3.054,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),-6498.0,+0/-0,70.796,0.0,52.0,52.0,76.27,0.37,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-7418.0,+0/-0,82.671,0.0,53.0,53.0,0.14,0.136,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/winrate_matrix.png.zip index 52cec7ac1f22d5e4c918348b45e02c96f2af6d47..5f2e52f8a499c7a5bd2253453b4f8c57fcfd7478 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:61f905981f0117f3c6a4910114b1377423f4c83aa5cb4aad2b0a61fda9c5f907 +oid sha256:b402a5925b593bd073ebb55be666125b164b9f9e37e9e3ee50468e6e22be98c1 size 1124587 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 800360a1fd3e8a00e53bb9e9145138cda8c844e2..88d24611b00895d840d4189e4331086b797fbfe9 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a30f09a5b5502336a6c235b7334d2ef5b990b30845b4d0c605b4514c4af75d32 +oid sha256:54c9680cdaaa0a1a71e0332aca1b99376a8af55a0fea3df04a39f8b83f95f03e size 429560 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip index 4d2fa973f758e8ba1b41d8d375dc9e8098666a7d..d96c884a60f7ff9e1c3e2e24900edb29426d3560 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d8c6ab44c4c00f4c19a55698ecdf7a9ac4ffa5932c181961be54db61142d1c1a -size 964875 +oid sha256:f409d541e29be69f132cc8808d06d2c0ad9f6e4183abe290a548ee1a24798dab +size 917138 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip index de97703f69efefa6f00202ed59fd8da2e9479339..cbb9e2bd9afd3f5c1213c0a4c51db365c140c3c7 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4b533194d520b4d9f9b83923ed3eddb5aa381dffa6d3b75afbb8c207364a0c57 +oid sha256:77f2d62b3b704530de5c1e8c319ef204be0bf686d6bb745780f4727861494132 size 116913 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/website_leaderboard.csv index 1c7c333a9032490222f8291a0c6935941fc9cd2c..4e7a5ce23546742c2917640f5292677bb556db44 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/website_leaderboard.csv @@ -1,54 +1,54 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1780.0,+190/-78,0.821,6.49,3.79,10.304,1154.67,7.879,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1755.0,+239/-99,0.812,7.18,3.12,7.171,2742.37,1.218,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1736.0,+281/-137,0.793,7.75,2.66,6.592,2742.37,7.945,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1648.0,+320/-180,0.703,10.83,3.96,8.027,11.21,0.845,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1613.0,+210/-135,0.58,12.31,6.11,11.362,4471.59,21.354,0.0,False,GPU -5,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1607.0,+176/-118,0.554,12.54,4.81,10.368,6.22,1.092,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1586.0,+139/-72,0.498,13.48,7.68,12.584,4471.59,3.76,0.0,False,GPU -7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1547.0,+151/-75,0.456,15.33,9.15,16.178,5929.46,1.73,0.0,False,GPU -8,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1538.0,+225/-90,0.37,15.8,7.22,16.193,6548.58,2.898,0.0,False,CPU -9,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1500.0,+152/-80,0.399,17.71,9.33,17.039,5929.46,0.337,0.0,False,GPU -10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1485.0,+193/-157,0.422,18.51,7.3,18.473,4631.38,190.647,0.0,False,GPU -11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1481.0,+104/-65,0.293,18.73,15.09,15.318,3272.91,1.185,0.0,False,CPU -12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1474.0,+153/-80,0.298,19.1,14.57,17.306,1502.45,6.33,0.0,False,CPU -13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1469.0,+142/-72,0.274,19.4,15.11,17.496,1502.45,1.367,0.0,False,CPU -14,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1454.0,+136/-100,0.328,20.23,15.27,18.046,684.78,4.816,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1450.0,+99/-57,0.224,20.42,16.47,16.053,3272.91,0.21,0.0,False,CPU -16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1447.0,+112/-130,0.308,20.59,13.61,18.168,2097.62,3.142,0.0,False,GPU -17,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1441.0,+204/-163,0.384,20.91,8.16,20.039,4631.38,30.351,0.0,False,GPU -18,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1432.0,+139/-101,0.197,21.42,13.4,16.443,7359.32,9.026,0.0,False,GPU -19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1429.0,+124/-86,0.232,21.59,16.53,17.647,2551.65,1.021,0.0,False,CPU -20,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1429.0,+84/-52,0.167,21.61,10.61,16.711,7359.32,0.596,0.0,False,GPU -21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1418.0,+138/-86,0.22,22.21,16.65,17.847,2551.65,4.527,0.0,False,CPU -22,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1411.0,+177/-111,0.278,22.59,14.24,19.041,19.65,0.236,0.0,False,GPU -23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1358.0,+80/-78,0.132,25.65,21.96,20.132,63121.14,0.827,0.0,False,CPU -24,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1354.0,+196/-181,0.241,25.83,12.52,22.31,717.53,1.628,0.0,False,CPU -25,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1347.0,+181/-110,0.118,26.24,18.68,20.717,4142.25,2.105,0.0,False,CPU -26,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1346.0,+232/-170,0.252,26.32,11.2,21.631,2775.63,12.592,0.0,False,CPU -27,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1338.0,+83/-58,0.094,26.77,23.03,19.786,64.18,0.255,0.0,False,CPU -28,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1334.0,+100/-110,0.191,26.97,15.29,20.501,2097.62,0.515,0.0,False,GPU -29,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1318.0,+195/-204,0.249,27.92,12.51,22.379,755.05,2.155,0.0,False,CPU -30,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1308.0,+86/-80,0.076,28.47,23.93,20.964,63121.14,0.173,0.0,False,CPU -31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1308.0,+227/-212,0.252,28.5,7.67,23.486,46.25,45.74,0.0,False,GPU -32,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1304.0,+202/-208,0.213,28.68,11.66,23.665,717.53,0.339,0.0,False,CPU -33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1294.0,+144/-116,0.111,29.24,22.8,22.606,20.91,0.126,0.0,False,CPU -34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1287.0,+213/-154,0.136,29.67,16.06,22.991,2775.63,1.431,0.0,False,CPU -35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1278.0,+174/-162,0.154,30.14,23.14,22.23,5.32,0.431,0.0,False,CPU -36,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1270.0,+136/-128,0.105,30.56,20.74,23.361,4142.25,0.287,0.0,False,CPU -37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1260.0,+190/-218,0.175,31.12,10.31,23.912,755.05,0.317,0.0,False,CPU -38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1239.0,+52/-47,0.0,32.26,30.21,21.716,5.48,0.599,0.0,False,CPU -39,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1224.0,+112/-120,0.035,33.05,26.03,25.903,20.96,0.482,0.0,False,GPU -40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1213.0,+75/-81,0.0,33.64,30.7,24.273,19.63,7.686,0.0,False,GPU -41,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1159.0,+107/-157,0.061,36.35,30.53,25.552,10.84,2.189,0.0,False,GPU -42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1087.0,+76/-138,0.0,39.56,30.71,28.351,14.75,0.337,0.0,False,CPU -43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1079.0,+217/-372,0.046,39.91,20.86,36.721,173.37,1.638,0.0,False,CPU -44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1050.0,+107/-233,0.0,41.06,38.81,38.3,905.14,0.875,0.0,False,CPU -45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1016.0,+105/-262,0.0,42.37,39.96,38.97,905.14,0.175,0.0,False,CPU -46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+128/-228,0.0,42.93,39.25,37.253,0.85,0.148,0.0,False,CPU -47,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,997.0,+163/-196,0.0,43.04,40.46,32.224,8.97,0.743,0.0,False,CPU -48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,943.0,+161/-236,0.0,44.77,41.62,39.279,0.78,0.147,0.0,False,CPU -49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,904.0,+45/-136,0.0,45.89,45.69,37.24,5.83,1.167,0.0,False,GPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,897.0,+264/-569,0.03,46.07,35.58,46.585,173.37,0.147,0.0,False,CPU -51,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,786.0,+98/-282,0.0,48.56,48.16,45.98,4.72,0.188,0.0,False,CPU -52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,268.0,+117/-657,0.0,52.74,52.72,68.787,0.57,0.067,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1780.0,+190/-78,10.304,0.821,6.49,3.79,1154.67,7.879,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1755.0,+239/-99,7.171,0.812,7.18,3.12,2742.37,1.218,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1736.0,+281/-137,6.592,0.793,7.75,2.66,2742.37,7.945,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1648.0,+320/-180,8.027,0.703,10.83,3.96,11.21,0.845,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1613.0,+210/-135,11.362,0.58,12.31,6.11,4471.59,21.354,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1607.0,+176/-118,10.368,0.554,12.54,4.81,6.22,1.092,➖,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1586.0,+139/-72,12.584,0.498,13.48,7.68,4471.59,3.76,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1547.0,+151/-75,16.178,0.456,15.33,9.15,5929.46,1.73,✔️,0.0,False,GPU +8,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1538.0,+225/-90,16.193,0.37,15.8,7.22,6548.58,2.898,✔️,0.0,False,CPU +9,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1500.0,+152/-80,17.039,0.399,17.71,9.33,5929.46,0.337,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1485.0,+193/-157,18.473,0.422,18.51,7.3,4631.38,190.647,✔️,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1481.0,+104/-65,15.318,0.293,18.73,15.09,3272.91,1.185,✔️,0.0,False,CPU +12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1474.0,+153/-80,17.306,0.298,19.1,14.57,1502.45,6.33,✔️,0.0,False,CPU +13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1469.0,+142/-72,17.496,0.274,19.4,15.11,1502.45,1.367,✔️,0.0,False,CPU +14,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1454.0,+136/-100,18.046,0.328,20.23,15.27,684.78,4.816,➖,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1450.0,+99/-57,16.053,0.224,20.42,16.47,3272.91,0.21,✔️,0.0,False,CPU +16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1447.0,+112/-130,18.168,0.308,20.59,13.61,2097.62,3.142,✔️,0.0,False,GPU +17,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1441.0,+204/-163,20.039,0.384,20.91,8.16,4631.38,30.351,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1432.0,+139/-101,16.443,0.197,21.42,13.4,7359.32,9.026,✔️,0.0,False,GPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1429.0,+124/-86,17.647,0.232,21.59,16.53,2551.65,1.021,✔️,0.0,False,CPU +20,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1429.0,+84/-52,16.711,0.167,21.61,10.61,7359.32,0.596,✔️,0.0,False,GPU +21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1418.0,+138/-86,17.847,0.22,22.21,16.65,2551.65,4.527,✔️,0.0,False,CPU +22,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1411.0,+177/-111,19.041,0.278,22.59,14.24,19.65,0.236,✔️,0.0,False,GPU +23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1358.0,+80/-78,20.132,0.132,25.65,21.96,63121.14,0.827,✔️,0.0,False,CPU +24,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1354.0,+196/-181,22.31,0.241,25.83,12.52,717.53,1.628,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1347.0,+181/-110,20.717,0.118,26.24,18.68,4142.25,2.105,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1346.0,+232/-170,21.631,0.252,26.32,11.2,2775.63,12.592,✔️,0.0,False,CPU +27,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1338.0,+83/-58,19.786,0.094,26.77,23.03,64.18,0.255,✔️,0.0,False,CPU +28,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1334.0,+100/-110,20.501,0.191,26.97,15.29,2097.62,0.515,✔️,0.0,False,GPU +29,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1318.0,+195/-204,22.379,0.249,27.92,12.51,755.05,2.155,✔️,0.0,False,CPU +30,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1308.0,+86/-80,20.964,0.076,28.47,23.93,63121.14,0.173,✔️,0.0,False,CPU +31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1308.0,+227/-212,23.486,0.252,28.5,7.67,46.25,45.74,✔️,0.0,False,GPU +32,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1304.0,+202/-208,23.665,0.213,28.68,11.66,717.53,0.339,✔️,0.0,False,CPU +33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1294.0,+144/-116,22.606,0.111,29.24,22.8,20.91,0.126,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1287.0,+213/-154,22.991,0.136,29.67,16.06,2775.63,1.431,✔️,0.0,False,CPU +35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1278.0,+174/-162,22.23,0.154,30.14,23.14,5.32,0.431,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1270.0,+136/-128,23.361,0.105,30.56,20.74,4142.25,0.287,✔️,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1260.0,+190/-218,23.912,0.175,31.12,10.31,755.05,0.317,✔️,0.0,False,CPU +38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1239.0,+52/-47,21.716,0.0,32.26,30.21,5.48,0.599,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1224.0,+112/-120,25.903,0.035,33.05,26.03,20.96,0.482,✔️,0.0,False,GPU +40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1213.0,+75/-81,24.273,0.0,33.64,30.7,19.63,7.686,✔️,0.0,False,GPU +41,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1159.0,+107/-157,25.552,0.061,36.35,30.53,10.84,2.189,✔️,0.0,False,GPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1087.0,+76/-138,28.351,0.0,39.56,30.71,14.75,0.337,✔️,0.0,False,CPU +43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1079.0,+217/-372,36.721,0.046,39.91,20.86,173.37,1.638,✔️,0.0,False,CPU +44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1050.0,+107/-233,38.3,0.0,41.06,38.81,905.14,0.875,✔️,0.0,False,CPU +45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1016.0,+105/-262,38.97,0.0,42.37,39.96,905.14,0.175,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+128/-228,37.253,0.0,42.93,39.25,0.85,0.148,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),997.0,+163/-196,32.224,0.0,43.04,40.46,8.97,0.743,✔️,0.0,False,CPU +48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),943.0,+161/-236,39.279,0.0,44.77,41.62,0.78,0.147,✔️,0.0,False,CPU +49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),904.0,+45/-136,37.24,0.0,45.89,45.69,5.83,1.167,➖,0.0,False,GPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),897.0,+264/-569,46.585,0.03,46.07,35.58,173.37,0.147,✔️,0.0,False,CPU +51,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),786.0,+98/-282,45.98,0.0,48.56,48.16,4.72,0.188,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),268.0,+117/-657,68.787,0.0,52.74,52.72,0.57,0.067,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/winrate_matrix.png.zip index 2156f75a0a4a0cc402f09864aaf97334150b2a95..acefb7f7c6a3194cc68db9a220d43ac134de9d52 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3b9b0cb60880aa87d9b98a26d5141a2d042c0e1f1238379454e72345cded33ce +oid sha256:85c7aa4c385a4748735564be0568f539d0d67e5239f3e6e80b8a548e9b66d69b size 2155304 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 228d094d15dd97439c7f8da4a003bfbe31c69b60..7f9727b6ba52ecc7a8f58d68658694be95bb94ea 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1b94285b128c76957a3efbb3a35732cd5e8b1346d7772a9f3ca6ceaa566af0eb +oid sha256:adc3070ee3202521b270613c1b0868ee11a3bd34faf5b3003d3a6b6a479ba922 size 470825 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip index 5df18ea584e78b74cd3709955baa605938e7004f..e0dc53880351c7747d9f4530aa31aee7e0058a6e 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:16c46bd27ffddf88aa29a898b88f0b063724f2a5658dc80c4ac391b5bfd2be8f -size 1035459 +oid sha256:bc4890a933182e29e072d239ee00b8f3fa5a480ed611c6d2df3f1871baea20c2 +size 993086 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip index 15df7042483862528755d89ac375babb50464fcd..6de5a9cb85033d171d50d972944fd7b37b86842d 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:06a34595139381a3b5d4bf7285bda5f1a8399d132ddf5a6057df4457303cb616 +oid sha256:fa8ef0077128aadf5752c93607dd57ad048d5d6b39a36abb7dd6e1a78d353097 size 122677 diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv index fe131ba2fc1d272b12013893e0a5463f60893637..76f7eaabc270279279db328ea3355f3fc07d6416 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1876.0,+270/-100,0.87,6.14,2.55,7.591,2710.27,7.889,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1852.0,+240/-92,0.851,6.75,3.29,8.338,2710.27,1.076,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1838.0,+207/-77,0.833,7.12,4.7,12.046,1374.26,12.843,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1799.0,+306/-137,0.807,8.24,3.94,8.785,10.23,0.844,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1788.0,+296/-112,0.791,8.58,4.37,12.979,3585.96,32.815,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1708.0,+168/-44,0.653,11.4,6.17,12.954,4150.25,19.131,0.0,False,GPU -6,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1688.0,+166/-61,0.628,12.2,5.26,11.831,5.85,0.91,0.0,False,GPU -7,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1680.0,+197/-107,0.622,12.51,7.31,15.607,205.23,4.488,0.0,False,GPU -8,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1633.0,+220/-107,0.562,14.52,6.55,15.646,3585.96,1.167,0.0,False,GPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1630.0,+135/-43,0.553,14.68,8.59,14.49,4150.25,3.423,0.0,False,GPU -10,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1592.0,+203/-102,0.476,16.45,9.67,13.798,6.47,0.427,0.0,False,GPU -11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1575.0,+284/-135,0.414,17.28,7.5,18.07,7307.18,4.205,0.0,False,CPU -12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1549.0,+239/-110,0.412,18.59,10.08,18.854,5885.87,1.632,0.0,False,GPU -13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1513.0,+108/-46,0.303,20.47,17.01,17.651,3104.39,1.095,0.0,False,CPU -14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1497.0,+197/-114,0.346,21.34,10.08,19.852,5885.87,0.299,0.0,False,GPU -15,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1481.0,+240/-202,0.377,22.22,8.34,21.539,4513.63,201.505,0.0,False,GPU -16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1474.0,+122/-64,0.203,22.61,14.46,18.788,7367.31,7.816,0.0,False,GPU -17,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1470.0,+95/-49,0.218,22.86,18.63,18.528,3104.39,0.18,0.0,False,CPU -18,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1468.0,+142/-63,0.236,22.95,18.11,20.186,2210.02,1.236,0.0,False,CPU -19,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1460.0,+224/-87,0.219,23.37,17.51,20.164,1377.29,6.652,0.0,False,CPU -20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1455.0,+147/-63,0.221,23.69,18.61,20.414,2210.02,7.723,0.0,False,CPU -21,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1450.0,+196/-70,0.188,23.97,18.81,20.379,1377.29,1.395,0.0,False,CPU -22,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1433.0,+202/-128,0.245,24.92,19.16,21.045,696.38,4.428,0.0,False,GPU -23,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1431.0,+237/-220,0.338,25.04,8.66,23.355,4513.63,26.967,0.0,False,GPU -24,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1424.0,+226/-151,0.294,25.44,10.59,21.754,12.57,2.243,0.0,False,GPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1420.0,+212/-145,0.267,25.68,15.1,22.067,17.83,0.197,0.0,False,GPU -26,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1412.0,+144/-153,0.205,26.15,21.17,21.235,1880.81,2.744,0.0,False,GPU -27,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1410.0,+86/-72,0.092,26.26,14.18,19.439,7367.31,0.587,0.0,False,GPU -28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1381.0,+315/-274,0.276,27.89,11.42,24.53,2644.99,12.716,0.0,False,CPU -29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1347.0,+200/-164,0.121,29.89,21.48,23.884,4654.35,2.028,0.0,False,CPU -30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1340.0,+132/-106,0.076,30.27,25.3,23.387,36484.23,0.712,0.0,False,CPU -31,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1320.0,+134/-78,0.06,31.45,27.13,22.957,264.83,0.227,0.0,False,CPU -32,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1318.0,+331/-279,0.291,31.55,7.6,27.104,40.51,46.546,0.0,False,GPU -33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1289.0,+146/-150,0.099,33.19,27.21,23.905,1880.81,0.353,0.0,False,GPU -34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1289.0,+262/-248,0.134,33.2,18.09,26.079,2644.99,1.47,0.0,False,CPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1282.0,+107/-99,0.027,33.59,27.61,24.339,36484.23,0.127,0.0,False,CPU -36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1278.0,+176/-198,0.106,33.8,27.13,26.121,717.36,1.838,0.0,False,CPU -37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1255.0,+115/-83,0.029,35.05,28.39,26.62,17.94,0.418,0.0,False,GPU -38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1247.0,+64/-58,0.0,35.52,33.72,24.705,4.72,0.714,0.0,False,CPU -39,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1239.0,+180/-243,0.119,35.94,22.24,26.195,737.91,2.289,0.0,False,CPU -40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1232.0,+70/-89,0.0,36.29,33.66,26.605,20.73,8.179,0.0,False,GPU -41,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1230.0,+181/-140,0.06,36.45,24.6,27.149,4654.35,0.223,0.0,False,CPU -42,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1223.0,+101/-89,0.0,36.81,34.51,26.38,14.75,0.108,0.0,False,CPU -43,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1212.0,+159/-217,0.065,37.39,29.45,27.709,717.36,0.326,0.0,False,CPU -44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1204.0,+196/-142,0.055,37.78,34.29,25.943,5.04,0.449,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1160.0,+160/-231,0.033,40.02,25.17,27.997,737.91,0.316,0.0,False,CPU -46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1156.0,+183/-247,0.067,40.22,33.93,29.106,9.81,2.345,0.0,False,GPU -47,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1069.0,+131/-286,0.0,44.15,42.1,36.78,892.95,0.895,0.0,False,CPU -48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1066.0,+132/-195,0.0,44.29,33.03,32.333,13.69,0.314,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1031.0,+119/-328,0.0,45.67,43.14,37.562,892.95,0.146,0.0,False,CPU -50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+170/-313,0.0,46.83,42.97,37.99,0.74,0.137,0.0,False,CPU -51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,985.0,+174/-259,0.0,47.35,44.55,34.643,8.51,0.828,0.0,False,CPU -52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,954.0,+212/-444,0.0,48.38,41.73,42.698,151.69,1.252,0.0,False,CPU -53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,932.0,+195/-360,0.0,49.06,45.92,40.173,0.69,0.137,0.0,False,CPU -54,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,851.0,+29/-142,0.0,51.27,51.09,42.312,5.05,1.013,0.0,False,GPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,775.0,+103/-312,0.0,52.93,52.56,43.542,4.53,0.164,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,580.0,+156/-543,0.0,55.58,55.16,54.165,151.69,0.099,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,178.0,+157/-1027,0.0,57.75,57.73,69.817,0.52,0.056,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1876.0,+270/-100,7.591,0.87,6.14,2.55,2710.27,7.889,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1852.0,+240/-92,8.338,0.851,6.75,3.29,2710.27,1.076,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1838.0,+207/-77,12.046,0.833,7.12,4.7,1374.26,12.843,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1799.0,+306/-137,8.785,0.807,8.24,3.94,10.23,0.844,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1788.0,+296/-112,12.979,0.791,8.58,4.37,3585.96,32.815,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1708.0,+168/-44,12.954,0.653,11.4,6.17,4150.25,19.131,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1688.0,+166/-61,11.831,0.628,12.2,5.26,5.85,0.91,➖,0.0,False,GPU +7,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1680.0,+197/-107,15.607,0.622,12.51,7.31,205.23,4.488,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1633.0,+220/-107,15.646,0.562,14.52,6.55,3585.96,1.167,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1630.0,+135/-43,14.49,0.553,14.68,8.59,4150.25,3.423,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1592.0,+203/-102,13.798,0.476,16.45,9.67,6.47,0.427,✔️,0.0,False,GPU +11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1575.0,+284/-135,18.07,0.414,17.28,7.5,7307.18,4.205,✔️,0.0,False,CPU +12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1549.0,+239/-110,18.854,0.412,18.59,10.08,5885.87,1.632,✔️,0.0,False,GPU +13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1513.0,+108/-46,17.651,0.303,20.47,17.01,3104.39,1.095,✔️,0.0,False,CPU +14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1497.0,+197/-114,19.852,0.346,21.34,10.08,5885.87,0.299,✔️,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1481.0,+240/-202,21.539,0.377,22.22,8.34,4513.63,201.505,✔️,0.0,False,GPU +16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1474.0,+122/-64,18.788,0.203,22.61,14.46,7367.31,7.816,✔️,0.0,False,GPU +17,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1470.0,+95/-49,18.528,0.218,22.86,18.63,3104.39,0.18,✔️,0.0,False,CPU +18,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1468.0,+142/-63,20.186,0.236,22.95,18.11,2210.02,1.236,✔️,0.0,False,CPU +19,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1460.0,+224/-87,20.164,0.219,23.37,17.51,1377.29,6.652,✔️,0.0,False,CPU +20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1455.0,+147/-63,20.414,0.221,23.69,18.61,2210.02,7.723,✔️,0.0,False,CPU +21,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1450.0,+196/-70,20.379,0.188,23.97,18.81,1377.29,1.395,✔️,0.0,False,CPU +22,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1433.0,+202/-128,21.045,0.245,24.92,19.16,696.38,4.428,➖,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1431.0,+237/-220,23.355,0.338,25.04,8.66,4513.63,26.967,✔️,0.0,False,GPU +24,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1424.0,+226/-151,21.754,0.294,25.44,10.59,12.57,2.243,✔️,0.0,False,GPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1420.0,+212/-145,22.067,0.267,25.68,15.1,17.83,0.197,✔️,0.0,False,GPU +26,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1412.0,+144/-153,21.235,0.205,26.15,21.17,1880.81,2.744,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1410.0,+86/-72,19.439,0.092,26.26,14.18,7367.31,0.587,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1381.0,+315/-274,24.53,0.276,27.89,11.42,2644.99,12.716,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1347.0,+200/-164,23.884,0.121,29.89,21.48,4654.35,2.028,✔️,0.0,False,CPU +30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1340.0,+132/-106,23.387,0.076,30.27,25.3,36484.23,0.712,✔️,0.0,False,CPU +31,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1320.0,+134/-78,22.957,0.06,31.45,27.13,264.83,0.227,✔️,0.0,False,CPU +32,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1318.0,+331/-279,27.104,0.291,31.55,7.6,40.51,46.546,✔️,0.0,False,GPU +33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1289.0,+146/-150,23.905,0.099,33.19,27.21,1880.81,0.353,✔️,0.0,False,GPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1289.0,+262/-248,26.079,0.134,33.2,18.09,2644.99,1.47,✔️,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1282.0,+107/-99,24.339,0.027,33.59,27.61,36484.23,0.127,✔️,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1278.0,+176/-198,26.121,0.106,33.8,27.13,717.36,1.838,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1255.0,+115/-83,26.62,0.029,35.05,28.39,17.94,0.418,✔️,0.0,False,GPU +38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1247.0,+64/-58,24.705,0.0,35.52,33.72,4.72,0.714,✔️,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1239.0,+180/-243,26.195,0.119,35.94,22.24,737.91,2.289,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1232.0,+70/-89,26.605,0.0,36.29,33.66,20.73,8.179,✔️,0.0,False,GPU +41,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1230.0,+181/-140,27.149,0.06,36.45,24.6,4654.35,0.223,✔️,0.0,False,CPU +42,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1223.0,+101/-89,26.38,0.0,36.81,34.51,14.75,0.108,✔️,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1212.0,+159/-217,27.709,0.065,37.39,29.45,717.36,0.326,✔️,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1204.0,+196/-142,25.943,0.055,37.78,34.29,5.04,0.449,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1160.0,+160/-231,27.997,0.033,40.02,25.17,737.91,0.316,✔️,0.0,False,CPU +46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1156.0,+183/-247,29.106,0.067,40.22,33.93,9.81,2.345,✔️,0.0,False,GPU +47,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1069.0,+131/-286,36.78,0.0,44.15,42.1,892.95,0.895,✔️,0.0,False,CPU +48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1066.0,+132/-195,32.333,0.0,44.29,33.03,13.69,0.314,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1031.0,+119/-328,37.562,0.0,45.67,43.14,892.95,0.146,✔️,0.0,False,CPU +50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+170/-313,37.99,0.0,46.83,42.97,0.74,0.137,✔️,0.0,False,CPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),985.0,+174/-259,34.643,0.0,47.35,44.55,8.51,0.828,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),954.0,+212/-444,42.698,0.0,48.38,41.73,151.69,1.252,✔️,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),932.0,+195/-360,40.173,0.0,49.06,45.92,0.69,0.137,✔️,0.0,False,CPU +54,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),851.0,+29/-142,42.312,0.0,51.27,51.09,5.05,1.013,➖,0.0,False,GPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),775.0,+103/-312,43.542,0.0,52.93,52.56,4.53,0.164,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),580.0,+156/-543,54.165,0.0,55.58,55.16,151.69,0.099,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),178.0,+157/-1027,69.817,0.0,57.75,57.73,0.52,0.056,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip index 3d9cb05d365111ebbcf4ca4bf7c1729fd581d7c3..f9369f578812394cc77fb5dc2be4a0855d94677c 100644 --- a/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:338def1117e6a5f95eff7222b2bcbabb135b83c7c94b5554c62379a384092ff9 +oid sha256:fd0f60c3e4b688ef0737f83dcf74f06811cce4837bb6e781879bcdec34214d2d size 2656649 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index 43f5626eb1eaaf4e4730c5856f7dd1c781fd80ad..ebd6a4e73a6ef2725059aacb7bef29e9c80d7f6c 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4195800e25572a8cf53c95aa8c69184693b7a6059dce16df3ad7a7d3f969d5aa +oid sha256:2dde6e64557305b14e1790b0dc1baf513553178052b4ec48416b28e00c400cae size 426338 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip index 130ee7b391adc274659c9dfa7c66dfe4ab063b5a..0ed3362c7a2ececff5e5e14cf5f5773b1ae4b692 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7be5821fb3d875758fefb2469588323f10272cbfde09df1f9dc09e595f006f05 -size 981158 +oid sha256:fad1246a0a408de16105ba444d7223cc1fa3968f2f552abcd95e669d774adad5 +size 955147 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_all/tuning-impact-elo.png.zip index ac11590dcbfa97bb300b20442e4010a2f35fec89..d29f80872247c959aba5ab6242db991c353f6c39 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:32f6b5437f704b5e45fe03193895778030c08a6a24725b2343dee80043c431fb +oid sha256:72aa314fa5566052afb622b54aa1eee415f9c752941946ebd4bcb98eda09e086 size 117379 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_regression/datasets_all/website_leaderboard.csv index de82f05722078727620c0c6195bec24da34e9a98..1c8baffa1660d8d9fa27d71b937f731283ea2a14 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_all/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_regression/datasets_all/website_leaderboard.csv @@ -1,52 +1,52 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1816.0,+189/-125,0.812,6.26,2.56,1.486,1709.05,8.122,0.0,False,GPU -1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1783.0,+136/-84,0.791,7.1,2.64,2.397,4786.55,239.537,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1760.0,+117/-75,0.696,7.73,3.92,2.908,649.34,5.869,0.0,False,GPU -3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1730.0,+107/-72,0.729,8.63,4.27,2.848,4786.55,38.501,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1728.0,+113/-66,0.677,8.69,5.41,3.196,3995.01,10.051,0.0,False,GPU -5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1689.0,+174/-116,0.697,9.94,4.34,2.246,1709.05,0.812,0.0,False,GPU -6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1675.0,+108/-89,0.567,10.44,6.37,4.64,1866.07,6.07,0.0,False,CPU -7,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1632.0,+135/-71,0.601,11.98,4.77,3.791,46.62,39.212,0.0,False,GPU -8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1606.0,+109/-79,0.517,12.99,7.92,4.177,3995.01,0.844,0.0,False,GPU -9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1600.0,+132/-117,0.506,13.21,5.41,5.528,3779.52,7.69,0.0,False,GPU -10,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1600.0,+168/-119,0.557,13.21,7.34,3.758,7.04,0.508,0.0,False,GPU -11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1535.0,+106/-61,0.369,15.79,9.82,6.221,3552.96,0.966,0.0,False,CPU -12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1527.0,+93/-80,0.361,16.1,12.59,6.683,700.15,9.322,0.0,False,CPU -13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1516.0,+120/-91,0.379,16.58,10.22,5.711,714.5,1.376,0.0,False,GPU -14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1509.0,+110/-76,0.348,16.87,9.48,6.341,3552.96,0.105,0.0,False,CPU -15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1496.0,+140/-90,0.349,17.46,12.61,5.114,4158.29,1.41,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1458.0,+87/-74,0.274,19.17,15.01,7.275,700.15,0.968,0.0,False,CPU -17,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1447.0,+191/-193,0.423,19.66,5.5,13.147,5.2,0.755,0.0,False,GPU -18,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1442.0,+56/-45,0.255,19.92,17.26,7.217,834.93,2.614,0.0,False,CPU -19,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1426.0,+95/-84,0.262,20.63,16.37,6.387,714.5,0.099,0.0,False,GPU -20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1420.0,+60/-49,0.221,20.92,17.74,7.365,834.93,0.388,0.0,False,CPU -21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1419.0,+116/-86,0.265,20.96,12.86,7.527,3779.52,0.396,0.0,False,GPU -22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1409.0,+128/-104,0.25,21.44,14.82,6.071,4158.29,0.167,0.0,False,GPU -23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1404.0,+106/-103,0.262,21.68,15.62,7.922,10.89,0.092,0.0,False,CPU -24,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1307.0,+92/-77,0.118,26.06,20.62,9.067,15.5,0.299,0.0,False,GPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1304.0,+146/-119,0.161,26.23,20.14,7.789,13.32,0.131,0.0,False,GPU -26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1275.0,+127/-103,0.116,27.53,19.36,9.279,4608.59,1.233,0.0,False,CPU -27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1262.0,+132/-92,0.087,28.11,21.77,8.803,8.9,1.645,0.0,False,GPU -28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1234.0,+125/-104,0.107,29.43,21.54,11.617,158.22,0.844,0.0,False,CPU -29,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1226.0,+40/-39,0.005,29.76,28.4,9.708,2.11,0.275,0.0,False,CPU -30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1211.0,+115/-100,0.081,30.44,22.91,10.071,4608.59,0.097,0.0,False,CPU -31,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1206.0,+111/-97,0.082,30.68,20.81,11.83,158.22,0.151,0.0,False,CPU -32,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1204.0,+84/-82,0.061,30.79,27.39,10.368,2.24,0.242,0.0,False,CPU -33,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1179.0,+133/-168,0.117,31.89,23.15,12.864,2929.85,0.294,0.0,False,CPU -34,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1172.0,+114/-119,0.039,32.19,24.29,12.134,2.45,0.743,0.0,False,GPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1170.0,+70/-88,0.035,32.28,29.71,12.388,515.73,0.771,0.0,False,CPU -36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1133.0,+66/-94,0.025,33.8,31.32,12.876,515.73,0.124,0.0,False,CPU -37,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1129.0,+136/-171,0.088,33.95,24.71,13.402,2929.85,0.03,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1084.0,+104/-126,0.014,35.69,31.52,13.769,0.47,0.055,0.0,False,CPU -39,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1060.0,+128/-181,0.049,36.6,31.21,14.338,8.47,0.035,0.0,False,CPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1057.0,+101/-120,0.02,36.71,34.25,13.746,540.06,2.672,0.0,False,CPU -41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1043.0,+84/-102,0.0,37.2,35.18,13.502,20.48,0.08,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1000.0,+97/-105,0.01,38.71,37.01,14.233,540.06,0.325,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+54/-79,0.0,38.71,37.38,14.393,0.53,0.062,0.0,False,CPU -44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,912.0,+150/-180,0.007,41.42,36.02,19.563,92.55,0.897,0.0,False,CPU -45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,869.0,+106/-170,0.0,42.56,41.65,18.494,2.6,0.392,0.0,False,CPU -46,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,806.0,+150/-188,0.0,43.99,39.8,21.987,92.55,0.052,0.0,False,CPU -47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,672.0,+199/-297,0.0,46.39,39.43,29.17,0.19,0.04,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,506.0,+138/-315,0.0,48.39,48.21,36.388,193.95,0.173,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,459.0,+162/-378,0.0,48.83,48.63,36.561,193.95,0.074,0.0,False,CPU -50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,248.0,+141/-390,0.0,50.31,50.29,38.996,0.95,0.097,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1816.0,+189/-125,1.486,0.812,6.26,2.56,1709.05,8.122,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1783.0,+136/-84,2.397,0.791,7.1,2.64,4786.55,239.537,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1760.0,+117/-75,2.908,0.696,7.73,3.92,649.34,5.869,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1730.0,+107/-72,2.848,0.729,8.63,4.27,4786.55,38.501,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1728.0,+113/-66,3.196,0.677,8.69,5.41,3995.01,10.051,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1689.0,+174/-116,2.246,0.697,9.94,4.34,1709.05,0.812,✔️,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1675.0,+108/-89,4.64,0.567,10.44,6.37,1866.07,6.07,✔️,0.0,False,CPU +7,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1632.0,+135/-71,3.791,0.601,11.98,4.77,46.62,39.212,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1606.0,+109/-79,4.177,0.517,12.99,7.92,3995.01,0.844,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1600.0,+132/-117,5.528,0.506,13.21,5.41,3779.52,7.69,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1600.0,+168/-119,3.758,0.557,13.21,7.34,7.04,0.508,✔️,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1535.0,+106/-61,6.221,0.369,15.79,9.82,3552.96,0.966,✔️,0.0,False,CPU +12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1527.0,+93/-80,6.683,0.361,16.1,12.59,700.15,9.322,✔️,0.0,False,CPU +13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1516.0,+120/-91,5.711,0.379,16.58,10.22,714.5,1.376,✔️,0.0,False,GPU +14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1509.0,+110/-76,6.341,0.348,16.87,9.48,3552.96,0.105,✔️,0.0,False,CPU +15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1496.0,+140/-90,5.114,0.349,17.46,12.61,4158.29,1.41,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1458.0,+87/-74,7.275,0.274,19.17,15.01,700.15,0.968,✔️,0.0,False,CPU +17,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1447.0,+191/-193,13.147,0.423,19.66,5.5,5.2,0.755,➖,0.0,False,GPU +18,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1442.0,+56/-45,7.217,0.255,19.92,17.26,834.93,2.614,✔️,0.0,False,CPU +19,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1426.0,+95/-84,6.387,0.262,20.63,16.37,714.5,0.099,✔️,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1420.0,+60/-49,7.365,0.221,20.92,17.74,834.93,0.388,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1419.0,+116/-86,7.527,0.265,20.96,12.86,3779.52,0.396,✔️,0.0,False,GPU +22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1409.0,+128/-104,6.071,0.25,21.44,14.82,4158.29,0.167,✔️,0.0,False,GPU +23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1404.0,+106/-103,7.922,0.262,21.68,15.62,10.89,0.092,✔️,0.0,False,CPU +24,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1307.0,+92/-77,9.067,0.118,26.06,20.62,15.5,0.299,✔️,0.0,False,GPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1304.0,+146/-119,7.789,0.161,26.23,20.14,13.32,0.131,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1275.0,+127/-103,9.279,0.116,27.53,19.36,4608.59,1.233,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1262.0,+132/-92,8.803,0.087,28.11,21.77,8.9,1.645,✔️,0.0,False,GPU +28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1234.0,+125/-104,11.617,0.107,29.43,21.54,158.22,0.844,✔️,0.0,False,CPU +29,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1226.0,+40/-39,9.708,0.005,29.76,28.4,2.11,0.275,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1211.0,+115/-100,10.071,0.081,30.44,22.91,4608.59,0.097,✔️,0.0,False,CPU +31,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1206.0,+111/-97,11.83,0.082,30.68,20.81,158.22,0.151,✔️,0.0,False,CPU +32,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1204.0,+84/-82,10.368,0.061,30.79,27.39,2.24,0.242,✔️,0.0,False,CPU +33,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1179.0,+133/-168,12.864,0.117,31.89,23.15,2929.85,0.294,✔️,0.0,False,CPU +34,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1172.0,+114/-119,12.134,0.039,32.19,24.29,2.45,0.743,✔️,0.0,False,GPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1170.0,+70/-88,12.388,0.035,32.28,29.71,515.73,0.771,✔️,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1133.0,+66/-94,12.876,0.025,33.8,31.32,515.73,0.124,✔️,0.0,False,CPU +37,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1129.0,+136/-171,13.402,0.088,33.95,24.71,2929.85,0.03,✔️,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1084.0,+104/-126,13.769,0.014,35.69,31.52,0.47,0.055,✔️,0.0,False,CPU +39,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1060.0,+128/-181,14.338,0.049,36.6,31.21,8.47,0.035,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1057.0,+101/-120,13.746,0.02,36.71,34.25,540.06,2.672,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1043.0,+84/-102,13.502,0.0,37.2,35.18,20.48,0.08,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1000.0,+97/-105,14.233,0.01,38.71,37.01,540.06,0.325,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+54/-79,14.393,0.0,38.71,37.38,0.53,0.062,✔️,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),912.0,+150/-180,19.563,0.007,41.42,36.02,92.55,0.897,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),869.0,+106/-170,18.494,0.0,42.56,41.65,2.6,0.392,✔️,0.0,False,CPU +46,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),806.0,+150/-188,21.987,0.0,43.99,39.8,92.55,0.052,✔️,0.0,False,CPU +47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),672.0,+199/-297,29.17,0.0,46.39,39.43,0.19,0.04,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),506.0,+138/-315,36.388,0.0,48.39,48.21,193.95,0.173,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),459.0,+162/-378,36.561,0.0,48.83,48.63,193.95,0.074,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),248.0,+141/-390,38.996,0.0,50.31,50.29,0.95,0.097,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_all/winrate_matrix.png.zip index fe2eda6839fc2998126bd6c7477fe1bb0a5e916d..2c24c72725d42805b0c06c332f9394c206f218d8 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:db3b4b0ec1158ab053e54d2eee5caf5681bea698ded84437184856322d3c5dea +oid sha256:e93756f8142d92fec7354da521d9403a7c3f48839093c7e45ae10c72ba8a1ccf size 1808653 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index e0d4cc2eb3cecb1cfd968589453176bf224e22c5..f95d19af568ee2c0f3fb785ab82ca906c0224888 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7d8541e458f7a4a5ba3b69544da2bf1b07b75cc99a9760f65cf588d306d4acc3 +oid sha256:5b07713b4a66db34a4d074a568daff532d3fd43db354f700b23dd24f5874218f size 429005 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip index a7f74017c978bd88f65210914d58ed4652b8d43a..34e86756eebe9efba772006f1afadb29d833c57f 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:dfffe6e1a4b8a2181b87380c90a9b0959aec4c13e1af4d035a3003f1f28273ae -size 984608 +oid sha256:a9539158468ed4b66e71b28e2b8859bfa9e2b32fc149422010e9ea334685f276 +size 928150 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_medium/tuning-impact-elo.png.zip index 27849f9a35c081a2db43c506b9144674919ac336..f019a4f0a6279abf4f388f9aaf3fba6802fdb016 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:103a335ae2472aee78c057a242a72bce572ac210c3c04320b4a599772c41ad85 +oid sha256:429741eeb2a12d29145d231294aab5a683f0da69b9b15c8c7b264aebf63ee32c size 119720 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_regression/datasets_medium/website_leaderboard.csv index d41a2046e31a460a63a19e7ed98b54b547edec91..e3cd8978a55c78ce453557b9d24fccd2ef76e30c 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_medium/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_regression/datasets_medium/website_leaderboard.csv @@ -1,52 +1,52 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,2069.0,+1045/-120,0.862,4.4,2.67,1.209,735.58,9.557,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1971.0,+783/-126,0.802,6.31,4.21,1.696,735.58,0.714,0.0,False,GPU -2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1922.0,+681/-41,0.713,7.47,5.68,2.304,2526.28,3.717,0.0,False,GPU -3,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1917.0,+1210/-123,0.782,7.6,2.16,1.293,4786.55,361.217,0.0,False,GPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1909.0,+515/-11,0.661,7.8,4.92,3.331,473.01,8.73,0.0,False,CPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1900.0,+797/-114,0.688,8.04,5.61,2.7,2028.29,1.675,0.0,False,GPU -6,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1884.0,+532/-21,0.644,8.47,4.35,3.488,430.4,7.174,0.0,False,GPU -7,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1850.0,+1175/-128,0.717,9.42,4.22,1.707,4786.55,41.599,0.0,False,GPU -8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1802.0,+856/-218,0.564,10.87,7.04,2.636,1.6,0.508,0.0,False,GPU -9,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1753.0,+1086/-143,0.591,12.42,4.95,2.355,45.77,39.212,0.0,False,GPU -10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1721.0,+794/-134,0.535,13.47,8.53,3.741,2028.29,0.081,0.0,False,GPU -11,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1684.0,+503/-95,0.474,14.71,9.45,4.288,425.61,0.615,0.0,False,GPU -12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1671.0,+376/-133,0.388,15.18,11.21,5.936,222.05,11.236,0.0,False,CPU -13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1650.0,+347/-105,0.356,15.93,8.83,6.099,1658.41,0.742,0.0,False,CPU -14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1623.0,+616/-142,0.443,16.89,13.4,4.108,2526.28,0.166,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1619.0,+361/-178,0.35,17.04,7.26,6.197,1658.41,0.059,0.0,False,CPU -16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1618.0,+331/-121,0.326,17.09,13.63,6.24,222.05,0.968,0.0,False,CPU -17,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1602.0,+522/-214,0.341,17.64,14.24,5.117,3440.87,0.619,0.0,False,GPU -18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1555.0,+327/-104,0.296,19.4,17.23,5.243,425.61,0.04,0.0,False,GPU -19,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1545.0,+667/-210,0.305,19.76,5.04,4.564,2.06,0.755,0.0,False,GPU -20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1533.0,+425/-345,0.326,20.22,14.49,6.882,5.49,0.037,0.0,False,CPU -21,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1526.0,+421/-182,0.24,20.44,18.38,5.96,3440.87,0.052,0.0,False,GPU -22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1509.0,+268/-136,0.217,21.09,19.43,6.83,282.13,1.74,0.0,False,CPU -23,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1506.0,+647/-156,0.285,21.18,18.43,5.272,6.88,0.094,0.0,False,GPU -24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1497.0,+267/-142,0.197,21.51,19.3,6.953,282.13,0.388,0.0,False,CPU -25,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1360.0,+317/-308,0.159,26.42,21.95,8.067,0.6,0.08,0.0,False,CPU -26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1330.0,+330/-211,0.091,27.49,25.06,7.827,5.03,0.025,0.0,False,GPU -27,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1305.0,+240/-126,0.011,28.33,27.32,8.79,2.28,0.13,0.0,False,GPU -28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1301.0,+209/-114,0.061,28.49,26.91,8.242,4348.61,0.528,0.0,False,CPU -29,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1292.0,+283/-160,0.013,28.78,27.91,8.309,0.55,0.233,0.0,False,CPU -30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1219.0,+204/-297,0.038,31.24,29.53,9.231,4348.61,0.034,0.0,False,CPU -31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1199.0,+231/-316,0.0,31.89,30.73,9.7,6.75,0.235,0.0,False,GPU -32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1176.0,+359/-419,0.022,32.62,30.66,10.264,35.67,0.529,0.0,False,CPU -33,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1156.0,+87/-156,0.0,33.27,33.0,9.993,79.08,0.546,0.0,False,CPU -34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1144.0,+260/-394,0.006,33.64,32.41,10.361,35.67,0.05,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1127.0,+112/-248,0.0,34.16,33.79,10.297,79.08,0.057,0.0,False,CPU -36,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1102.0,+387/-661,0.143,34.91,29.95,13.832,2044.56,0.128,0.0,False,CPU -37,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1037.0,+193/-452,0.001,36.8,35.28,13.252,15.08,0.028,0.0,False,CPU -38,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1029.0,+364/-709,0.114,37.02,33.51,14.402,2044.56,0.01,0.0,False,CPU -39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1026.0,+358/-676,0.006,37.11,35.07,12.06,0.17,0.049,0.0,False,CPU -40,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+37/-248,0.0,37.82,37.67,11.446,0.43,0.053,0.0,False,CPU -41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,976.0,+370/-780,0.102,38.47,35.09,15.083,3.66,0.011,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,848.0,+60/-540,0.0,41.4,41.16,14.323,318.98,0.827,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,794.0,+54/-529,0.0,42.44,42.26,14.781,318.98,0.114,0.0,False,CPU -44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,789.0,+112/-640,0.0,42.53,42.33,18.087,142.58,1.184,0.0,False,CPU -45,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,664.0,+83/-704,0.0,44.53,44.43,20.643,142.58,0.064,0.0,False,CPU -46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,607.0,+116/-707,0.0,45.27,45.07,18.384,1.36,0.111,0.0,False,CPU -47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,349.0,+100/-1185,0.0,47.51,47.41,28.1,0.06,0.034,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,81.0,+131/-1638,0.0,48.92,48.92,39.828,44.3,0.048,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-114.0,+182/-2167,0.0,49.77,49.76,40.21,44.3,0.011,0.0,False,CPU -50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-414.0,+224/-2121,0.0,50.8,50.8,43.378,0.13,0.018,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),2069.0,+1045/-120,1.209,0.862,4.4,2.67,735.58,9.557,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1971.0,+783/-126,1.696,0.802,6.31,4.21,735.58,0.714,✔️,0.0,False,GPU +2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1922.0,+681/-41,2.304,0.713,7.47,5.68,2526.28,3.717,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1917.0,+1210/-123,1.293,0.782,7.6,2.16,4786.55,361.217,✔️,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1909.0,+515/-11,3.331,0.661,7.8,4.92,473.01,8.73,✔️,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1900.0,+797/-114,2.7,0.688,8.04,5.61,2028.29,1.675,✔️,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1884.0,+532/-21,3.488,0.644,8.47,4.35,430.4,7.174,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1850.0,+1175/-128,1.707,0.717,9.42,4.22,4786.55,41.599,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1802.0,+856/-218,2.636,0.564,10.87,7.04,1.6,0.508,✔️,0.0,False,GPU +9,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1753.0,+1086/-143,2.355,0.591,12.42,4.95,45.77,39.212,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1721.0,+794/-134,3.741,0.535,13.47,8.53,2028.29,0.081,✔️,0.0,False,GPU +11,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1684.0,+503/-95,4.288,0.474,14.71,9.45,425.61,0.615,✔️,0.0,False,GPU +12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1671.0,+376/-133,5.936,0.388,15.18,11.21,222.05,11.236,✔️,0.0,False,CPU +13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1650.0,+347/-105,6.099,0.356,15.93,8.83,1658.41,0.742,✔️,0.0,False,CPU +14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1623.0,+616/-142,4.108,0.443,16.89,13.4,2526.28,0.166,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1619.0,+361/-178,6.197,0.35,17.04,7.26,1658.41,0.059,✔️,0.0,False,CPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1618.0,+331/-121,6.24,0.326,17.09,13.63,222.05,0.968,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1602.0,+522/-214,5.117,0.341,17.64,14.24,3440.87,0.619,✔️,0.0,False,GPU +18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1555.0,+327/-104,5.243,0.296,19.4,17.23,425.61,0.04,✔️,0.0,False,GPU +19,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1545.0,+667/-210,4.564,0.305,19.76,5.04,2.06,0.755,➖,0.0,False,GPU +20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1533.0,+425/-345,6.882,0.326,20.22,14.49,5.49,0.037,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1526.0,+421/-182,5.96,0.24,20.44,18.38,3440.87,0.052,✔️,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1509.0,+268/-136,6.83,0.217,21.09,19.43,282.13,1.74,✔️,0.0,False,CPU +23,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1506.0,+647/-156,5.272,0.285,21.18,18.43,6.88,0.094,✔️,0.0,False,GPU +24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1497.0,+267/-142,6.953,0.197,21.51,19.3,282.13,0.388,✔️,0.0,False,CPU +25,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1360.0,+317/-308,8.067,0.159,26.42,21.95,0.6,0.08,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1330.0,+330/-211,7.827,0.091,27.49,25.06,5.03,0.025,✔️,0.0,False,GPU +27,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1305.0,+240/-126,8.79,0.011,28.33,27.32,2.28,0.13,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1301.0,+209/-114,8.242,0.061,28.49,26.91,4348.61,0.528,✔️,0.0,False,CPU +29,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1292.0,+283/-160,8.309,0.013,28.78,27.91,0.55,0.233,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1219.0,+204/-297,9.231,0.038,31.24,29.53,4348.61,0.034,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1199.0,+231/-316,9.7,0.0,31.89,30.73,6.75,0.235,✔️,0.0,False,GPU +32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1176.0,+359/-419,10.264,0.022,32.62,30.66,35.67,0.529,✔️,0.0,False,CPU +33,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1156.0,+87/-156,9.993,0.0,33.27,33.0,79.08,0.546,✔️,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1144.0,+260/-394,10.361,0.006,33.64,32.41,35.67,0.05,✔️,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1127.0,+112/-248,10.297,0.0,34.16,33.79,79.08,0.057,✔️,0.0,False,CPU +36,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1102.0,+387/-661,13.832,0.143,34.91,29.95,2044.56,0.128,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1037.0,+193/-452,13.252,0.001,36.8,35.28,15.08,0.028,✔️,0.0,False,CPU +38,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1029.0,+364/-709,14.402,0.114,37.02,33.51,2044.56,0.01,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1026.0,+358/-676,12.06,0.006,37.11,35.07,0.17,0.049,✔️,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+37/-248,11.446,0.0,37.82,37.67,0.43,0.053,✔️,0.0,False,CPU +41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),976.0,+370/-780,15.083,0.102,38.47,35.09,3.66,0.011,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),848.0,+60/-540,14.323,0.0,41.4,41.16,318.98,0.827,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),794.0,+54/-529,14.781,0.0,42.44,42.26,318.98,0.114,✔️,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),789.0,+112/-640,18.087,0.0,42.53,42.33,142.58,1.184,✔️,0.0,False,CPU +45,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),664.0,+83/-704,20.643,0.0,44.53,44.43,142.58,0.064,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),607.0,+116/-707,18.384,0.0,45.27,45.07,1.36,0.111,✔️,0.0,False,CPU +47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),349.0,+100/-1185,28.1,0.0,47.51,47.41,0.06,0.034,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),81.0,+131/-1638,39.828,0.0,48.92,48.92,44.3,0.048,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),-114.0,+182/-2167,40.21,0.0,49.77,49.76,44.3,0.011,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),-414.0,+224/-2121,43.378,0.0,50.8,50.8,0.13,0.018,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_medium/winrate_matrix.png.zip index 70bad7dfdde14268234e295d58f2033b14c688a3..9d25dc272f49cad02c500630df25fc825b19ec5b 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2eefb189956d2e0f6a276df4b6eff034d61a4713f4fb7887a5d2605f89ed0dc8 +oid sha256:39c31dc82bf30b40fa16056158f67331b71d259ff03d8542b3daf22c2e2525f7 size 1489576 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 89a2e2bbb7a360298dbc27c4a1b1d99830ec6a1d..57b9af2d66157c437c441f6646c3478865d8c35b 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ebbf464ff08f97e962caf0d71ffb944e18a72cad6a1783ab6b10fda4cf765445 +oid sha256:b7a1c2e8a752e20698083d943b23847e97c8af7502aa32b2186bb8b618b558d9 size 419998 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip index e4bdf9559430650d131a7730b5ee051e1781d71a..03b80186f5922ec3141d06d0af9925b4ab73050a 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3d57e80d26ed5be20540aa428a4fa13e103c0cdd3aa9c853334639d017c6baa0 -size 1002629 +oid sha256:b15da6955704a5aa6bcf2fe7dc27a7815a2b740e153b1b11992630ef054a1745 +size 987516 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_small/tuning-impact-elo.png.zip index de49784416401acb83f3a4fe0d417e427f2c2344..de0568ba1ec97563f9fa9c335932ee17f7548c90 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:dfa6b312790fa796a5e9dd65aa2fb36144a3b1dc90dbffdf4dd8692f01b484a3 +oid sha256:15d1c1846761635bbeba3ed64ca4f16a8c89ff90668ceeef435e5580802d0087 size 114452 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_regression/datasets_small/website_leaderboard.csv index 00c9eb1f3cc4032ce4a5f4acfe498e3e17482e7d..339952b7b479372c4ee9c422bc2e94c85f1c595c 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_small/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_regression/datasets_small/website_leaderboard.csv @@ -1,52 +1,52 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1763.0,+145/-63,0.797,6.79,3.07,3.087,4822.0,140.81,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1746.0,+229/-93,0.729,7.27,3.69,2.545,709.58,5.864,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1740.0,+293/-144,0.782,7.42,2.5,1.659,2306.87,7.231,0.0,False,GPU -3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1716.0,+119/-54,0.737,8.13,4.31,3.561,4822.0,22.324,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1686.0,+133/-56,0.67,9.09,5.29,3.505,4558.61,33.853,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1612.0,+126/-52,0.606,11.7,4.66,4.689,48.13,42.001,0.0,False,GPU -6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1602.0,+145/-76,0.509,12.08,7.82,5.458,5805.42,4.31,0.0,False,CPU -7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1599.0,+238/-134,0.631,12.2,4.42,2.59,2306.87,0.902,0.0,False,GPU -8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1587.0,+131/-72,0.506,12.69,7.58,4.449,4558.61,2.234,0.0,False,GPU -9,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1539.0,+225/-156,0.553,14.68,7.53,4.46,7.67,0.547,0.0,False,GPU -10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1516.0,+140/-75,0.377,15.7,10.55,6.298,4430.73,1.103,0.0,False,CPU -11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1494.0,+121/-71,0.345,16.68,13.64,7.15,1203.24,7.106,0.0,False,CPU -12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1492.0,+143/-72,0.347,16.76,11.72,6.43,4430.73,0.179,0.0,False,CPU -13,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1491.0,+180/-93,0.376,16.79,5.25,7.543,8977.36,8.017,0.0,False,GPU -14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1479.0,+203/-101,0.355,17.34,11.77,5.112,4438.37,1.981,0.0,False,GPU -15,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1471.0,+125/-94,0.321,17.74,10.77,6.601,2039.04,2.144,0.0,False,GPU -16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1440.0,+102/-35,0.278,19.19,16.13,7.459,1265.82,3.15,0.0,False,CPU -17,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1431.0,+292/-243,0.497,19.6,5.84,18.512,6.44,0.709,0.0,False,GPU -18,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1413.0,+102/-71,0.242,20.47,16.02,7.922,1203.24,0.956,0.0,False,CPU -19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1412.0,+96/-39,0.237,20.55,16.89,7.622,1265.82,0.476,0.0,False,CPU -20,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1394.0,+135/-83,0.241,21.4,15.88,7.101,2039.04,0.109,0.0,False,GPU -21,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1381.0,+203/-117,0.257,22.06,13.23,6.14,4438.37,0.231,0.0,False,GPU -22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1370.0,+154/-99,0.222,22.59,16.41,8.572,12.31,0.204,0.0,False,CPU -23,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1352.0,+157/-92,0.154,23.51,12.54,9.663,8977.36,0.421,0.0,False,GPU -24,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1313.0,+187/-132,0.205,25.44,17.93,7.765,22.14,0.179,0.0,False,GPU -25,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1307.0,+151/-99,0.142,25.74,18.41,8.242,21.51,8.356,0.0,False,GPU -26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1284.0,+166/-166,0.15,26.92,16.47,9.927,5150.31,1.592,0.0,False,CPU -27,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1273.0,+116/-149,0.159,27.44,18.16,12.463,664.2,1.363,0.0,False,CPU -28,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1245.0,+114/-152,0.13,28.82,17.01,12.747,664.2,0.274,0.0,False,CPU -29,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1239.0,+68/-67,0.013,29.12,22.27,11.438,26.36,0.302,0.0,False,GPU -30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1222.0,+166/-151,0.107,29.94,20.1,10.595,5150.31,0.136,0.0,False,CPU -31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1220.0,+186/-176,0.101,30.01,20.27,12.259,4333.58,0.422,0.0,False,CPU -32,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1213.0,+58/-45,0.0,30.38,28.71,10.582,4.3,0.318,0.0,False,CPU -33,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1185.0,+94/-140,0.057,31.67,27.97,13.884,675.17,1.541,0.0,False,CPU -34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1177.0,+185/-183,0.072,32.03,21.23,12.776,4333.58,0.037,0.0,False,CPU -35,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1144.0,+53/-50,0.0,33.52,32.42,11.807,4.06,0.299,0.0,False,CPU -36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1143.0,+98/-144,0.041,33.57,29.95,14.489,675.17,0.206,0.0,False,CPU -37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1138.0,+99/-130,0.033,33.78,31.0,13.385,2214.9,7.795,0.0,False,CPU -38,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1120.0,+153/-223,0.056,34.61,22.72,14.224,10.2,2.718,0.0,False,GPU -39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1115.0,+97/-173,0.018,34.79,29.64,14.837,0.5,0.116,0.0,False,CPU -40,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1100.0,+152/-182,0.016,35.44,29.19,13.873,16.01,0.041,0.0,False,CPU -41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1077.0,+95/-132,0.016,36.38,34.34,13.89,2214.9,0.838,0.0,False,CPU -42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1050.0,+118/-144,0.0,37.45,35.11,13.659,22.39,0.134,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+87/-145,0.0,39.27,37.2,16.235,0.63,0.12,0.0,False,CPU -44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,956.0,+163/-363,0.011,40.72,32.95,20.486,88.43,0.512,0.0,False,CPU -45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,952.0,+90/-160,0.0,40.87,39.76,18.564,6.97,0.818,0.0,False,CPU -46,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,852.0,+166/-381,0.0,43.65,37.37,22.826,88.43,0.051,0.0,False,CPU -47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,756.0,+184/-548,0.0,45.69,35.68,29.839,0.42,0.047,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,600.0,+123/-419,0.0,48.06,47.79,34.237,777.57,0.312,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,585.0,+130/-439,0.0,48.25,47.95,34.281,777.57,0.108,0.0,False,CPU -50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,399.0,+87/-397,0.0,50.0,49.97,36.257,3.91,0.126,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1763.0,+145/-63,3.087,0.797,6.79,3.07,4822.0,140.81,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1746.0,+229/-93,2.545,0.729,7.27,3.69,709.58,5.864,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1740.0,+293/-144,1.659,0.782,7.42,2.5,2306.87,7.231,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1716.0,+119/-54,3.561,0.737,8.13,4.31,4822.0,22.324,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1686.0,+133/-56,3.505,0.67,9.09,5.29,4558.61,33.853,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1612.0,+126/-52,4.689,0.606,11.7,4.66,48.13,42.001,✔️,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1602.0,+145/-76,5.458,0.509,12.08,7.82,5805.42,4.31,✔️,0.0,False,CPU +7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1599.0,+238/-134,2.59,0.631,12.2,4.42,2306.87,0.902,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1587.0,+131/-72,4.449,0.506,12.69,7.58,4558.61,2.234,✔️,0.0,False,GPU +9,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1539.0,+225/-156,4.46,0.553,14.68,7.53,7.67,0.547,✔️,0.0,False,GPU +10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1516.0,+140/-75,6.298,0.377,15.7,10.55,4430.73,1.103,✔️,0.0,False,CPU +11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1494.0,+121/-71,7.15,0.345,16.68,13.64,1203.24,7.106,✔️,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1492.0,+143/-72,6.43,0.347,16.76,11.72,4430.73,0.179,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1491.0,+180/-93,7.543,0.376,16.79,5.25,8977.36,8.017,✔️,0.0,False,GPU +14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1479.0,+203/-101,5.112,0.355,17.34,11.77,4438.37,1.981,✔️,0.0,False,GPU +15,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1471.0,+125/-94,6.601,0.321,17.74,10.77,2039.04,2.144,✔️,0.0,False,GPU +16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1440.0,+102/-35,7.459,0.278,19.19,16.13,1265.82,3.15,✔️,0.0,False,CPU +17,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1431.0,+292/-243,18.512,0.497,19.6,5.84,6.44,0.709,➖,0.0,False,GPU +18,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1413.0,+102/-71,7.922,0.242,20.47,16.02,1203.24,0.956,✔️,0.0,False,CPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1412.0,+96/-39,7.622,0.237,20.55,16.89,1265.82,0.476,✔️,0.0,False,CPU +20,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1394.0,+135/-83,7.101,0.241,21.4,15.88,2039.04,0.109,✔️,0.0,False,GPU +21,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1381.0,+203/-117,6.14,0.257,22.06,13.23,4438.37,0.231,✔️,0.0,False,GPU +22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1370.0,+154/-99,8.572,0.222,22.59,16.41,12.31,0.204,✔️,0.0,False,CPU +23,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1352.0,+157/-92,9.663,0.154,23.51,12.54,8977.36,0.421,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1313.0,+187/-132,7.765,0.205,25.44,17.93,22.14,0.179,✔️,0.0,False,GPU +25,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1307.0,+151/-99,8.242,0.142,25.74,18.41,21.51,8.356,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1284.0,+166/-166,9.927,0.15,26.92,16.47,5150.31,1.592,✔️,0.0,False,CPU +27,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1273.0,+116/-149,12.463,0.159,27.44,18.16,664.2,1.363,✔️,0.0,False,CPU +28,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1245.0,+114/-152,12.747,0.13,28.82,17.01,664.2,0.274,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1239.0,+68/-67,11.438,0.013,29.12,22.27,26.36,0.302,✔️,0.0,False,GPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1222.0,+166/-151,10.595,0.107,29.94,20.1,5150.31,0.136,✔️,0.0,False,CPU +31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1220.0,+186/-176,12.259,0.101,30.01,20.27,4333.58,0.422,✔️,0.0,False,CPU +32,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1213.0,+58/-45,10.582,0.0,30.38,28.71,4.3,0.318,✔️,0.0,False,CPU +33,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1185.0,+94/-140,13.884,0.057,31.67,27.97,675.17,1.541,✔️,0.0,False,CPU +34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1177.0,+185/-183,12.776,0.072,32.03,21.23,4333.58,0.037,✔️,0.0,False,CPU +35,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1144.0,+53/-50,11.807,0.0,33.52,32.42,4.06,0.299,✔️,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1143.0,+98/-144,14.489,0.041,33.57,29.95,675.17,0.206,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1138.0,+99/-130,13.385,0.033,33.78,31.0,2214.9,7.795,✔️,0.0,False,CPU +38,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1120.0,+153/-223,14.224,0.056,34.61,22.72,10.2,2.718,✔️,0.0,False,GPU +39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1115.0,+97/-173,14.837,0.018,34.79,29.64,0.5,0.116,✔️,0.0,False,CPU +40,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1100.0,+152/-182,13.873,0.016,35.44,29.19,16.01,0.041,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1077.0,+95/-132,13.89,0.016,36.38,34.34,2214.9,0.838,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1050.0,+118/-144,13.659,0.0,37.45,35.11,22.39,0.134,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+87/-145,16.235,0.0,39.27,37.2,0.63,0.12,✔️,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),956.0,+163/-363,20.486,0.011,40.72,32.95,88.43,0.512,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),952.0,+90/-160,18.564,0.0,40.87,39.76,6.97,0.818,✔️,0.0,False,CPU +46,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),852.0,+166/-381,22.826,0.0,43.65,37.37,88.43,0.051,✔️,0.0,False,CPU +47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),756.0,+184/-548,29.839,0.0,45.69,35.68,0.42,0.047,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),600.0,+123/-419,34.237,0.0,48.06,47.79,777.57,0.312,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),585.0,+130/-439,34.281,0.0,48.25,47.95,777.57,0.108,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),399.0,+87/-397,36.257,0.0,50.0,49.97,3.91,0.126,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_small/winrate_matrix.png.zip index a3b19ef7950240184b2448b8a2492edbf91141b9..2544357581e947bc6313f2e2cf324392626a2573 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d022054d916b1fb13939d83df8fcae625d5029d5ae4b0b9b579e78c379acd929 +oid sha256:e2b974c4b9319e180d327d4bd1394e9d7a2dca40e7d490bef851fd8160c88fa4 size 1820517 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 06173afb9960a414073c3b1f2e5ced673fee3953..193e38beb5962e8a2f37f35248b722d29fe2b5b0 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:79ab9a6d01cf85e3dbda94e820c2827b65864c1bc65edf6bf9830fa693a1fa64 +oid sha256:47c984a7d938d82dc253552c27707fd6428166cc13dd6a707963b8f3b77cc143 size 453334 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip index 7a267d5629d377cd5d8326072bdc42ffb52e4b0d..9a649cfb79d98d6d41fdd709070dc1742163e2ea 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ea7b722927aea2823e62720196d93cc9a41a45b51e8ffc7d222acb4cbc6cdfb2 -size 1046899 +oid sha256:db3e37f00a756464344b395ee4405cf9a773cb38023a6e91c3851aa7e9454a3c +size 1019357 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip index 7f65ab86bdf3d398b2a9c86027135dfabbd936c7..b196b2f3428f4c67513f7a563f2a8eb53525cea3 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9fba320f98ae4202658ec44d36911c52cae0e19c174b582a7cc018e5bd4a699c +oid sha256:a2a4d31d903566c633629b4f526b0de6ceec8aa949eb0946d3e0135d95f909d4 size 119094 diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/website_leaderboard.csv index 90377fef65b435ce248793b94ebd2127cb72e3d0..0e833c45528f5d3f913975bf8a03140d92d16394 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/website_leaderboard.csv @@ -1,56 +1,56 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1738.0,+227/-92,0.724,8.07,4.14,2.676,669.92,5.859,0.0,False,GPU -1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1729.0,+168/-38,0.767,8.36,3.52,3.492,4663.16,132.911,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1723.0,+332/-141,0.764,8.56,2.6,1.797,2166.18,6.341,0.0,False,GPU -3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1699.0,+128/-34,0.721,9.38,4.73,3.903,4663.16,18.295,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1682.0,+131/-57,0.676,10.01,5.56,3.736,4823.88,33.539,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1626.0,+176/-56,0.627,12.17,4.57,4.903,47.35,39.185,0.0,False,GPU -6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1607.0,+195/-116,0.601,13.01,5.36,3.411,6577.99,56.795,0.0,False,GPU -7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1592.0,+287/-164,0.619,13.66,4.39,2.725,2166.18,0.812,0.0,False,GPU -8,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1568.0,+116/-54,0.451,14.73,9.53,6.052,6530.9,4.363,0.0,False,CPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1565.0,+155/-64,0.462,14.89,8.5,4.84,4823.88,1.952,0.0,False,GPU -10,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1557.0,+236/-169,0.583,15.25,7.94,4.606,7.15,0.501,0.0,False,GPU -11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1540.0,+116/-42,0.392,16.08,10.73,6.578,3558.14,0.966,0.0,False,CPU -12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1520.0,+129/-52,0.354,17.06,12.43,6.697,3558.14,0.105,0.0,False,CPU -13,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1503.0,+250/-242,0.546,17.88,5.52,19.511,5.94,0.652,0.0,False,GPU -14,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1492.0,+191/-91,0.36,18.42,6.11,6.52,99.14,2.48,0.0,False,GPU -15,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1476.0,+131/-107,0.309,19.23,11.16,7.027,2045.39,1.388,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1475.0,+112/-68,0.303,19.3,15.45,7.822,1003.11,4.891,0.0,False,CPU -17,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1459.0,+217/-133,0.328,20.15,13.53,5.499,4158.29,1.946,0.0,False,GPU -18,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1440.0,+231/-181,0.391,21.16,9.88,5.382,6577.99,0.498,0.0,False,GPU -19,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1437.0,+138/-67,0.266,21.28,11.17,8.679,9308.22,7.885,0.0,False,GPU -20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1428.0,+88/-37,0.229,21.77,18.45,8.1,933.42,2.759,0.0,False,CPU -21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1412.0,+127/-98,0.246,22.61,16.94,7.406,2045.39,0.104,0.0,False,GPU -22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1401.0,+91/-46,0.192,23.23,19.21,8.265,933.42,0.388,0.0,False,CPU -23,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1399.0,+114/-70,0.201,23.35,18.34,8.659,1003.11,0.894,0.0,False,CPU -24,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1385.0,+140/-87,0.219,24.07,16.92,9.129,10.91,0.197,0.0,False,CPU -25,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1362.0,+218/-156,0.219,25.36,14.47,6.622,4158.29,0.208,0.0,False,GPU -26,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1354.0,+232/-206,0.284,25.77,13.92,7.982,9.1,0.85,0.0,False,GPU -27,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1299.0,+96/-60,0.073,28.81,20.91,10.902,9308.22,0.416,0.0,False,GPU -28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1293.0,+194/-163,0.171,29.15,18.85,13.262,623.85,1.144,0.0,False,CPU -29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1292.0,+215/-168,0.168,29.2,20.9,8.427,15.01,0.154,0.0,False,GPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1287.0,+170/-106,0.118,29.44,22.47,8.943,23.19,8.523,0.0,False,GPU -31,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1272.0,+170/-162,0.135,30.3,17.5,13.412,623.85,0.291,0.0,False,CPU -32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1250.0,+149/-168,0.091,31.47,21.51,12.707,2929.85,0.41,0.0,False,CPU -33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1228.0,+90/-90,0.015,32.61,24.17,12.445,27.82,0.299,0.0,False,GPU -34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1219.0,+163/-152,0.07,33.04,27.68,11.15,5692.02,1.354,0.0,False,CPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1204.0,+166/-180,0.074,33.82,22.52,13.286,2929.85,0.037,0.0,False,CPU -36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1202.0,+47/-44,0.0,33.94,32.18,11.45,4.57,0.275,0.0,False,CPU -37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1198.0,+126/-178,0.06,34.14,29.95,14.807,633.57,1.512,0.0,False,CPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1187.0,+188/-175,0.077,34.7,22.5,11.641,5692.02,0.131,0.0,False,CPU -39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1161.0,+120/-186,0.042,35.98,31.92,15.352,633.57,0.248,0.0,False,CPU -40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1142.0,+154/-178,0.055,36.87,25.15,14.894,10.27,3.409,0.0,False,GPU -41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1142.0,+46/-60,0.0,36.87,35.63,12.598,3.79,0.311,0.0,False,CPU -42,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1131.0,+130/-155,0.001,37.4,30.44,14.18,11.19,0.038,0.0,False,CPU -43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1128.0,+134/-196,0.002,37.53,32.07,15.73,0.47,0.117,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1085.0,+62/-94,0.0,39.46,38.31,14.849,2248.07,7.53,0.0,False,CPU -45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1025.0,+68/-114,0.0,41.94,41.02,15.336,2248.07,0.89,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1013.0,+116/-188,0.0,42.38,40.15,14.924,24.31,0.134,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+109/-207,0.0,42.87,40.52,17.325,0.53,0.118,0.0,False,CPU -48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,982.0,+155/-419,0.0,43.54,34.61,21.384,84.3,0.438,0.0,False,CPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,902.0,+53/-155,0.0,46.1,45.46,20.488,7.09,0.843,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,877.0,+150/-428,0.0,46.82,39.94,23.664,84.3,0.05,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,801.0,+167/-577,0.0,48.7,36.63,30.315,0.39,0.044,0.0,False,CPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,607.0,+111/-430,0.0,52.0,51.7,36.055,779.18,0.247,0.0,False,CPU -53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,594.0,+126/-470,0.0,52.17,51.85,36.078,779.18,0.089,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,421.0,+89/-422,0.0,53.9,53.88,37.463,3.97,0.122,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1738.0,+227/-92,2.676,0.724,8.07,4.14,669.92,5.859,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1729.0,+168/-38,3.492,0.767,8.36,3.52,4663.16,132.911,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1723.0,+332/-141,1.797,0.764,8.56,2.6,2166.18,6.341,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1699.0,+128/-34,3.903,0.721,9.38,4.73,4663.16,18.295,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1682.0,+131/-57,3.736,0.676,10.01,5.56,4823.88,33.539,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1626.0,+176/-56,4.903,0.627,12.17,4.57,47.35,39.185,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1607.0,+195/-116,3.411,0.601,13.01,5.36,6577.99,56.795,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1592.0,+287/-164,2.725,0.619,13.66,4.39,2166.18,0.812,✔️,0.0,False,GPU +8,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1568.0,+116/-54,6.052,0.451,14.73,9.53,6530.9,4.363,✔️,0.0,False,CPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1565.0,+155/-64,4.84,0.462,14.89,8.5,4823.88,1.952,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1557.0,+236/-169,4.606,0.583,15.25,7.94,7.15,0.501,✔️,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1540.0,+116/-42,6.578,0.392,16.08,10.73,3558.14,0.966,✔️,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1520.0,+129/-52,6.697,0.354,17.06,12.43,3558.14,0.105,✔️,0.0,False,CPU +13,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1503.0,+250/-242,19.511,0.546,17.88,5.52,5.94,0.652,➖,0.0,False,GPU +14,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1492.0,+191/-91,6.52,0.36,18.42,6.11,99.14,2.48,✔️,0.0,False,GPU +15,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1476.0,+131/-107,7.027,0.309,19.23,11.16,2045.39,1.388,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1475.0,+112/-68,7.822,0.303,19.3,15.45,1003.11,4.891,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1459.0,+217/-133,5.499,0.328,20.15,13.53,4158.29,1.946,✔️,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1440.0,+231/-181,5.382,0.391,21.16,9.88,6577.99,0.498,✔️,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1437.0,+138/-67,8.679,0.266,21.28,11.17,9308.22,7.885,✔️,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1428.0,+88/-37,8.1,0.229,21.77,18.45,933.42,2.759,✔️,0.0,False,CPU +21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1412.0,+127/-98,7.406,0.246,22.61,16.94,2045.39,0.104,✔️,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1401.0,+91/-46,8.265,0.192,23.23,19.21,933.42,0.388,✔️,0.0,False,CPU +23,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1399.0,+114/-70,8.659,0.201,23.35,18.34,1003.11,0.894,✔️,0.0,False,CPU +24,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1385.0,+140/-87,9.129,0.219,24.07,16.92,10.91,0.197,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1362.0,+218/-156,6.622,0.219,25.36,14.47,4158.29,0.208,✔️,0.0,False,GPU +26,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1354.0,+232/-206,7.982,0.284,25.77,13.92,9.1,0.85,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1299.0,+96/-60,10.902,0.073,28.81,20.91,9308.22,0.416,✔️,0.0,False,GPU +28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1293.0,+194/-163,13.262,0.171,29.15,18.85,623.85,1.144,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1292.0,+215/-168,8.427,0.168,29.2,20.9,15.01,0.154,✔️,0.0,False,GPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1287.0,+170/-106,8.943,0.118,29.44,22.47,23.19,8.523,✔️,0.0,False,GPU +31,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1272.0,+170/-162,13.412,0.135,30.3,17.5,623.85,0.291,✔️,0.0,False,CPU +32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1250.0,+149/-168,12.707,0.091,31.47,21.51,2929.85,0.41,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1228.0,+90/-90,12.445,0.015,32.61,24.17,27.82,0.299,✔️,0.0,False,GPU +34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1219.0,+163/-152,11.15,0.07,33.04,27.68,5692.02,1.354,✔️,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1204.0,+166/-180,13.286,0.074,33.82,22.52,2929.85,0.037,✔️,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1202.0,+47/-44,11.45,0.0,33.94,32.18,4.57,0.275,✔️,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1198.0,+126/-178,14.807,0.06,34.14,29.95,633.57,1.512,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1187.0,+188/-175,11.641,0.077,34.7,22.5,5692.02,0.131,✔️,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1161.0,+120/-186,15.352,0.042,35.98,31.92,633.57,0.248,✔️,0.0,False,CPU +40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1142.0,+154/-178,14.894,0.055,36.87,25.15,10.27,3.409,✔️,0.0,False,GPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1142.0,+46/-60,12.598,0.0,36.87,35.63,3.79,0.311,✔️,0.0,False,CPU +42,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1131.0,+130/-155,14.18,0.001,37.4,30.44,11.19,0.038,✔️,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1128.0,+134/-196,15.73,0.002,37.53,32.07,0.47,0.117,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1085.0,+62/-94,14.849,0.0,39.46,38.31,2248.07,7.53,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1025.0,+68/-114,15.336,0.0,41.94,41.02,2248.07,0.89,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1013.0,+116/-188,14.924,0.0,42.38,40.15,24.31,0.134,✔️,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+109/-207,17.325,0.0,42.87,40.52,0.53,0.118,✔️,0.0,False,CPU +48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),982.0,+155/-419,21.384,0.0,43.54,34.61,84.3,0.438,✔️,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),902.0,+53/-155,20.488,0.0,46.1,45.46,7.09,0.843,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),877.0,+150/-428,23.664,0.0,46.82,39.94,84.3,0.05,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),801.0,+167/-577,30.315,0.0,48.7,36.63,0.39,0.044,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),607.0,+111/-430,36.055,0.0,52.0,51.7,779.18,0.247,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),594.0,+126/-470,36.078,0.0,52.17,51.85,779.18,0.089,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),421.0,+89/-422,37.463,0.0,53.9,53.88,3.97,0.122,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip index 67d13c61ff2a8acc88679d0f15f80d7672a7da83..d0addc0af4c5786fb1c548f75151d364832a8fd2 100644 --- a/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_no/splits_all/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9e507d90a9dcfd362b60a1916d0fa251f185125461583ddda9f451158cc0f42d +oid sha256:570288d387692cf01750596fe7f47174e67fd37ca2168c1acdcda8843b018f30 size 2122484 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index b2a2efdaa09236541bda3e813fb2e99cdddc9d0e..3ffc470fdf4fbfd1ca5453887890a79f8f12d0f7 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0398d05006a861ff463a29080f79a072c4ce7697620ec5d5944ecbcb16ca9f10 +oid sha256:158a7b8fcee64d8e1dfc1f68ee03f6f20d4cbd3ea6cbf7de5b1cef28e1cfb94c size 406340 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_all/pareto_n_configs_imp.png.zip index c430998c2493ec0cabf58fd44d82f1f2ed9459e4..262857be434bf759d0e2a7164bd467d7caddb593 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6f9c279491f0b5d7b5792ce60983973b5a1d9971b78419aadef3912fb14e89d3 -size 991847 +oid sha256:bf031fc1a26f2b9c1d2e0c68a667655db3e31f124f9edc1612ff1cd4dff35d91 +size 943947 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_all/tuning-impact-elo.png.zip index 59d3043203ad1a64190709707c0e8600bdbe12e0..1b25a32e189700e1b86e578ffd060124e4f41498 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:820a15afae04f81bcbfd042d66b17131ccf26177600152bf545291e099c7cc75 +oid sha256:f0dca378cfa632370cda1dccb30509925c960ae1b327ad172ec8223a93904f7f size 108706 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_all/datasets_all/website_leaderboard.csv index dbf3d054ddaf2bdcf4b6bc7acab261970cdcbb70..86b69b95a6d8e562f990a96c7bc3486a170ac05b 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_all/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_all/datasets_all/website_leaderboard.csv @@ -1,51 +1,51 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1652.0,+102/-67,0.686,7.61,2.54,3.671,2059.94,9.785,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1647.0,+88/-77,0.664,7.77,3.65,5.78,556.15,6.31,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1603.0,+92/-71,0.614,9.21,3.81,5.787,2059.94,1.03,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1583.0,+63/-68,0.569,9.92,4.53,7.556,1754.94,1.767,0.0,False,CPU -4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1562.0,+83/-55,0.558,10.74,4.89,6.429,5.71,0.611,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1546.0,+58/-54,0.509,11.35,6.92,8.669,2791.97,13.886,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1462.0,+55/-60,0.416,15.05,7.54,10.194,2791.97,0.373,0.0,False,GPU -7,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1461.0,+74/-64,0.448,15.12,4.62,9.079,6154.73,386.167,0.0,False,GPU -8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1444.0,+42/-42,0.318,15.93,12.3,11.616,416.56,2.236,0.0,False,CPU -9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1441.0,+75/-47,0.376,16.06,8.26,10.656,3133.91,1.273,0.0,False,GPU -10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1417.0,+57/-51,0.325,17.27,11.49,11.2,1665.53,0.559,0.0,False,CPU -11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1416.0,+79/-61,0.4,17.34,5.58,10.724,6154.73,39.452,0.0,False,GPU -12,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1413.0,+85/-64,0.385,17.49,7.38,11.505,4618.5,7.737,0.0,False,GPU -13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1392.0,+51/-47,0.293,18.55,12.16,11.63,1665.53,0.065,0.0,False,CPU -14,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1390.0,+42/-50,0.265,18.62,12.58,12.245,700.96,1.439,0.0,False,CPU -15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1375.0,+56/-49,0.249,19.43,14.97,12.525,416.56,0.381,0.0,False,CPU -16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1371.0,+73/-56,0.294,19.64,11.95,11.592,3133.91,0.13,0.0,False,GPU -17,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1370.0,+61/-61,0.296,19.69,10.44,12.118,4618.5,0.47,0.0,False,GPU -18,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1369.0,+42/-45,0.252,19.73,11.47,12.212,6.7,0.088,0.0,False,CPU -19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1364.0,+46/-44,0.24,19.98,12.06,12.431,700.96,0.213,0.0,False,CPU -20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1363.0,+58/-46,0.284,20.04,11.85,12.395,866.11,2.007,0.0,False,GPU -21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1311.0,+58/-49,0.207,22.85,12.6,13.767,866.11,0.097,0.0,False,GPU -22,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1305.0,+76/-71,0.283,23.15,8.71,13.42,45.42,39.406,0.0,False,GPU -23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1301.0,+56/-53,0.212,23.39,16.03,14.0,11.56,0.127,0.0,False,GPU -24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1292.0,+53/-56,0.191,23.88,14.66,14.955,2961.52,0.482,0.0,False,CPU -25,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1272.0,+47/-54,0.148,24.92,19.48,13.751,2832.8,1.801,0.0,False,CPU -26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1272.0,+47/-53,0.146,24.92,17.63,14.37,10.44,1.714,0.0,False,GPU -27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1235.0,+65/-59,0.146,26.94,14.02,15.791,2961.52,0.048,0.0,False,CPU -28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1226.0,+62/-53,0.125,27.43,15.34,16.621,13.74,0.316,0.0,False,GPU -29,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1210.0,+58/-59,0.113,28.25,20.79,17.039,191.44,0.76,0.0,False,CPU -30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1210.0,+59/-63,0.13,28.27,15.82,16.737,7.66,0.046,0.0,False,CPU -31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1205.0,+49/-65,0.117,28.49,22.59,15.617,2832.8,0.112,0.0,False,CPU -32,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1195.0,+61/-57,0.108,29.04,17.06,15.665,2.06,0.122,0.0,False,CPU -33,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1187.0,+67/-72,0.119,29.46,21.04,17.388,594.95,4.651,0.0,False,CPU -34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1174.0,+70/-72,0.115,30.11,18.25,18.088,191.44,0.101,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1169.0,+61/-69,0.084,30.35,22.42,17.981,377.08,0.747,0.0,False,CPU -36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1160.0,+58/-50,0.066,30.85,27.1,16.457,2.2,0.171,0.0,False,CPU -37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1125.0,+53/-54,0.045,32.55,26.69,18.796,377.08,0.091,0.0,False,CPU -38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1109.0,+66/-85,0.081,33.35,21.78,19.015,594.95,0.337,0.0,False,CPU -39,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1033.0,+72/-74,0.058,36.76,25.12,23.637,3.14,0.741,0.0,False,GPU -40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1027.0,+59/-69,0.026,37.0,33.69,20.862,8.96,0.129,0.0,False,CPU -41,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+69/-66,0.014,38.12,30.41,23.662,0.43,0.053,0.0,False,CPU -42,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,972.0,+68/-86,0.023,39.2,35.14,25.503,129.1,1.627,0.0,False,CPU -43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,970.0,+77/-89,0.014,39.27,35.71,25.257,0.26,0.054,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,969.0,+73/-80,0.029,39.31,36.05,22.958,3.12,0.312,0.0,False,CPU -45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,908.0,+84/-111,0.022,41.43,22.85,32.143,240.73,0.308,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,871.0,+82/-112,0.017,42.56,30.24,32.729,240.73,0.068,0.0,False,CPU -47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,808.0,+88/-128,0.01,44.25,41.39,35.299,1.23,0.115,0.0,False,CPU -48,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,804.0,+77/-92,0.01,44.33,42.72,31.641,129.1,0.103,0.0,False,CPU -49,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,601.0,+82/-142,0.0,48.0,47.66,44.249,0.19,0.037,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1652.0,+102/-67,3.671,0.686,7.61,2.54,2059.94,9.785,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1647.0,+88/-77,5.78,0.664,7.77,3.65,556.15,6.31,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1603.0,+92/-71,5.787,0.614,9.21,3.81,2059.94,1.03,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1583.0,+63/-68,7.556,0.569,9.92,4.53,1754.94,1.767,✔️,0.0,False,CPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1562.0,+83/-55,6.429,0.558,10.74,4.89,5.71,0.611,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1546.0,+58/-54,8.669,0.509,11.35,6.92,2791.97,13.886,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1462.0,+55/-60,10.194,0.416,15.05,7.54,2791.97,0.373,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1461.0,+74/-64,9.079,0.448,15.12,4.62,6154.73,386.167,✔️,0.0,False,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1444.0,+42/-42,11.616,0.318,15.93,12.3,416.56,2.236,✔️,0.0,False,CPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1441.0,+75/-47,10.656,0.376,16.06,8.26,3133.91,1.273,✔️,0.0,False,GPU +10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1417.0,+57/-51,11.2,0.325,17.27,11.49,1665.53,0.559,✔️,0.0,False,CPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1416.0,+79/-61,10.724,0.4,17.34,5.58,6154.73,39.452,✔️,0.0,False,GPU +12,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1413.0,+85/-64,11.505,0.385,17.49,7.38,4618.5,7.737,✔️,0.0,False,GPU +13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1392.0,+51/-47,11.63,0.293,18.55,12.16,1665.53,0.065,✔️,0.0,False,CPU +14,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1390.0,+42/-50,12.245,0.265,18.62,12.58,700.96,1.439,✔️,0.0,False,CPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1375.0,+56/-49,12.525,0.249,19.43,14.97,416.56,0.381,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1371.0,+73/-56,11.592,0.294,19.64,11.95,3133.91,0.13,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1370.0,+61/-61,12.118,0.296,19.69,10.44,4618.5,0.47,✔️,0.0,False,GPU +18,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1369.0,+42/-45,12.212,0.252,19.73,11.47,6.7,0.088,✔️,0.0,False,CPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1364.0,+46/-44,12.431,0.24,19.98,12.06,700.96,0.213,✔️,0.0,False,CPU +20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1363.0,+58/-46,12.395,0.284,20.04,11.85,866.11,2.007,✔️,0.0,False,GPU +21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1311.0,+58/-49,13.767,0.207,22.85,12.6,866.11,0.097,✔️,0.0,False,GPU +22,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1305.0,+76/-71,13.42,0.283,23.15,8.71,45.42,39.406,✔️,0.0,False,GPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1301.0,+56/-53,14.0,0.212,23.39,16.03,11.56,0.127,✔️,0.0,False,GPU +24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1292.0,+53/-56,14.955,0.191,23.88,14.66,2961.52,0.482,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1272.0,+47/-54,13.751,0.148,24.92,19.48,2832.8,1.801,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1272.0,+47/-53,14.37,0.146,24.92,17.63,10.44,1.714,✔️,0.0,False,GPU +27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1235.0,+65/-59,15.791,0.146,26.94,14.02,2961.52,0.048,✔️,0.0,False,CPU +28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1226.0,+62/-53,16.621,0.125,27.43,15.34,13.74,0.316,✔️,0.0,False,GPU +29,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1210.0,+58/-59,17.039,0.113,28.25,20.79,191.44,0.76,✔️,0.0,False,CPU +30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1210.0,+59/-63,16.737,0.13,28.27,15.82,7.66,0.046,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1205.0,+49/-65,15.617,0.117,28.49,22.59,2832.8,0.112,✔️,0.0,False,CPU +32,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1195.0,+61/-57,15.665,0.108,29.04,17.06,2.06,0.122,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1187.0,+67/-72,17.388,0.119,29.46,21.04,594.95,4.651,✔️,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1174.0,+70/-72,18.088,0.115,30.11,18.25,191.44,0.101,✔️,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1169.0,+61/-69,17.981,0.084,30.35,22.42,377.08,0.747,✔️,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1160.0,+58/-50,16.457,0.066,30.85,27.1,2.2,0.171,✔️,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1125.0,+53/-54,18.796,0.045,32.55,26.69,377.08,0.091,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1109.0,+66/-85,19.015,0.081,33.35,21.78,594.95,0.337,✔️,0.0,False,CPU +39,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1033.0,+72/-74,23.637,0.058,36.76,25.12,3.14,0.741,✔️,0.0,False,GPU +40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1027.0,+59/-69,20.862,0.026,37.0,33.69,8.96,0.129,✔️,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+69/-66,23.662,0.014,38.12,30.41,0.43,0.053,✔️,0.0,False,CPU +42,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),972.0,+68/-86,25.503,0.023,39.2,35.14,129.1,1.627,✔️,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),970.0,+77/-89,25.257,0.014,39.27,35.71,0.26,0.054,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),969.0,+73/-80,22.958,0.029,39.31,36.05,3.12,0.312,✔️,0.0,False,CPU +45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),908.0,+84/-111,32.143,0.022,41.43,22.85,240.73,0.308,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),871.0,+82/-112,32.729,0.017,42.56,30.24,240.73,0.068,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),808.0,+88/-128,35.299,0.01,44.25,41.39,1.23,0.115,✔️,0.0,False,CPU +48,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),804.0,+77/-92,31.641,0.01,44.33,42.72,129.1,0.103,✔️,0.0,False,CPU +49,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),601.0,+82/-142,44.249,0.0,48.0,47.66,0.19,0.037,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_all/winrate_matrix.png.zip index ca6f0c9baec1f4999659ddc588d0ffd6804d8c2c..b002816800ce81041958baff94e10bec68935dbd 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6a2c4d1562b9065f114d063a87eae87a1d23dbad4b09be6e3a6d0917c374c06e +oid sha256:4cb3d20a2e52fbd76fabbb31a203eea2c789f512a53d83479839b98b271f9791 size 1650238 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index 490c0681e21a977fbda40d575bd384b180fe3fdf..46bb31c6634cf0d2c6367c022ff741eb8dd98757 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f9ec4506ec71725269c8706c526e12472ecbf61d0e2cac79eb499db0d58fb99e +oid sha256:8f8117dc0a5ca7eb0157549bc2f7bff0aca0941e31b5af2d0057cab5528f3f3d size 412803 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip index 59b58dc955f745acce0bbdf2b919a9d3bd473039..bb24c85f6035ad84630095f727c8378ed273d0bd 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b02cb89e554cd8ce20c8cb6c832ebb6e9dd239e5967c382c6506278b51a4cb62 -size 958450 +oid sha256:9401766c7b58398a4a7f0529b22fe32d5e2afe2e790d248ba4b58153a16d832e +size 960289 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_medium/tuning-impact-elo.png.zip index 86d151cd52c823b81ca7f26ecb576f6f97eb550a..c5d7f75c0b1611fb4f84507987bd6b1e887a5da9 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:07e8caca5c524cd021065ed087f58e3e6ad12469692e47c3ed6ecd1bec3950f3 +oid sha256:bfda76b2aa6770e7f801ae9784dcaadfa1140dbcb322797c287feb138ee71bb1 size 110829 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_all/datasets_medium/website_leaderboard.csv index d388f8e0e7be6be20dc26315fd6b192968365e70..b58532d50918e26fe5b2c33c220618212f95184d 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_medium/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_all/datasets_medium/website_leaderboard.csv @@ -1,51 +1,51 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1903.0,+146/-69,0.802,4.47,2.35,2.092,432.14,4.214,0.0,False,CPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1861.0,+154/-67,0.768,5.27,2.85,2.561,302.05,2.868,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1746.0,+184/-126,0.659,8.13,2.86,3.897,739.1,11.65,0.0,False,GPU -3,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1705.0,+184/-133,0.633,9.4,4.51,4.31,2522.58,6.234,0.0,False,GPU -4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1694.0,+166/-114,0.606,9.73,4.35,5.33,739.1,1.134,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1643.0,+104/-92,0.521,11.53,7.94,5.912,1714.29,1.743,0.0,False,GPU -6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1600.0,+93/-65,0.422,13.2,9.88,7.706,129.25,2.621,0.0,False,CPU -7,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1596.0,+112/-75,0.446,13.33,8.51,6.757,763.16,0.246,0.0,False,CPU -8,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1566.0,+103/-69,0.421,14.57,8.16,6.886,763.16,0.04,0.0,False,CPU -9,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1564.0,+147/-120,0.434,14.67,10.09,7.852,2522.58,0.318,0.0,False,GPU -10,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1560.0,+96/-65,0.371,14.8,10.43,7.44,289.35,0.706,0.0,False,CPU -11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1552.0,+161/-99,0.397,15.13,6.74,7.607,1993.57,0.581,0.0,False,GPU -12,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1543.0,+109/-91,0.352,15.53,11.78,8.332,129.25,0.117,0.0,False,CPU -13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1532.0,+138/-116,0.396,16.0,9.61,7.939,1714.29,0.079,0.0,False,GPU -14,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1529.0,+164/-124,0.4,16.13,8.01,6.506,1.91,0.668,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1524.0,+111/-86,0.393,16.37,7.52,7.158,2.93,0.045,0.0,False,CPU -16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1502.0,+106/-85,0.332,17.33,12.89,7.674,289.35,0.08,0.0,False,CPU -17,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1475.0,+149/-129,0.341,18.53,5.34,8.076,4532.14,387.927,0.0,False,GPU -18,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1465.0,+149/-93,0.296,19.0,10.82,8.336,1993.57,0.06,0.0,False,GPU -19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1423.0,+116/-118,0.283,20.93,14.58,9.191,559.64,0.88,0.0,False,GPU -20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1411.0,+152/-149,0.299,21.5,8.17,10.666,4532.14,39.452,0.0,False,GPU -21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1382.0,+104/-97,0.179,22.87,17.76,11.539,559.64,0.051,0.0,False,GPU -22,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1378.0,+131/-134,0.235,23.07,8.97,9.53,0.48,0.047,0.0,False,CPU -23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1375.0,+127/-114,0.191,23.2,19.49,10.925,5.42,0.078,0.0,False,GPU -24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1346.0,+80/-69,0.125,24.6,21.11,10.049,1803.23,1.199,0.0,False,CPU -25,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1331.0,+130/-148,0.21,25.33,17.48,13.131,1535.55,0.182,0.0,False,CPU -26,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1290.0,+130/-142,0.188,27.23,13.1,13.594,41.5,39.095,0.0,False,GPU -27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1288.0,+81/-70,0.055,27.33,25.1,12.893,6.54,0.294,0.0,False,GPU -28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1276.0,+95/-102,0.099,27.87,23.83,11.609,9.07,0.224,0.0,False,GPU -29,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1239.0,+83/-96,0.057,29.6,27.08,12.534,1803.23,0.051,0.0,False,CPU -30,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1236.0,+103/-99,0.048,29.73,24.05,12.859,65.73,0.23,0.0,False,CPU -31,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1222.0,+98/-130,0.044,30.33,27.5,12.599,36.73,0.359,0.0,False,CPU -32,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1218.0,+80/-93,0.03,30.53,28.9,12.025,0.34,0.07,0.0,False,CPU -33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1216.0,+135/-138,0.121,30.6,24.81,14.424,1535.55,0.013,0.0,False,CPU -34,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1204.0,+113/-137,0.109,31.13,27.61,15.52,2.99,0.016,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1203.0,+93/-86,0.037,31.17,27.59,13.195,65.73,0.025,0.0,False,CPU -36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1189.0,+115/-139,0.063,31.8,26.56,15.027,354.91,1.636,0.0,False,CPU -37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1148.0,+95/-121,0.021,33.53,31.16,14.978,36.73,0.038,0.0,False,CPU -38,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1074.0,+77/-88,0.005,36.47,35.3,16.346,2.58,0.039,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1049.0,+90/-96,0.011,37.4,35.95,17.593,354.91,0.094,0.0,False,CPU -40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1005.0,+164/-230,0.017,38.93,33.65,25.288,2.55,0.157,0.0,False,GPU -41,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+73/-113,0.0,39.1,38.22,18.839,0.24,0.022,0.0,False,CPU -42,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,977.0,+96/-131,0.011,39.87,37.93,22.57,150.58,4.952,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,933.0,+106/-153,0.015,41.2,39.26,21.097,1.49,0.113,0.0,False,CPU -44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,914.0,+167/-227,0.008,41.77,38.82,32.011,43.96,0.081,0.0,False,CPU -45,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,908.0,+108/-152,0.001,41.93,40.6,22.415,0.13,0.022,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,856.0,+156/-254,0.0,43.3,41.29,32.774,43.96,0.017,0.0,False,CPU -47,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,787.0,+59/-118,0.0,44.87,44.64,29.026,150.58,0.546,0.0,False,CPU -48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,716.0,+107/-199,0.0,46.2,45.88,34.827,0.2,0.02,0.0,False,CPU -49,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,540.0,+101/-189,0.0,48.47,48.38,42.676,0.1,0.068,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1903.0,+146/-69,2.092,0.802,4.47,2.35,432.14,4.214,✔️,0.0,False,CPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1861.0,+154/-67,2.561,0.768,5.27,2.85,302.05,2.868,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1746.0,+184/-126,3.897,0.659,8.13,2.86,739.1,11.65,✔️,0.0,False,GPU +3,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1705.0,+184/-133,4.31,0.633,9.4,4.51,2522.58,6.234,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1694.0,+166/-114,5.33,0.606,9.73,4.35,739.1,1.134,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1643.0,+104/-92,5.912,0.521,11.53,7.94,1714.29,1.743,✔️,0.0,False,GPU +6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1600.0,+93/-65,7.706,0.422,13.2,9.88,129.25,2.621,✔️,0.0,False,CPU +7,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1596.0,+112/-75,6.757,0.446,13.33,8.51,763.16,0.246,✔️,0.0,False,CPU +8,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1566.0,+103/-69,6.886,0.421,14.57,8.16,763.16,0.04,✔️,0.0,False,CPU +9,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1564.0,+147/-120,7.852,0.434,14.67,10.09,2522.58,0.318,✔️,0.0,False,GPU +10,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1560.0,+96/-65,7.44,0.371,14.8,10.43,289.35,0.706,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1552.0,+161/-99,7.607,0.397,15.13,6.74,1993.57,0.581,✔️,0.0,False,GPU +12,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1543.0,+109/-91,8.332,0.352,15.53,11.78,129.25,0.117,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1532.0,+138/-116,7.939,0.396,16.0,9.61,1714.29,0.079,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1529.0,+164/-124,6.506,0.4,16.13,8.01,1.91,0.668,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1524.0,+111/-86,7.158,0.393,16.37,7.52,2.93,0.045,✔️,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1502.0,+106/-85,7.674,0.332,17.33,12.89,289.35,0.08,✔️,0.0,False,CPU +17,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1475.0,+149/-129,8.076,0.341,18.53,5.34,4532.14,387.927,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1465.0,+149/-93,8.336,0.296,19.0,10.82,1993.57,0.06,✔️,0.0,False,GPU +19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1423.0,+116/-118,9.191,0.283,20.93,14.58,559.64,0.88,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1411.0,+152/-149,10.666,0.299,21.5,8.17,4532.14,39.452,✔️,0.0,False,GPU +21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1382.0,+104/-97,11.539,0.179,22.87,17.76,559.64,0.051,✔️,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1378.0,+131/-134,9.53,0.235,23.07,8.97,0.48,0.047,✔️,0.0,False,CPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1375.0,+127/-114,10.925,0.191,23.2,19.49,5.42,0.078,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1346.0,+80/-69,10.049,0.125,24.6,21.11,1803.23,1.199,✔️,0.0,False,CPU +25,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1331.0,+130/-148,13.131,0.21,25.33,17.48,1535.55,0.182,✔️,0.0,False,CPU +26,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1290.0,+130/-142,13.594,0.188,27.23,13.1,41.5,39.095,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1288.0,+81/-70,12.893,0.055,27.33,25.1,6.54,0.294,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1276.0,+95/-102,11.609,0.099,27.87,23.83,9.07,0.224,✔️,0.0,False,GPU +29,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1239.0,+83/-96,12.534,0.057,29.6,27.08,1803.23,0.051,✔️,0.0,False,CPU +30,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1236.0,+103/-99,12.859,0.048,29.73,24.05,65.73,0.23,✔️,0.0,False,CPU +31,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1222.0,+98/-130,12.599,0.044,30.33,27.5,36.73,0.359,✔️,0.0,False,CPU +32,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1218.0,+80/-93,12.025,0.03,30.53,28.9,0.34,0.07,✔️,0.0,False,CPU +33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1216.0,+135/-138,14.424,0.121,30.6,24.81,1535.55,0.013,✔️,0.0,False,CPU +34,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1204.0,+113/-137,15.52,0.109,31.13,27.61,2.99,0.016,✔️,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1203.0,+93/-86,13.195,0.037,31.17,27.59,65.73,0.025,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1189.0,+115/-139,15.027,0.063,31.8,26.56,354.91,1.636,✔️,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1148.0,+95/-121,14.978,0.021,33.53,31.16,36.73,0.038,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1074.0,+77/-88,16.346,0.005,36.47,35.3,2.58,0.039,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1049.0,+90/-96,17.593,0.011,37.4,35.95,354.91,0.094,✔️,0.0,False,CPU +40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1005.0,+164/-230,25.288,0.017,38.93,33.65,2.55,0.157,✔️,0.0,False,GPU +41,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+73/-113,18.839,0.0,39.1,38.22,0.24,0.022,✔️,0.0,False,CPU +42,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),977.0,+96/-131,22.57,0.011,39.87,37.93,150.58,4.952,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),933.0,+106/-153,21.097,0.015,41.2,39.26,1.49,0.113,✔️,0.0,False,CPU +44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),914.0,+167/-227,32.011,0.008,41.77,38.82,43.96,0.081,✔️,0.0,False,CPU +45,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),908.0,+108/-152,22.415,0.001,41.93,40.6,0.13,0.022,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),856.0,+156/-254,32.774,0.0,43.3,41.29,43.96,0.017,✔️,0.0,False,CPU +47,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),787.0,+59/-118,29.026,0.0,44.87,44.64,150.58,0.546,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),716.0,+107/-199,34.827,0.0,46.2,45.88,0.2,0.02,✔️,0.0,False,CPU +49,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),540.0,+101/-189,42.676,0.0,48.47,48.38,0.1,0.068,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_medium/winrate_matrix.png.zip index 855d27292f2f5ebf35cb689bcb7be40fb2201782..89a3b924987d574001bdf460591415c02b131f74 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:518a6a4dbcad5381156ad0db009272679376d99b4922d59efff7a12cad879cfc +oid sha256:5410cbb32d9e2aeff5bd82f889e9720c1d3ee926c6143e1e0ecf6bcf4bd72dad size 1388476 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index d4894317f7644fb88b1a41c8d2e2e7cf15be16d1..4117bbaa6d00050a56d8f38cefac37f0af46531e 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:80b1418472ffa11a08598dc879a89413c116cd506fd3a29db14b742e851e8e5a +oid sha256:91d739cbb1cfc689afaaf966ca7c72018a0b6a84c671bd03df26a16f539d06e7 size 407490 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_small/pareto_n_configs_imp.png.zip index 21f8adb761d8be307c5bf904e5de941cb0a0e8a6..a385570874e57edb00749df07790d76e2de6c4c7 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6831e05eb05b9dd58e2f230ae6526a890a379dc2d2e73b5ab601f77443415492 -size 971065 +oid sha256:74ed447e0ee8f207330e742b9b5c03021122de8c1251f8bffd4ec72aa438243d +size 954131 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_small/tuning-impact-elo.png.zip index 3c30eb94e21b49dc2cb39e4602a51c862acb1861..9543e285311e61d10b13baa29e245ba81cd37256 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:43fcba54351b0d0b5f70196fa648c5b62fac67d5fe68f268b9d5c21e02f66b4d +oid sha256:115bbdc8ff466f498833834d3eaaea57a96136b5abb5ec1631f1378ec2e150e6 size 114869 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_all/datasets_small/website_leaderboard.csv index 7f6a634a9368effd7419dcf0b09dd58ccebef717..be00b71d8fa0b31f4a52dd0993ac310691f7e032 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_small/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_all/datasets_small/website_leaderboard.csv @@ -1,52 +1,52 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1636.0,+133/-80,0.686,7.58,2.52,3.789,2319.48,8.957,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1601.0,+120/-70,0.613,8.71,4.3,6.609,6.58,0.586,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1586.0,+93/-71,0.607,9.25,4.24,7.334,657.21,7.989,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1583.0,+118/-73,0.605,9.35,3.72,6.191,2319.48,0.942,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1521.0,+89/-61,0.484,11.78,6.89,10.035,3855.22,24.935,0.0,False,GPU -5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1501.0,+59/-54,0.459,12.64,7.65,10.041,3351.09,1.56,0.0,False,CPU -6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1468.0,+97/-73,0.488,14.14,4.45,9.692,8059.77,364.288,0.0,False,GPU -7,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1456.0,+93/-77,0.474,14.71,5.45,12.275,5.11,0.624,0.0,False,GPU -8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1446.0,+76/-75,0.402,15.21,7.8,11.353,3855.22,0.954,0.0,False,GPU -9,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1426.0,+92/-83,0.438,16.17,5.13,10.94,8059.77,39.456,0.0,False,GPU -10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1410.0,+74/-63,0.35,17.03,9.34,12.139,3629.74,1.48,0.0,False,GPU -11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1395.0,+49/-43,0.264,17.76,14.34,13.451,833.03,2.182,0.0,False,CPU -12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1361.0,+67/-55,0.258,19.58,14.1,13.265,2448.79,0.814,0.0,False,CPU -13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1347.0,+74/-65,0.271,20.36,12.18,13.932,1346.37,2.149,0.0,False,GPU -14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1343.0,+73/-70,0.276,20.57,12.83,13.161,3629.74,0.189,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1337.0,+60/-54,0.23,20.9,15.67,13.815,2448.79,0.078,0.0,False,CPU -16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1336.0,+60/-43,0.207,20.96,14.19,14.459,886.51,1.982,0.0,False,CPU -17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1325.0,+71/-76,0.272,21.61,10.38,14.708,6341.76,8.556,0.0,False,GPU -18,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1322.0,+52/-49,0.198,21.78,17.8,14.478,833.03,0.435,0.0,False,CPU -19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1321.0,+62/-49,0.193,21.83,12.01,14.619,886.51,0.318,0.0,False,CPU -20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1320.0,+50/-45,0.181,21.85,16.14,14.531,8.39,0.139,0.0,False,CPU -21,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1317.0,+87/-85,0.321,22.03,7.85,13.538,51.78,39.927,0.0,False,GPU -22,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1309.0,+68/-67,0.231,22.5,10.84,14.095,6341.76,0.501,0.0,False,GPU -23,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1288.0,+68/-61,0.208,23.68,11.95,14.901,1346.37,0.103,0.0,False,GPU -24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1282.0,+74/-56,0.175,24.03,14.51,15.925,3249.24,0.548,0.0,False,CPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1279.0,+71/-62,0.215,24.17,15.22,15.488,13.71,0.149,0.0,False,GPU -26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1270.0,+62/-68,0.169,24.69,16.32,15.198,16.6,4.299,0.0,False,GPU -27,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1250.0,+50/-60,0.153,25.81,19.29,15.496,3693.32,2.14,0.0,False,CPU -28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1244.0,+64/-60,0.15,26.17,12.07,16.566,3249.24,0.059,0.0,False,CPU -29,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1213.0,+77/-61,0.127,27.92,14.0,17.458,9.93,0.052,0.0,False,CPU -30,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1212.0,+73/-78,0.125,27.97,13.54,18.902,14.9,0.323,0.0,False,GPU -31,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1208.0,+82/-76,0.138,28.17,19.51,19.083,497.58,1.147,0.0,False,CPU -32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1196.0,+64/-71,0.137,28.83,21.74,17.098,3693.32,0.131,0.0,False,CPU -33,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1188.0,+83/-87,0.136,29.26,20.25,18.568,1666.78,7.827,0.0,False,CPU -34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1185.0,+86/-87,0.15,29.46,16.02,19.577,497.58,0.163,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1148.0,+76/-74,0.093,31.44,23.1,20.32,575.62,1.155,0.0,False,CPU -36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1140.0,+56/-61,0.079,31.85,27.22,18.504,3.63,0.218,0.0,False,CPU -37,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1130.0,+56/-64,0.052,32.36,28.08,18.427,3.1,0.207,0.0,False,CPU -38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1128.0,+71/-93,0.106,32.47,20.92,19.793,1666.78,0.561,0.0,False,CPU -39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1099.0,+66/-74,0.046,33.99,27.62,21.333,575.62,0.147,0.0,False,CPU -40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1041.0,+93/-104,0.07,36.75,23.22,23.137,6.09,0.951,0.0,False,GPU -41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1011.0,+60/-89,0.028,38.11,33.87,22.946,12.68,0.155,0.0,False,CPU -42,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+72/-86,0.02,38.57,28.53,25.864,0.51,0.085,0.0,False,CPU -43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,988.0,+78/-86,0.018,39.08,35.07,26.623,0.43,0.081,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,979.0,+57/-96,0.03,39.44,35.96,23.922,5.83,0.571,0.0,False,CPU -45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,970.0,+65/-114,0.028,39.81,34.88,26.92,116.3,1.243,0.0,False,CPU -46,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,904.0,+86/-105,0.028,42.24,19.72,32.392,520.62,0.491,0.0,False,CPU -47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,874.0,+89/-114,0.024,43.19,27.61,32.903,520.62,0.097,0.0,False,CPU -48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,834.0,+102/-146,0.014,44.39,40.65,35.691,2.59,0.129,0.0,False,CPU -49,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,808.0,+77/-124,0.014,45.08,43.04,32.92,116.3,0.075,0.0,False,CPU -50,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,616.0,+96/-191,0.0,48.78,48.35,45.084,0.37,0.035,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1636.0,+133/-80,3.789,0.686,7.58,2.52,2319.48,8.957,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1601.0,+120/-70,6.609,0.613,8.71,4.3,6.58,0.586,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1586.0,+93/-71,7.334,0.607,9.25,4.24,657.21,7.989,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1583.0,+118/-73,6.191,0.605,9.35,3.72,2319.48,0.942,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1521.0,+89/-61,10.035,0.484,11.78,6.89,3855.22,24.935,✔️,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1501.0,+59/-54,10.041,0.459,12.64,7.65,3351.09,1.56,✔️,0.0,False,CPU +6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1468.0,+97/-73,9.692,0.488,14.14,4.45,8059.77,364.288,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1456.0,+93/-77,12.275,0.474,14.71,5.45,5.11,0.624,➖,0.0,False,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1446.0,+76/-75,11.353,0.402,15.21,7.8,3855.22,0.954,✔️,0.0,False,GPU +9,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1426.0,+92/-83,10.94,0.438,16.17,5.13,8059.77,39.456,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1410.0,+74/-63,12.139,0.35,17.03,9.34,3629.74,1.48,✔️,0.0,False,GPU +11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1395.0,+49/-43,13.451,0.264,17.76,14.34,833.03,2.182,✔️,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1361.0,+67/-55,13.265,0.258,19.58,14.1,2448.79,0.814,✔️,0.0,False,CPU +13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1347.0,+74/-65,13.932,0.271,20.36,12.18,1346.37,2.149,✔️,0.0,False,GPU +14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1343.0,+73/-70,13.161,0.276,20.57,12.83,3629.74,0.189,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1337.0,+60/-54,13.815,0.23,20.9,15.67,2448.79,0.078,✔️,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1336.0,+60/-43,14.459,0.207,20.96,14.19,886.51,1.982,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1325.0,+71/-76,14.708,0.272,21.61,10.38,6341.76,8.556,✔️,0.0,False,GPU +18,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1322.0,+52/-49,14.478,0.198,21.78,17.8,833.03,0.435,✔️,0.0,False,CPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1321.0,+62/-49,14.619,0.193,21.83,12.01,886.51,0.318,✔️,0.0,False,CPU +20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1320.0,+50/-45,14.531,0.181,21.85,16.14,8.39,0.139,✔️,0.0,False,CPU +21,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1317.0,+87/-85,13.538,0.321,22.03,7.85,51.78,39.927,✔️,0.0,False,GPU +22,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1309.0,+68/-67,14.095,0.231,22.5,10.84,6341.76,0.501,✔️,0.0,False,GPU +23,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1288.0,+68/-61,14.901,0.208,23.68,11.95,1346.37,0.103,✔️,0.0,False,GPU +24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1282.0,+74/-56,15.925,0.175,24.03,14.51,3249.24,0.548,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1279.0,+71/-62,15.488,0.215,24.17,15.22,13.71,0.149,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1270.0,+62/-68,15.198,0.169,24.69,16.32,16.6,4.299,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1250.0,+50/-60,15.496,0.153,25.81,19.29,3693.32,2.14,✔️,0.0,False,CPU +28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1244.0,+64/-60,16.566,0.15,26.17,12.07,3249.24,0.059,✔️,0.0,False,CPU +29,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1213.0,+77/-61,17.458,0.127,27.92,14.0,9.93,0.052,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1212.0,+73/-78,18.902,0.125,27.97,13.54,14.9,0.323,✔️,0.0,False,GPU +31,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1208.0,+82/-76,19.083,0.138,28.17,19.51,497.58,1.147,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1196.0,+64/-71,17.098,0.137,28.83,21.74,3693.32,0.131,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1188.0,+83/-87,18.568,0.136,29.26,20.25,1666.78,7.827,✔️,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1185.0,+86/-87,19.577,0.15,29.46,16.02,497.58,0.163,✔️,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1148.0,+76/-74,20.32,0.093,31.44,23.1,575.62,1.155,✔️,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1140.0,+56/-61,18.504,0.079,31.85,27.22,3.63,0.218,✔️,0.0,False,CPU +37,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1130.0,+56/-64,18.427,0.052,32.36,28.08,3.1,0.207,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1128.0,+71/-93,19.793,0.106,32.47,20.92,1666.78,0.561,✔️,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1099.0,+66/-74,21.333,0.046,33.99,27.62,575.62,0.147,✔️,0.0,False,CPU +40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1041.0,+93/-104,23.137,0.07,36.75,23.22,6.09,0.951,✔️,0.0,False,GPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1011.0,+60/-89,22.946,0.028,38.11,33.87,12.68,0.155,✔️,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+72/-86,25.864,0.02,38.57,28.53,0.51,0.085,✔️,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),988.0,+78/-86,26.623,0.018,39.08,35.07,0.43,0.081,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),979.0,+57/-96,23.922,0.03,39.44,35.96,5.83,0.571,✔️,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),970.0,+65/-114,26.92,0.028,39.81,34.88,116.3,1.243,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),904.0,+86/-105,32.392,0.028,42.24,19.72,520.62,0.491,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),874.0,+89/-114,32.903,0.024,43.19,27.61,520.62,0.097,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),834.0,+102/-146,35.691,0.014,44.39,40.65,2.59,0.129,✔️,0.0,False,CPU +49,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),808.0,+77/-124,32.92,0.014,45.08,43.04,116.3,0.075,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),616.0,+96/-191,45.084,0.0,48.78,48.35,0.37,0.035,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_small/winrate_matrix.png.zip index 06175863fae6274eb2d7d30dd8ef2c944826cf27..02bd67635acd2e4c7c57d502dc7c597c71a22ca8 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8080ebf1b937af12ee817a53b9dfc7c55193a86885889400f3d01df47b8d10fe +oid sha256:9c7038bc67071760bfefd8b80f28a534cb77859a0a2eeef694039b91e6d32f9e size 1755538 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index dab9e9a82a807625edc4dfb378b49ad557ca1a9d..2ec82b7644e0db3c85d9532612fc3d110c5a259d 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7b5b226efa578385cbd7c75214d056d74b2bea80633da9010082c5dadd60e1f0 +oid sha256:6a037e55d2eae743df3a904bd20f113f91fdb39d32a6d1d5f0c30a0136554a44 size 436370 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip index accc773041d8689cc277a0abe27a68cbcb49951c..0a2cb9ba8e4277a715fe6a20888f119dffa551ac 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2ce1d227a5d5353d2bc3e7da959e4ce1358609b45716e941793690feb613372e -size 1028883 +oid sha256:51da469fe975914a0c5714e7de394c3db21c44c9989feabd9bc000e08512d4d6 +size 1010232 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip index a06945e8dc24dcc41d8e72799dea6e1f110f64f4..8f448b80c2ee6ae2f02fddece319d0f512964ebe 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a041aff989a737503ffb2d0b30d8fd2ba6c324e75102f666055c013e4476645e +oid sha256:b5ef270574c931cdda27084bbff9f8968185d2eeac86a7447adb4d2b2d68395d size 116602 diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/website_leaderboard.csv index 97101c0243328cfe99938a4a6ed039932d45f645..2b2509f0606eb4c0fc2bea5440bc1b08a39c4d47 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/website_leaderboard.csv @@ -1,56 +1,56 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1662.0,+123/-83,0.681,7.94,2.68,4.083,2200.35,8.314,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1641.0,+86/-70,0.624,8.62,4.92,6.963,6.21,0.561,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1608.0,+112/-89,0.599,9.85,4.13,6.6,2200.35,0.882,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1581.0,+100/-81,0.579,10.88,4.62,7.89,625.63,7.662,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1544.0,+104/-79,0.512,12.47,5.25,9.548,3650.56,52.333,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1542.0,+73/-57,0.484,12.55,7.17,10.583,3565.03,23.942,0.0,False,GPU -6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1494.0,+64/-52,0.422,14.85,8.48,10.816,3035.75,1.498,0.0,False,CPU -7,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1492.0,+85/-71,0.426,14.95,7.23,10.796,137.76,3.046,0.0,False,GPU -8,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1478.0,+93/-77,0.462,15.65,5.54,12.913,4.66,0.611,0.0,False,GPU -9,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1469.0,+101/-74,0.458,16.12,4.82,10.352,7821.64,382.96,0.0,False,GPU -10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1462.0,+66/-66,0.396,16.47,8.59,11.945,3565.03,0.706,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1430.0,+110/-75,0.415,18.24,5.58,11.612,7821.64,39.414,0.0,False,GPU -12,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1419.0,+91/-76,0.355,18.83,10.17,12.117,3650.56,0.543,0.0,False,GPU -13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1405.0,+86/-73,0.325,19.64,10.49,12.789,3407.57,1.449,0.0,False,GPU -14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1376.0,+57/-44,0.204,21.32,17.79,14.548,780.12,2.129,0.0,False,CPU -15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1366.0,+66/-62,0.232,21.92,15.71,14.21,2234.83,0.653,0.0,False,CPU -16,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1356.0,+87/-62,0.324,22.52,8.57,13.444,4.97,0.545,0.0,False,GPU -17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1336.0,+83/-72,0.248,23.68,15.28,13.869,3407.57,0.173,0.0,False,GPU -18,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1334.0,+105/-86,0.327,23.82,7.82,14.157,51.52,40.033,0.0,False,GPU -19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1333.0,+73/-58,0.222,23.85,16.69,14.964,1216.65,2.007,0.0,False,GPU -20,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1333.0,+59/-51,0.194,23.85,18.67,14.839,2234.83,0.071,0.0,False,CPU -21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1327.0,+50/-54,0.165,24.2,19.28,15.598,848.99,1.866,0.0,False,CPU -22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1326.0,+95/-77,0.244,24.3,16.0,15.646,6308.39,7.737,0.0,False,GPU -23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1320.0,+58/-47,0.16,24.62,17.69,15.572,8.13,0.126,0.0,False,CPU -24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1317.0,+78/-84,0.217,24.82,12.21,14.8,6308.39,0.477,0.0,False,GPU -25,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1313.0,+60/-51,0.151,25.06,13.64,15.742,848.99,0.298,0.0,False,CPU -26,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1304.0,+72/-62,0.151,25.61,20.96,15.595,780.12,0.381,0.0,False,CPU -27,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1294.0,+73/-68,0.176,26.21,15.54,16.739,2961.52,0.521,0.0,False,CPU -28,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1290.0,+65/-57,0.164,26.45,16.92,15.72,16.29,4.082,0.0,False,GPU -29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1284.0,+80/-82,0.207,26.82,17.19,16.14,13.3,0.146,0.0,False,GPU -30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1274.0,+70/-67,0.157,27.44,19.34,15.938,1216.65,0.097,0.0,False,GPU -31,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1250.0,+74/-66,0.142,28.85,12.92,17.471,2961.52,0.054,0.0,False,CPU -32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1250.0,+68/-64,0.137,28.88,22.23,16.216,3460.09,2.057,0.0,False,CPU -33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1224.0,+80/-78,0.116,30.42,14.17,18.085,9.77,0.046,0.0,False,CPU -34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1223.0,+69/-63,0.113,30.52,13.73,19.271,14.5,0.3,0.0,False,GPU -35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1203.0,+87/-86,0.138,31.68,21.35,19.33,1453.07,6.678,0.0,False,CPU -36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1196.0,+73/-84,0.103,32.11,24.77,20.448,418.39,0.972,0.0,False,CPU -37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1183.0,+69/-66,0.1,32.85,26.26,17.821,3460.09,0.121,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1174.0,+74/-87,0.126,33.33,19.28,20.873,418.39,0.147,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1141.0,+80/-84,0.104,35.24,21.86,20.613,1453.07,0.637,0.0,False,CPU -40,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1134.0,+78/-99,0.067,35.61,26.54,21.785,563.93,0.997,0.0,False,CPU -41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1123.0,+53/-61,0.028,36.21,33.38,19.728,2.86,0.19,0.0,False,CPU -42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1112.0,+53/-68,0.037,36.83,33.62,19.956,3.28,0.179,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1091.0,+71/-90,0.042,37.89,31.53,22.688,563.93,0.149,0.0,False,CPU -44,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1048.0,+93/-107,0.073,40.09,23.76,24.424,5.55,0.867,0.0,False,GPU -45,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1011.0,+78/-85,0.028,41.82,36.56,23.923,9.39,0.15,0.0,False,CPU -46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+86/-110,0.021,42.29,32.32,26.668,0.48,0.074,0.0,False,CPU -47,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,988.0,+92/-122,0.02,42.79,38.51,27.3,0.38,0.075,0.0,False,CPU -48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,984.0,+82/-97,0.027,42.97,39.0,24.547,5.2,0.522,0.0,False,CPU -49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,958.0,+74/-86,0.007,44.06,41.64,28.135,101.2,1.135,0.0,False,CPU -50,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,919.0,+86/-112,0.03,45.55,19.46,32.177,497.63,0.481,0.0,False,CPU -51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,891.0,+85/-107,0.025,46.5,27.98,32.668,497.63,0.084,0.0,False,CPU -52,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,855.0,+97/-122,0.012,47.67,43.57,34.952,2.47,0.121,0.0,False,CPU -53,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,779.0,+85/-116,0.0,49.73,48.99,34.443,101.2,0.065,0.0,False,CPU -54,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,631.0,+109/-168,0.0,52.58,52.11,45.322,0.28,0.026,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1662.0,+123/-83,4.083,0.681,7.94,2.68,2200.35,8.314,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1641.0,+86/-70,6.963,0.624,8.62,4.92,6.21,0.561,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1608.0,+112/-89,6.6,0.599,9.85,4.13,2200.35,0.882,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1581.0,+100/-81,7.89,0.579,10.88,4.62,625.63,7.662,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1544.0,+104/-79,9.548,0.512,12.47,5.25,3650.56,52.333,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1542.0,+73/-57,10.583,0.484,12.55,7.17,3565.03,23.942,✔️,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1494.0,+64/-52,10.816,0.422,14.85,8.48,3035.75,1.498,✔️,0.0,False,CPU +7,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1492.0,+85/-71,10.796,0.426,14.95,7.23,137.76,3.046,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1478.0,+93/-77,12.913,0.462,15.65,5.54,4.66,0.611,➖,0.0,False,GPU +9,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1469.0,+101/-74,10.352,0.458,16.12,4.82,7821.64,382.96,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1462.0,+66/-66,11.945,0.396,16.47,8.59,3565.03,0.706,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1430.0,+110/-75,11.612,0.415,18.24,5.58,7821.64,39.414,✔️,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1419.0,+91/-76,12.117,0.355,18.83,10.17,3650.56,0.543,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1405.0,+86/-73,12.789,0.325,19.64,10.49,3407.57,1.449,✔️,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1376.0,+57/-44,14.548,0.204,21.32,17.79,780.12,2.129,✔️,0.0,False,CPU +15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1366.0,+66/-62,14.21,0.232,21.92,15.71,2234.83,0.653,✔️,0.0,False,CPU +16,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1356.0,+87/-62,13.444,0.324,22.52,8.57,4.97,0.545,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1336.0,+83/-72,13.869,0.248,23.68,15.28,3407.57,0.173,✔️,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1334.0,+105/-86,14.157,0.327,23.82,7.82,51.52,40.033,✔️,0.0,False,GPU +19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1333.0,+73/-58,14.964,0.222,23.85,16.69,1216.65,2.007,✔️,0.0,False,GPU +20,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1333.0,+59/-51,14.839,0.194,23.85,18.67,2234.83,0.071,✔️,0.0,False,CPU +21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1327.0,+50/-54,15.598,0.165,24.2,19.28,848.99,1.866,✔️,0.0,False,CPU +22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1326.0,+95/-77,15.646,0.244,24.3,16.0,6308.39,7.737,✔️,0.0,False,GPU +23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1320.0,+58/-47,15.572,0.16,24.62,17.69,8.13,0.126,✔️,0.0,False,CPU +24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1317.0,+78/-84,14.8,0.217,24.82,12.21,6308.39,0.477,✔️,0.0,False,GPU +25,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1313.0,+60/-51,15.742,0.151,25.06,13.64,848.99,0.298,✔️,0.0,False,CPU +26,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1304.0,+72/-62,15.595,0.151,25.61,20.96,780.12,0.381,✔️,0.0,False,CPU +27,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1294.0,+73/-68,16.739,0.176,26.21,15.54,2961.52,0.521,✔️,0.0,False,CPU +28,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1290.0,+65/-57,15.72,0.164,26.45,16.92,16.29,4.082,✔️,0.0,False,GPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1284.0,+80/-82,16.14,0.207,26.82,17.19,13.3,0.146,✔️,0.0,False,GPU +30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1274.0,+70/-67,15.938,0.157,27.44,19.34,1216.65,0.097,✔️,0.0,False,GPU +31,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1250.0,+74/-66,17.471,0.142,28.85,12.92,2961.52,0.054,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1250.0,+68/-64,16.216,0.137,28.88,22.23,3460.09,2.057,✔️,0.0,False,CPU +33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1224.0,+80/-78,18.085,0.116,30.42,14.17,9.77,0.046,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1223.0,+69/-63,19.271,0.113,30.52,13.73,14.5,0.3,✔️,0.0,False,GPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1203.0,+87/-86,19.33,0.138,31.68,21.35,1453.07,6.678,✔️,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1196.0,+73/-84,20.448,0.103,32.11,24.77,418.39,0.972,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1183.0,+69/-66,17.821,0.1,32.85,26.26,3460.09,0.121,✔️,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1174.0,+74/-87,20.873,0.126,33.33,19.28,418.39,0.147,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1141.0,+80/-84,20.613,0.104,35.24,21.86,1453.07,0.637,✔️,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1134.0,+78/-99,21.785,0.067,35.61,26.54,563.93,0.997,✔️,0.0,False,CPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1123.0,+53/-61,19.728,0.028,36.21,33.38,2.86,0.19,✔️,0.0,False,CPU +42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1112.0,+53/-68,19.956,0.037,36.83,33.62,3.28,0.179,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1091.0,+71/-90,22.688,0.042,37.89,31.53,563.93,0.149,✔️,0.0,False,CPU +44,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1048.0,+93/-107,24.424,0.073,40.09,23.76,5.55,0.867,✔️,0.0,False,GPU +45,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1011.0,+78/-85,23.923,0.028,41.82,36.56,9.39,0.15,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+86/-110,26.668,0.021,42.29,32.32,0.48,0.074,✔️,0.0,False,CPU +47,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),988.0,+92/-122,27.3,0.02,42.79,38.51,0.38,0.075,✔️,0.0,False,CPU +48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),984.0,+82/-97,24.547,0.027,42.97,39.0,5.2,0.522,✔️,0.0,False,CPU +49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),958.0,+74/-86,28.135,0.007,44.06,41.64,101.2,1.135,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),919.0,+86/-112,32.177,0.03,45.55,19.46,497.63,0.481,✔️,0.0,False,CPU +51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),891.0,+85/-107,32.668,0.025,46.5,27.98,497.63,0.084,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),855.0,+97/-122,34.952,0.012,47.67,43.57,2.47,0.121,✔️,0.0,False,CPU +53,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),779.0,+85/-116,34.443,0.0,49.73,48.99,101.2,0.065,✔️,0.0,False,CPU +54,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),631.0,+109/-168,45.322,0.0,52.58,52.11,0.28,0.026,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/winrate_matrix.png.zip index faa2a793636c251865ab2b008fe72ec139bb89c3..49707a685280d951e2ec78cd9a74c70ced604a01 100644 --- a/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_all/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2c1f706601fc4ac8b5895763a6a7cdf6331b39cd18e4c7f46613b74cd8ff0cb9 +oid sha256:df24977c336ef0261fe7105a917f6bf8e476e9e01cd601feaef0b2bac86afa73 size 1975031 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index 4f922b4bf5ef66ada3ef0b45ac2526b0bf9a6897..72bef4906d784f510ed721540d7edc808bc4580e 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0893b8d9b7118066ddcbd449d173c9aeefa2bf4a8cd56f01b671977fdd7a109c +oid sha256:5161b0a70ebf831535ad94964ddcfa48e5fe7c31b44599d9ca5fc246d7294076 size 440584 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip index 2e1db656a942071988a75d9a277edc1c9983d601..9f318a441520c5cc6d9a47623ceb1d771498256a 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d4b60d55c562818660f54351637dee3af17f83339b7e50d0e12c35b3134c6bfa -size 980762 +oid sha256:81ee7058d1801d257cfb925194c06aa707778878d949e6178589079565d79741 +size 955001 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_all/tuning-impact-elo.png.zip index bad542e5e4ce7dd94cfaec433c8c2b2386b9c67f..7d437bae9291427f982b44fd6377fce6f9224435 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:dfdf93aee84a27e96cbfc235152c7fc8e33c2229798e812cb802eac57f082785 +oid sha256:89f490d57e59d9ea4a2e5a4c010a537ef9c49409be22c5e7066c16501974ceef size 122415 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_binary/datasets_all/website_leaderboard.csv index 8d49af7208550bec5bee20cd0ca3cfd35f80781e..f2ea4ed05810176238cf6cfee34acb7f6d2a241b 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_all/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_all/website_leaderboard.csv @@ -1,53 +1,53 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1593.0,+118/-94,0.652,9.15,3.43,5.769,442.6,6.737,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1580.0,+109/-77,0.61,9.6,3.24,5.263,1877.76,10.745,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1580.0,+106/-77,0.598,9.63,4.44,7.063,1231.21,1.36,0.0,False,CPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1564.0,+103/-72,0.58,10.23,4.57,6.622,4.79,0.708,0.0,False,GPU -4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1550.0,+111/-66,0.574,10.77,4.42,6.293,1877.76,1.035,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1528.0,+75/-64,0.507,11.7,7.56,8.807,2721.85,9.311,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1446.0,+86/-72,0.42,15.52,6.33,10.77,2721.85,0.265,0.0,False,GPU -7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1437.0,+90/-66,0.415,16.0,6.81,10.645,2180.12,1.135,0.0,False,GPU -8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1414.0,+62/-51,0.31,17.22,12.93,12.259,328.64,0.774,0.0,False,CPU -9,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1390.0,+80/-71,0.311,18.52,10.69,12.822,462.92,0.613,0.0,False,CPU -10,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1389.0,+65/-54,0.315,18.53,9.21,11.84,3.84,0.068,0.0,False,CPU -11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1388.0,+81/-62,0.349,18.6,6.75,9.185,5890.6,409.986,0.0,False,GPU -12,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1387.0,+104/-82,0.342,18.65,8.68,11.692,3436.74,0.405,0.0,False,GPU -13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1375.0,+74/-75,0.324,19.3,11.38,11.971,1043.89,0.478,0.0,False,CPU -14,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1373.0,+104/-101,0.372,19.42,7.62,11.837,3436.74,8.478,0.0,False,GPU -15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1363.0,+94/-75,0.328,19.98,10.69,11.784,2180.12,0.117,0.0,False,GPU -16,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1351.0,+62/-60,0.287,20.67,13.8,12.509,1043.89,0.041,0.0,False,CPU -17,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1350.0,+77/-81,0.265,20.7,9.89,13.208,462.92,0.108,0.0,False,CPU -18,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1350.0,+102/-71,0.278,20.73,11.47,14.058,248.47,1.7,0.0,False,GPU -19,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1343.0,+84/-84,0.305,21.12,7.47,11.401,5890.6,39.565,0.0,False,GPU -20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1338.0,+62/-58,0.238,21.4,17.03,13.439,328.64,0.089,0.0,False,CPU -21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1332.0,+76/-56,0.23,21.7,13.18,14.224,1579.5,0.428,0.0,False,CPU -22,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1316.0,+78/-87,0.245,22.63,13.61,13.107,559.77,1.513,0.0,False,GPU -23,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1315.0,+56/-53,0.189,22.7,15.2,13.331,9.04,1.162,0.0,False,GPU -24,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1301.0,+94/-79,0.245,23.5,15.1,14.33,8.0,0.124,0.0,False,GPU -25,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1287.0,+72/-59,0.187,24.33,17.4,13.514,2206.58,2.301,0.0,False,CPU -26,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1272.0,+67/-73,0.177,25.18,17.14,14.761,559.77,0.086,0.0,False,GPU -27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1269.0,+68/-58,0.158,25.33,12.79,14.999,1579.5,0.05,0.0,False,CPU -28,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1266.0,+73/-64,0.167,25.53,12.12,15.996,4.79,0.045,0.0,False,CPU -29,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1231.0,+74/-95,0.189,27.52,10.76,14.605,47.89,39.446,0.0,False,GPU -30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1227.0,+75/-90,0.154,27.75,19.43,16.538,561.29,4.462,0.0,False,CPU -31,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1209.0,+88/-82,0.145,28.77,13.58,16.117,1.42,0.118,0.0,False,CPU -32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1204.0,+76/-65,0.14,29.1,22.97,15.932,2206.58,0.107,0.0,False,CPU -33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1195.0,+66/-70,0.12,29.6,12.96,16.944,11.15,0.312,0.0,False,GPU -34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1173.0,+71/-64,0.086,30.82,25.14,17.891,122.9,0.583,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1144.0,+70/-84,0.064,32.37,26.56,19.294,171.61,0.552,0.0,False,CPU -36,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1142.0,+83/-115,0.111,32.5,18.23,18.653,561.29,0.236,0.0,False,CPU -37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1140.0,+67/-71,0.074,32.62,28.52,17.125,0.93,0.094,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1131.0,+76/-90,0.093,33.07,19.56,19.504,122.9,0.072,0.0,False,CPU -39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1108.0,+66/-74,0.022,34.28,30.56,19.946,171.61,0.053,0.0,False,CPU -40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1041.0,+74/-76,0.044,37.6,32.81,21.201,4.97,0.092,0.0,False,CPU -41,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1013.0,+111/-133,0.08,38.87,29.06,26.33,0.81,0.115,0.0,False,GPU -42,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+77/-100,0.025,39.45,27.55,24.112,0.29,0.031,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1000.0,+76/-108,0.042,39.47,35.71,21.671,2.79,0.259,0.0,False,CPU -44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,984.0,+68/-95,0.006,40.13,38.17,24.306,140.66,3.209,0.0,False,CPU -45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,974.0,+97/-117,0.038,40.53,17.35,27.936,155.53,0.277,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,938.0,+95/-130,0.029,41.95,25.15,28.597,155.53,0.05,0.0,False,CPU -47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,938.0,+92/-121,0.047,41.97,36.6,26.856,2.71,0.392,0.0,False,GPU -48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,932.0,+96/-116,0.017,42.17,37.91,25.941,0.18,0.035,0.0,False,CPU -49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,902.0,+102/-109,0.017,43.27,39.53,30.347,0.8,0.088,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,816.0,+76/-102,0.0,45.87,45.13,30.586,140.66,0.188,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,600.0,+97/-211,0.0,50.0,49.65,43.693,0.15,0.024,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1593.0,+118/-94,5.769,0.652,9.15,3.43,442.6,6.737,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1580.0,+109/-77,5.263,0.61,9.6,3.24,1877.76,10.745,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1580.0,+106/-77,7.063,0.598,9.63,4.44,1231.21,1.36,✔️,0.0,False,CPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1564.0,+103/-72,6.622,0.58,10.23,4.57,4.79,0.708,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1550.0,+111/-66,6.293,0.574,10.77,4.42,1877.76,1.035,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1528.0,+75/-64,8.807,0.507,11.7,7.56,2721.85,9.311,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1446.0,+86/-72,10.77,0.42,15.52,6.33,2721.85,0.265,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1437.0,+90/-66,10.645,0.415,16.0,6.81,2180.12,1.135,✔️,0.0,False,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1414.0,+62/-51,12.259,0.31,17.22,12.93,328.64,0.774,✔️,0.0,False,CPU +9,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1390.0,+80/-71,12.822,0.311,18.52,10.69,462.92,0.613,✔️,0.0,False,CPU +10,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1389.0,+65/-54,11.84,0.315,18.53,9.21,3.84,0.068,✔️,0.0,False,CPU +11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1388.0,+81/-62,9.185,0.349,18.6,6.75,5890.6,409.986,✔️,0.0,False,GPU +12,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1387.0,+104/-82,11.692,0.342,18.65,8.68,3436.74,0.405,✔️,0.0,False,GPU +13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1375.0,+74/-75,11.971,0.324,19.3,11.38,1043.89,0.478,✔️,0.0,False,CPU +14,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1373.0,+104/-101,11.837,0.372,19.42,7.62,3436.74,8.478,✔️,0.0,False,GPU +15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1363.0,+94/-75,11.784,0.328,19.98,10.69,2180.12,0.117,✔️,0.0,False,GPU +16,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1351.0,+62/-60,12.509,0.287,20.67,13.8,1043.89,0.041,✔️,0.0,False,CPU +17,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1350.0,+77/-81,13.208,0.265,20.7,9.89,462.92,0.108,✔️,0.0,False,CPU +18,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1350.0,+102/-71,14.058,0.278,20.73,11.47,248.47,1.7,➖,0.0,False,GPU +19,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1343.0,+84/-84,11.401,0.305,21.12,7.47,5890.6,39.565,✔️,0.0,False,GPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1338.0,+62/-58,13.439,0.238,21.4,17.03,328.64,0.089,✔️,0.0,False,CPU +21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1332.0,+76/-56,14.224,0.23,21.7,13.18,1579.5,0.428,✔️,0.0,False,CPU +22,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1316.0,+78/-87,13.107,0.245,22.63,13.61,559.77,1.513,✔️,0.0,False,GPU +23,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1315.0,+56/-53,13.331,0.189,22.7,15.2,9.04,1.162,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1301.0,+94/-79,14.33,0.245,23.5,15.1,8.0,0.124,✔️,0.0,False,GPU +25,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1287.0,+72/-59,13.514,0.187,24.33,17.4,2206.58,2.301,✔️,0.0,False,CPU +26,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1272.0,+67/-73,14.761,0.177,25.18,17.14,559.77,0.086,✔️,0.0,False,GPU +27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1269.0,+68/-58,14.999,0.158,25.33,12.79,1579.5,0.05,✔️,0.0,False,CPU +28,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1266.0,+73/-64,15.996,0.167,25.53,12.12,4.79,0.045,✔️,0.0,False,CPU +29,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1231.0,+74/-95,14.605,0.189,27.52,10.76,47.89,39.446,✔️,0.0,False,GPU +30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1227.0,+75/-90,16.538,0.154,27.75,19.43,561.29,4.462,✔️,0.0,False,CPU +31,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1209.0,+88/-82,16.117,0.145,28.77,13.58,1.42,0.118,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1204.0,+76/-65,15.932,0.14,29.1,22.97,2206.58,0.107,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1195.0,+66/-70,16.944,0.12,29.6,12.96,11.15,0.312,✔️,0.0,False,GPU +34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1173.0,+71/-64,17.891,0.086,30.82,25.14,122.9,0.583,✔️,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1144.0,+70/-84,19.294,0.064,32.37,26.56,171.61,0.552,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1142.0,+83/-115,18.653,0.111,32.5,18.23,561.29,0.236,✔️,0.0,False,CPU +37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1140.0,+67/-71,17.125,0.074,32.62,28.52,0.93,0.094,✔️,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1131.0,+76/-90,19.504,0.093,33.07,19.56,122.9,0.072,✔️,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1108.0,+66/-74,19.946,0.022,34.28,30.56,171.61,0.053,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1041.0,+74/-76,21.201,0.044,37.6,32.81,4.97,0.092,✔️,0.0,False,CPU +41,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1013.0,+111/-133,26.33,0.08,38.87,29.06,0.81,0.115,➖,0.0,False,GPU +42,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+77/-100,24.112,0.025,39.45,27.55,0.29,0.031,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1000.0,+76/-108,21.671,0.042,39.47,35.71,2.79,0.259,✔️,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),984.0,+68/-95,24.306,0.006,40.13,38.17,140.66,3.209,✔️,0.0,False,CPU +45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),974.0,+97/-117,27.936,0.038,40.53,17.35,155.53,0.277,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),938.0,+95/-130,28.597,0.029,41.95,25.15,155.53,0.05,✔️,0.0,False,CPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),938.0,+92/-121,26.856,0.047,41.97,36.6,2.71,0.392,✔️,0.0,False,GPU +48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),932.0,+96/-116,25.941,0.017,42.17,37.91,0.18,0.035,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),902.0,+102/-109,30.347,0.017,43.27,39.53,0.8,0.088,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),816.0,+76/-102,30.586,0.0,45.87,45.13,140.66,0.188,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),600.0,+97/-211,43.693,0.0,50.0,49.65,0.15,0.024,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_all/winrate_matrix.png.zip index 7af4d1791b31c2be9cc0c9f2e4f9e5c961a3698f..21dff1c633778c0e0c7f113a248dffa5c1e1689d 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2594cdd792059af937d31e9363f6a3d305209ab6c6900399450bc2f6d47e5c93 +oid sha256:9aa6ca4ec171e2913de5d7aceca21dd3a1013adbd31555704cbdde2447c84609 size 1822134 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index 4427220a9c1938cbf6be2aa3a4726d63d6f4e4bb..e62e84aaf26cdff335d76561e9cc4ad6cae1745a 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:85f32cacb3f4a35f820052ff92f31175f92f419c549e9baff27fef1392c634a1 +oid sha256:81450a2529565558aed95b36264f627abe0bcbe8edad4c195ea612e2d223dbee size 431992 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip index dd0f5f80be319cd8a628fa131cb4140228eea878..e3fd8a97ec3ca36d349ef97b2a93865c94b57a38 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fffbc9d93552f2469ea3584acb346dcf3407cf00d192a8dd879f095747b5fd25 -size 994534 +oid sha256:75842db9e1a6a82d9e096f53ca36205440c3a539473f83ea6cbfcc676747b09b +size 927206 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/tuning-impact-elo.png.zip index 488575eebfdf086994283a700ca4c144e7622914..e7c104317bb1afa443c92566ac07bf58bacff8ec 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:56b287b209e5dd2042b0657e5b098681e6015d5396c7f307ffdb041063f07a9f +oid sha256:9fbcf2445416dccaa6f8874e26a565a6487cdc33ec2372302e129543afc3c9f4 size 120976 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/website_leaderboard.csv index f61a3bf6a31bef9e066a755c2a294347f2835805..dbc80d37b1bc450cc0369387db113d169aa7336c 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/website_leaderboard.csv @@ -1,54 +1,54 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,2079.0,+240/-68,0.866,3.33,2.13,1.537,310.55,1.824,0.0,False,CPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,2016.0,+291/-84,0.837,4.22,2.22,2.096,256.14,2.868,0.0,False,GPU -2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1765.0,+277/-164,0.621,10.56,3.82,4.453,2149.06,6.373,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1743.0,+238/-173,0.527,11.33,4.11,5.875,841.84,11.882,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1734.0,+140/-56,0.515,11.67,8.77,5.562,1655.56,2.483,0.0,False,GPU -5,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1725.0,+191/-108,0.458,12.0,8.28,7.345,114.48,0.404,0.0,False,CPU -6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1705.0,+217/-120,0.517,12.78,7.57,6.556,618.23,0.184,0.0,False,CPU -7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1688.0,+228/-141,0.45,13.44,7.6,7.872,841.84,1.263,0.0,False,GPU -8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1686.0,+225/-118,0.477,13.56,5.11,7.382,807.72,0.575,0.0,False,GPU -9,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1678.0,+210/-108,0.431,13.89,9.94,8.481,87.3,0.533,0.0,False,CPU -10,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1676.0,+166/-104,0.487,13.94,8.68,6.655,618.23,0.031,0.0,False,CPU -11,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1656.0,+188/-119,0.422,14.78,10.05,9.295,2149.06,0.318,0.0,False,GPU -12,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1653.0,+159/-92,0.458,14.94,5.57,6.571,2.12,0.043,0.0,False,CPU -13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1631.0,+170/-121,0.399,15.89,10.8,7.667,114.48,0.058,0.0,False,CPU -14,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1628.0,+183/-107,0.39,16.0,9.31,8.174,1655.56,0.086,0.0,False,GPU -15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1596.0,+183/-119,0.328,17.44,13.37,9.425,87.3,0.069,0.0,False,CPU -16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1594.0,+229/-161,0.393,17.56,8.34,8.03,807.72,0.07,0.0,False,GPU -17,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1562.0,+182/-145,0.326,19.0,8.97,7.783,2.32,0.889,0.0,False,GPU -18,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1548.0,+184/-161,0.295,19.67,9.57,9.48,9.96,4.847,0.0,False,GPU -19,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1488.0,+188/-198,0.248,22.56,18.43,10.588,5.77,0.078,0.0,False,GPU -20,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1475.0,+188/-199,0.267,23.22,6.55,10.147,0.48,0.047,0.0,False,CPU -21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1473.0,+155/-148,0.247,23.33,15.26,9.606,559.89,1.156,0.0,False,GPU -22,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1473.0,+120/-74,0.175,23.33,19.14,9.902,1128.19,1.674,0.0,False,CPU -23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1457.0,+181/-128,0.212,24.11,20.3,13.211,262.92,0.205,0.0,False,CPU -24,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1453.0,+174/-154,0.181,24.33,18.53,10.102,4560.06,421.968,0.0,False,GPU -25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1444.0,+165/-123,0.166,24.78,17.43,12.787,559.89,0.056,0.0,False,GPU -26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1424.0,+89/-76,0.079,25.78,23.59,11.863,6.55,0.295,0.0,False,GPU -27,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1390.0,+114/-157,0.11,27.44,22.3,12.364,368.88,2.507,0.0,False,CPU -28,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1376.0,+211/-169,0.137,28.17,17.73,13.801,4560.06,39.452,0.0,False,GPU -29,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1347.0,+190/-192,0.13,29.56,19.19,13.488,92.9,0.786,0.0,False,GPU -30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1333.0,+181/-120,0.073,30.22,27.34,16.174,0.92,0.018,0.0,False,CPU -31,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1324.0,+203/-161,0.093,30.67,25.33,14.896,262.92,0.015,0.0,False,CPU -32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1317.0,+122/-138,0.079,31.0,27.5,13.481,1128.19,0.055,0.0,False,CPU -33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1305.0,+130/-166,0.036,31.56,28.19,13.452,40.45,0.331,0.0,False,CPU -34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1300.0,+57/-72,0.018,31.78,30.79,14.731,43.93,0.204,0.0,False,CPU -35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1271.0,+112/-155,0.042,33.11,30.95,12.558,0.34,0.052,0.0,False,CPU -36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1257.0,+53/-66,0.001,33.78,33.41,15.152,43.93,0.018,0.0,False,CPU -37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1234.0,+108/-127,0.01,34.78,33.46,14.1,9.93,0.224,0.0,False,GPU -38,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1220.0,+141/-229,0.025,35.39,31.53,18.215,41.8,39.131,0.0,False,GPU -39,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1198.0,+131/-157,0.022,36.33,33.39,17.05,40.45,0.028,0.0,False,CPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1185.0,+110/-188,0.018,36.89,34.91,16.514,368.88,0.084,0.0,False,CPU -41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1144.0,+102/-133,0.01,38.56,37.02,16.247,1.81,0.048,0.0,False,CPU -42,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1107.0,+174/-278,0.016,39.94,35.92,25.559,43.96,0.099,0.0,False,CPU -43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1102.0,+121/-204,0.02,40.11,37.38,20.908,514.21,52.341,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1071.0,+139/-245,0.025,41.22,38.13,19.261,1.61,0.113,0.0,False,CPU -45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1057.0,+170/-263,0.002,41.72,38.93,26.294,43.96,0.024,0.0,False,CPU -46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+126/-288,0.0,43.56,42.51,22.447,0.11,0.012,0.0,False,CPU -47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,898.0,+241/-628,0.027,46.33,38.97,30.801,2.77,0.157,0.0,False,GPU -48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,884.0,+102/-218,0.0,46.67,46.31,28.037,0.22,0.022,0.0,False,CPU -49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,860.0,+106/-244,0.0,47.22,46.92,25.027,0.1,0.02,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,850.0,+64/-159,0.0,47.44,47.16,28.826,514.21,2.49,0.0,False,CPU -51,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,823.0,+118/-263,0.0,48.0,47.49,33.173,0.2,0.035,0.0,False,GPU -52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,491.0,+97/-381,0.0,52.11,52.08,46.122,0.11,0.089,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",2079.0,+240/-68,1.537,0.866,3.33,2.13,310.55,1.824,✔️,0.0,False,CPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",2016.0,+291/-84,2.096,0.837,4.22,2.22,256.14,2.868,✔️,0.0,False,GPU +2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1765.0,+277/-164,4.453,0.621,10.56,3.82,2149.06,6.373,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1743.0,+238/-173,5.875,0.527,11.33,4.11,841.84,11.882,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1734.0,+140/-56,5.562,0.515,11.67,8.77,1655.56,2.483,✔️,0.0,False,GPU +5,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1725.0,+191/-108,7.345,0.458,12.0,8.28,114.48,0.404,✔️,0.0,False,CPU +6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1705.0,+217/-120,6.556,0.517,12.78,7.57,618.23,0.184,✔️,0.0,False,CPU +7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1688.0,+228/-141,7.872,0.45,13.44,7.6,841.84,1.263,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1686.0,+225/-118,7.382,0.477,13.56,5.11,807.72,0.575,✔️,0.0,False,GPU +9,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1678.0,+210/-108,8.481,0.431,13.89,9.94,87.3,0.533,✔️,0.0,False,CPU +10,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1676.0,+166/-104,6.655,0.487,13.94,8.68,618.23,0.031,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1656.0,+188/-119,9.295,0.422,14.78,10.05,2149.06,0.318,✔️,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1653.0,+159/-92,6.571,0.458,14.94,5.57,2.12,0.043,✔️,0.0,False,CPU +13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1631.0,+170/-121,7.667,0.399,15.89,10.8,114.48,0.058,✔️,0.0,False,CPU +14,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1628.0,+183/-107,8.174,0.39,16.0,9.31,1655.56,0.086,✔️,0.0,False,GPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1596.0,+183/-119,9.425,0.328,17.44,13.37,87.3,0.069,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1594.0,+229/-161,8.03,0.393,17.56,8.34,807.72,0.07,✔️,0.0,False,GPU +17,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1562.0,+182/-145,7.783,0.326,19.0,8.97,2.32,0.889,✔️,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1548.0,+184/-161,9.48,0.295,19.67,9.57,9.96,4.847,✔️,0.0,False,GPU +19,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1488.0,+188/-198,10.588,0.248,22.56,18.43,5.77,0.078,✔️,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1475.0,+188/-199,10.147,0.267,23.22,6.55,0.48,0.047,✔️,0.0,False,CPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1473.0,+155/-148,9.606,0.247,23.33,15.26,559.89,1.156,✔️,0.0,False,GPU +22,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1473.0,+120/-74,9.902,0.175,23.33,19.14,1128.19,1.674,✔️,0.0,False,CPU +23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1457.0,+181/-128,13.211,0.212,24.11,20.3,262.92,0.205,✔️,0.0,False,CPU +24,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1453.0,+174/-154,10.102,0.181,24.33,18.53,4560.06,421.968,✔️,0.0,False,GPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1444.0,+165/-123,12.787,0.166,24.78,17.43,559.89,0.056,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1424.0,+89/-76,11.863,0.079,25.78,23.59,6.55,0.295,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1390.0,+114/-157,12.364,0.11,27.44,22.3,368.88,2.507,✔️,0.0,False,CPU +28,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1376.0,+211/-169,13.801,0.137,28.17,17.73,4560.06,39.452,✔️,0.0,False,GPU +29,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1347.0,+190/-192,13.488,0.13,29.56,19.19,92.9,0.786,➖,0.0,False,GPU +30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1333.0,+181/-120,16.174,0.073,30.22,27.34,0.92,0.018,✔️,0.0,False,CPU +31,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1324.0,+203/-161,14.896,0.093,30.67,25.33,262.92,0.015,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1317.0,+122/-138,13.481,0.079,31.0,27.5,1128.19,0.055,✔️,0.0,False,CPU +33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1305.0,+130/-166,13.452,0.036,31.56,28.19,40.45,0.331,✔️,0.0,False,CPU +34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1300.0,+57/-72,14.731,0.018,31.78,30.79,43.93,0.204,✔️,0.0,False,CPU +35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1271.0,+112/-155,12.558,0.042,33.11,30.95,0.34,0.052,✔️,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1257.0,+53/-66,15.152,0.001,33.78,33.41,43.93,0.018,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1234.0,+108/-127,14.1,0.01,34.78,33.46,9.93,0.224,✔️,0.0,False,GPU +38,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1220.0,+141/-229,18.215,0.025,35.39,31.53,41.8,39.131,✔️,0.0,False,GPU +39,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1198.0,+131/-157,17.05,0.022,36.33,33.39,40.45,0.028,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1185.0,+110/-188,16.514,0.018,36.89,34.91,368.88,0.084,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1144.0,+102/-133,16.247,0.01,38.56,37.02,1.81,0.048,✔️,0.0,False,CPU +42,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1107.0,+174/-278,25.559,0.016,39.94,35.92,43.96,0.099,✔️,0.0,False,CPU +43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1102.0,+121/-204,20.908,0.02,40.11,37.38,514.21,52.341,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1071.0,+139/-245,19.261,0.025,41.22,38.13,1.61,0.113,✔️,0.0,False,CPU +45,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1057.0,+170/-263,26.294,0.002,41.72,38.93,43.96,0.024,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+126/-288,22.447,0.0,43.56,42.51,0.11,0.012,✔️,0.0,False,CPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),898.0,+241/-628,30.801,0.027,46.33,38.97,2.77,0.157,✔️,0.0,False,GPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),884.0,+102/-218,28.037,0.0,46.67,46.31,0.22,0.022,✔️,0.0,False,CPU +49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),860.0,+106/-244,25.027,0.0,47.22,46.92,0.1,0.02,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),850.0,+64/-159,28.826,0.0,47.44,47.16,514.21,2.49,✔️,0.0,False,CPU +51,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),823.0,+118/-263,33.173,0.0,48.0,47.49,0.2,0.035,➖,0.0,False,GPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),491.0,+97/-381,46.122,0.0,52.11,52.08,0.11,0.089,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/winrate_matrix.png.zip index 7a1481066ba52bd95038f1dd68e123c5ca6dbf8b..92c23bb6a2af41170a11a7bdef81503d0a91dd80 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f4f3a4b13730e439be35e150444b8ab8964d84f0eda7fd9755d501314790d895 +oid sha256:92b6d384d79fa1d0954e106681952232f0c41c64ad197237057eb230808cff08 size 1490980 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index d6bfcdd11a6466ae46505098275983f7ccdce699..0c82f0b250cbcadc9a3817fd76e5e8e2e4cfe4bc 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b9e5ff714838dadd692110df3542d134826a4b5c37a191fd57bb6ec3b6f5a595 +oid sha256:37b1aa80825290d522b308ae9b2f6505be4c33febf780a766f6718feaddc569e size 466124 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip index d387fe6da682dbf737dcf59b92cb47afdcd8e4de..76bb300c22481f8c84ae52b812b9d848739bb8d0 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:214e39ac534cf6c8c4e7f3893a52eb70ecde041dcdeddfc718a01bbab271ff32 -size 962589 +oid sha256:289fe235e4e8107b3852e9f1ed2ccdf6b908d4df1e5951b4053e1c5dd9c8d3ad +size 957268 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_small/tuning-impact-elo.png.zip index 6f347e416c4b048bd9aee68239b585fac23f546a..ae8b7aef0fb173779ad38eed586885de6cc09eeb 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:eb1b2bf76a31a37f8a62248df6dfa0f3c5e2dcb13ed3ab8f7e253071b91551a5 +oid sha256:a21daf9761de761552eb0b92ff1bcfaa697d7dd20a61ad5f7bd2e4cf0bc92270 size 115824 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_binary/datasets_small/website_leaderboard.csv index d22508901ed91885d279572be2130babbc5e0544..af456e458133095b3cda755fc8cc6f847ce53b5a 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_small/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_small/website_leaderboard.csv @@ -1,54 +1,54 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1635.0,+100/-58,0.669,6.71,3.81,6.489,5.53,0.55,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1560.0,+146/-88,0.628,9.14,3.21,5.365,2125.17,9.785,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1535.0,+151/-108,0.607,10.05,3.84,5.981,2125.17,0.882,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1494.0,+147/-96,0.552,11.76,4.59,7.707,613.33,8.472,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1480.0,+91/-88,0.469,12.38,7.57,10.57,2791.97,23.942,0.0,False,GPU -5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1470.0,+101/-81,0.465,12.81,9.12,9.79,1886.83,1.333,0.0,False,CPU -6,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1420.0,+121/-84,0.459,15.26,5.29,10.535,4.14,0.557,0.0,False,GPU -7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1405.0,+110/-75,0.395,16.02,6.49,12.26,2791.97,0.544,0.0,False,GPU -8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1386.0,+108/-84,0.413,17.05,5.48,9.125,6616.87,408.881,0.0,False,GPU -9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1373.0,+114/-86,0.361,17.76,8.13,12.409,2237.77,1.273,0.0,False,GPU -10,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1372.0,+126/-89,0.33,17.81,10.09,14.65,298.33,2.036,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1350.0,+95/-92,0.371,19.0,6.14,10.701,6616.87,39.713,0.0,False,GPU -12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1341.0,+69/-50,0.247,19.55,15.7,14.232,512.11,1.17,0.0,False,CPU -13,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1316.0,+82/-52,0.233,20.98,14.65,14.463,4.51,0.124,0.0,False,CPU -14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1314.0,+154/-86,0.295,21.12,8.66,13.061,4618.5,0.426,0.0,False,GPU -15,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1304.0,+95/-85,0.223,21.67,12.32,15.018,1640.58,0.552,0.0,False,CPU -16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1301.0,+114/-106,0.273,21.83,12.55,13.757,2237.77,0.147,0.0,False,GPU -17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1296.0,+91/-80,0.229,22.17,12.6,15.531,700.96,0.743,0.0,False,CPU -18,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1291.0,+81/-60,0.211,22.43,13.73,14.324,11.09,2.46,0.0,False,GPU -19,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1283.0,+89/-81,0.218,22.95,15.45,14.659,1103.84,0.559,0.0,False,CPU -20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1277.0,+88/-86,0.227,23.29,14.99,14.953,536.23,1.738,0.0,False,GPU -21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1271.0,+78/-80,0.197,23.67,9.72,15.938,700.96,0.203,0.0,False,CPU -22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1265.0,+125/-91,0.252,24.02,13.76,15.354,4618.5,9.218,0.0,False,GPU -23,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1264.0,+69/-68,0.187,24.05,20.71,15.511,512.11,0.122,0.0,False,CPU -24,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1264.0,+82/-66,0.176,24.1,10.77,15.396,1640.58,0.059,0.0,False,CPU -25,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1258.0,+84/-78,0.189,24.45,18.99,15.373,1103.84,0.051,0.0,False,CPU -26,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1256.0,+90/-88,0.185,24.57,10.24,16.286,7.16,0.046,0.0,False,CPU -27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1251.0,+103/-86,0.231,24.86,14.35,16.289,11.3,0.149,0.0,False,GPU -28,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1245.0,+106/-96,0.257,25.19,8.48,13.383,52.04,39.82,0.0,False,GPU -29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1236.0,+100/-83,0.184,25.76,17.18,15.412,2535.0,2.546,0.0,False,CPU -30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1223.0,+86/-93,0.167,26.55,18.75,15.96,536.23,0.097,0.0,False,GPU -31,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1191.0,+106/-102,0.153,28.48,10.4,18.493,13.39,0.329,0.0,False,GPU -32,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1183.0,+100/-107,0.164,28.93,19.1,18.662,650.63,5.616,0.0,False,CPU -33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1176.0,+98/-91,0.16,29.38,22.22,17.319,2535.0,0.112,0.0,False,CPU -34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1138.0,+87/-95,0.106,31.6,24.84,20.126,206.04,0.946,0.0,False,CPU -35,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1136.0,+107/-144,0.145,31.71,17.48,19.888,650.63,0.442,0.0,False,CPU -36,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1127.0,+84/-79,0.089,32.24,26.22,19.028,1.98,0.122,0.0,False,CPU -37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1117.0,+106/-113,0.122,32.81,16.91,20.888,206.04,0.098,0.0,False,CPU -38,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1101.0,+75/-85,0.081,33.69,28.83,19.426,2.47,0.168,0.0,False,CPU -39,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1098.0,+92/-102,0.079,33.86,26.1,21.6,494.48,0.936,0.0,False,CPU -40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1064.0,+84/-103,0.029,35.74,30.46,22.348,494.48,0.093,0.0,False,CPU -41,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1053.0,+114/-152,0.105,36.29,25.95,23.728,1.29,0.149,0.0,False,GPU -42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1011.0,+84/-133,0.049,38.43,32.24,23.67,7.76,0.147,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+105/-146,0.034,38.98,24.44,25.154,0.39,0.041,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,978.0,+79/-121,0.039,40.0,36.13,23.027,3.48,0.427,0.0,False,CPU -45,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,947.0,+116/-143,0.023,41.38,36.36,26.643,0.33,0.044,0.0,False,CPU -46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,947.0,+96/-134,0.05,41.38,36.81,25.487,2.66,0.702,0.0,False,GPU -47,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,946.0,+68/-106,0.0,41.43,39.93,26.094,89.61,1.627,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,931.0,+108/-155,0.048,42.07,14.35,29.287,280.42,0.382,0.0,False,CPU -49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,903.0,+115/-164,0.024,43.19,38.32,31.672,1.48,0.127,0.0,False,CPU -50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,899.0,+113/-150,0.041,43.33,22.16,29.916,280.42,0.072,0.0,False,CPU -51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,800.0,+87/-136,0.0,46.62,45.73,31.665,89.61,0.08,0.0,False,CPU -52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,615.0,+107/-184,0.0,50.52,50.07,42.96,0.24,0.02,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1635.0,+100/-58,6.489,0.669,6.71,3.81,5.53,0.55,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1560.0,+146/-88,5.365,0.628,9.14,3.21,2125.17,9.785,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1535.0,+151/-108,5.981,0.607,10.05,3.84,2125.17,0.882,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1494.0,+147/-96,7.707,0.552,11.76,4.59,613.33,8.472,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1480.0,+91/-88,10.57,0.469,12.38,7.57,2791.97,23.942,✔️,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1470.0,+101/-81,9.79,0.465,12.81,9.12,1886.83,1.333,✔️,0.0,False,CPU +6,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1420.0,+121/-84,10.535,0.459,15.26,5.29,4.14,0.557,➖,0.0,False,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1405.0,+110/-75,12.26,0.395,16.02,6.49,2791.97,0.544,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1386.0,+108/-84,9.125,0.413,17.05,5.48,6616.87,408.881,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1373.0,+114/-86,12.409,0.361,17.76,8.13,2237.77,1.273,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1372.0,+126/-89,14.65,0.33,17.81,10.09,298.33,2.036,➖,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1350.0,+95/-92,10.701,0.371,19.0,6.14,6616.87,39.713,✔️,0.0,False,GPU +12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1341.0,+69/-50,14.232,0.247,19.55,15.7,512.11,1.17,✔️,0.0,False,CPU +13,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1316.0,+82/-52,14.463,0.233,20.98,14.65,4.51,0.124,✔️,0.0,False,CPU +14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1314.0,+154/-86,13.061,0.295,21.12,8.66,4618.5,0.426,✔️,0.0,False,GPU +15,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1304.0,+95/-85,15.018,0.223,21.67,12.32,1640.58,0.552,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1301.0,+114/-106,13.757,0.273,21.83,12.55,2237.77,0.147,✔️,0.0,False,GPU +17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1296.0,+91/-80,15.531,0.229,22.17,12.6,700.96,0.743,✔️,0.0,False,CPU +18,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1291.0,+81/-60,14.324,0.211,22.43,13.73,11.09,2.46,✔️,0.0,False,GPU +19,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1283.0,+89/-81,14.659,0.218,22.95,15.45,1103.84,0.559,✔️,0.0,False,CPU +20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1277.0,+88/-86,14.953,0.227,23.29,14.99,536.23,1.738,✔️,0.0,False,GPU +21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1271.0,+78/-80,15.938,0.197,23.67,9.72,700.96,0.203,✔️,0.0,False,CPU +22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1265.0,+125/-91,15.354,0.252,24.02,13.76,4618.5,9.218,✔️,0.0,False,GPU +23,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1264.0,+69/-68,15.511,0.187,24.05,20.71,512.11,0.122,✔️,0.0,False,CPU +24,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1264.0,+82/-66,15.396,0.176,24.1,10.77,1640.58,0.059,✔️,0.0,False,CPU +25,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1258.0,+84/-78,15.373,0.189,24.45,18.99,1103.84,0.051,✔️,0.0,False,CPU +26,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1256.0,+90/-88,16.286,0.185,24.57,10.24,7.16,0.046,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1251.0,+103/-86,16.289,0.231,24.86,14.35,11.3,0.149,✔️,0.0,False,GPU +28,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1245.0,+106/-96,13.383,0.257,25.19,8.48,52.04,39.82,✔️,0.0,False,GPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1236.0,+100/-83,15.412,0.184,25.76,17.18,2535.0,2.546,✔️,0.0,False,CPU +30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1223.0,+86/-93,15.96,0.167,26.55,18.75,536.23,0.097,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1191.0,+106/-102,18.493,0.153,28.48,10.4,13.39,0.329,✔️,0.0,False,GPU +32,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1183.0,+100/-107,18.662,0.164,28.93,19.1,650.63,5.616,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1176.0,+98/-91,17.319,0.16,29.38,22.22,2535.0,0.112,✔️,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1138.0,+87/-95,20.126,0.106,31.6,24.84,206.04,0.946,✔️,0.0,False,CPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1136.0,+107/-144,19.888,0.145,31.71,17.48,650.63,0.442,✔️,0.0,False,CPU +36,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1127.0,+84/-79,19.028,0.089,32.24,26.22,1.98,0.122,✔️,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1117.0,+106/-113,20.888,0.122,32.81,16.91,206.04,0.098,✔️,0.0,False,CPU +38,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1101.0,+75/-85,19.426,0.081,33.69,28.83,2.47,0.168,✔️,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1098.0,+92/-102,21.6,0.079,33.86,26.1,494.48,0.936,✔️,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1064.0,+84/-103,22.348,0.029,35.74,30.46,494.48,0.093,✔️,0.0,False,CPU +41,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1053.0,+114/-152,23.728,0.105,36.29,25.95,1.29,0.149,➖,0.0,False,GPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1011.0,+84/-133,23.67,0.049,38.43,32.24,7.76,0.147,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+105/-146,25.154,0.034,38.98,24.44,0.39,0.041,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),978.0,+79/-121,23.027,0.039,40.0,36.13,3.48,0.427,✔️,0.0,False,CPU +45,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),947.0,+116/-143,26.643,0.023,41.38,36.36,0.33,0.044,✔️,0.0,False,CPU +46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),947.0,+96/-134,25.487,0.05,41.38,36.81,2.66,0.702,✔️,0.0,False,GPU +47,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),946.0,+68/-106,26.094,0.0,41.43,39.93,89.61,1.627,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),931.0,+108/-155,29.287,0.048,42.07,14.35,280.42,0.382,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),903.0,+115/-164,31.672,0.024,43.19,38.32,1.48,0.127,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),899.0,+113/-150,29.916,0.041,43.33,22.16,280.42,0.072,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),800.0,+87/-136,31.665,0.0,46.62,45.73,89.61,0.08,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),615.0,+107/-184,42.96,0.0,50.52,50.07,0.24,0.02,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_small/winrate_matrix.png.zip index a4d0d7be8d675781ce84bcde1bfe0aa6da3e1ac4..8a0590fd1fb6301747ee7da428415f13b2fd8062 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a20bcd1caa820cbdbe9b800eeddde82e64653263e2db4cb100aa5b2ebcc2cfce +oid sha256:933a4cd30b519b539a9c8f72be249b9ea9d2b56c98d15b6eb6c5471555e6ca75 size 1849420 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index a075b356063975e1d2271c145a8bba070b3dbd07..304c71bdc61ed19882b123373741fc17957f201f 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d84bd738950c43f964a4eb10aa64dd17ba0991b218e52b39501705c9f006d48d +oid sha256:bf77c37a3d2bb57e48e80595f15af3386fcace5f8426d01ccb0c6175c4911b6c size 495281 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip index 392052993d053110bedf3786622bfb02bf1f5b51..71f50d1619f4e6ee30cb19d3f72300446e9d56ea 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1b0ac3febadf712c2336a82cef7e39886a421463fc72993924634d88732caf34 -size 1035456 +oid sha256:bc8c92b92b7c54b0e1dcfc4088484c23dd7921f61c208379a000cd9d1d95e528 +size 1020703 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip index c4e4d9c62ab0fc72e2cf5ba8cb4dbd46e44b13b5..eefbe20def7f0bf5192f0ea448dc149832f4ad9d 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b67f94993552d63f41a9aec69e1862fcd32ccbc15bbcbf6aa86a9d1df3582a13 +oid sha256:7ea16d6a5bd9fd2347df00f32bfcc883a94ad747eb036a7f9aed8041fdf04cec size 118622 diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/website_leaderboard.csv index c424e67f04943ef0c170f2786ba8b9c0ddad8d85..28185c72fd447069c3a0aa2fa1086235606fd735 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1644.0,+93/-60,0.637,7.8,5.36,6.865,5.25,0.517,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1567.0,+153/-98,0.586,10.65,4.0,5.801,2092.56,8.655,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1556.0,+140/-99,0.575,11.12,4.48,6.297,2092.56,0.867,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1493.0,+130/-78,0.506,14.1,5.36,8.203,598.6,8.421,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1492.0,+91/-82,0.441,14.2,8.25,10.934,2773.9,21.391,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1476.0,+124/-90,0.456,15.02,7.12,10.58,3361.32,65.579,0.0,False,GPU -6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1471.0,+85/-74,0.413,15.3,10.76,10.37,1881.65,1.26,0.0,False,CPU -7,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1443.0,+115/-72,0.407,16.82,6.69,10.913,145.75,4.071,0.0,False,GPU -8,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1426.0,+123/-84,0.419,17.77,5.81,10.972,4.09,0.544,0.0,False,GPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1420.0,+120/-99,0.371,18.18,7.33,12.618,2773.9,0.534,0.0,False,GPU -10,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1402.0,+115/-88,0.33,19.25,11.6,14.276,292.15,1.882,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1401.0,+118/-76,0.371,19.27,6.01,10.185,8.05,1.864,0.0,False,GPU -12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1399.0,+132/-82,0.38,19.4,6.06,9.42,6408.95,397.026,0.0,False,GPU -13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1389.0,+117/-88,0.351,20.05,8.88,12.618,2217.3,1.258,0.0,False,GPU -14,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1384.0,+124/-88,0.331,20.32,10.16,13.232,3361.32,0.527,0.0,False,GPU -15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1368.0,+131/-93,0.347,21.3,7.46,10.945,6408.95,39.606,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1334.0,+58/-48,0.186,23.52,20.03,15.024,457.0,1.064,0.0,False,CPU -17,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1333.0,+110/-76,0.293,23.58,9.54,13.112,4442.23,0.405,0.0,False,GPU -18,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1320.0,+84/-79,0.206,24.4,13.69,15.346,1618.79,0.502,0.0,False,CPU -19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1315.0,+112/-89,0.269,24.72,14.55,13.982,2217.3,0.137,0.0,False,GPU -20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1315.0,+77/-60,0.186,24.72,16.46,15.224,4.49,0.115,0.0,False,CPU -21,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1314.0,+80/-69,0.194,24.8,14.28,14.446,10.77,2.16,0.0,False,GPU -22,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1312.0,+104/-98,0.295,24.95,7.59,15.25,3.89,0.544,0.0,False,GPU -23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1287.0,+83/-75,0.178,26.58,19.37,16.445,673.88,0.681,0.0,False,CPU -24,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1284.0,+85/-81,0.162,26.78,17.38,15.452,1103.74,0.549,0.0,False,CPU -25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1281.0,+88/-92,0.186,27.0,18.24,15.662,516.54,1.513,0.0,False,GPU -26,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1278.0,+83/-91,0.17,27.2,10.48,16.268,6.77,0.046,0.0,False,CPU -27,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1276.0,+99/-108,0.245,27.28,16.43,15.707,4442.23,8.478,0.0,False,GPU -28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1275.0,+83/-79,0.154,27.4,11.18,15.789,1618.79,0.055,0.0,False,CPU -29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1272.0,+120/-112,0.23,27.55,17.0,16.267,10.18,0.149,0.0,False,GPU -30,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1268.0,+136/-120,0.251,27.85,8.65,13.464,51.78,39.866,0.0,False,GPU -31,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1264.0,+74/-64,0.146,28.1,24.41,16.249,457.0,0.111,0.0,False,CPU -32,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1262.0,+76/-68,0.15,28.2,11.24,16.819,673.88,0.177,0.0,False,CPU -33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1255.0,+98/-87,0.183,28.7,20.47,15.396,2533.55,2.301,0.0,False,CPU -34,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1252.0,+87/-54,0.133,28.9,24.07,16.217,1103.74,0.05,0.0,False,CPU -35,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1227.0,+80/-86,0.12,30.58,21.9,16.637,516.54,0.086,0.0,False,GPU -36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1212.0,+84/-118,0.139,31.52,10.49,18.467,11.78,0.312,0.0,False,GPU -37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1204.0,+96/-119,0.154,32.02,20.34,18.721,622.79,5.389,0.0,False,CPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1196.0,+97/-94,0.153,32.6,24.44,17.128,2533.55,0.11,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1157.0,+98/-138,0.13,35.05,20.0,19.935,622.79,0.398,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1141.0,+94/-87,0.076,36.08,28.32,20.989,204.47,0.853,0.0,False,CPU -41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1121.0,+117/-112,0.109,37.3,18.33,21.645,204.47,0.091,0.0,False,CPU -42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1117.0,+81/-93,0.043,37.55,33.65,20.016,1.88,0.121,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1094.0,+98/-125,0.057,38.9,30.42,22.532,448.77,0.841,0.0,False,CPU -44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1091.0,+80/-100,0.043,39.08,35.18,20.402,2.39,0.159,0.0,False,CPU -45,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1078.0,+103/-162,0.089,39.85,28.74,23.197,1.17,0.148,0.0,False,GPU -46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1065.0,+93/-110,0.03,40.55,34.95,23.044,448.77,0.092,0.0,False,CPU -47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1033.0,+109/-126,0.046,42.3,34.74,23.554,7.36,0.138,0.0,False,CPU -48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+98/-164,0.035,44.0,29.07,26.004,0.37,0.04,0.0,False,CPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,997.0,+107/-123,0.031,44.15,39.68,22.94,3.3,0.395,0.0,False,CPU -50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,963.0,+65/-111,0.0,45.75,44.14,26.281,89.46,1.579,0.0,False,CPU -51,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,953.0,+121/-156,0.044,46.2,41.0,26.17,2.66,0.553,0.0,False,GPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,949.0,+114/-129,0.05,46.38,14.32,29.0,276.0,0.368,0.0,False,CPU -53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,948.0,+121/-161,0.024,46.45,40.96,27.548,0.29,0.041,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,926.0,+106/-145,0.019,47.35,41.98,31.115,1.45,0.123,0.0,False,CPU -55,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,920.0,+122/-145,0.042,47.6,22.6,29.565,276.0,0.065,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,810.0,+84/-146,0.0,51.45,50.54,31.853,89.46,0.075,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,624.0,+110/-269,0.0,55.5,55.08,43.289,0.22,0.02,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1644.0,+93/-60,6.865,0.637,7.8,5.36,5.25,0.517,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1567.0,+153/-98,5.801,0.586,10.65,4.0,2092.56,8.655,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1556.0,+140/-99,6.297,0.575,11.12,4.48,2092.56,0.867,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1493.0,+130/-78,8.203,0.506,14.1,5.36,598.6,8.421,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1492.0,+91/-82,10.934,0.441,14.2,8.25,2773.9,21.391,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1476.0,+124/-90,10.58,0.456,15.02,7.12,3361.32,65.579,✔️,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1471.0,+85/-74,10.37,0.413,15.3,10.76,1881.65,1.26,✔️,0.0,False,CPU +7,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1443.0,+115/-72,10.913,0.407,16.82,6.69,145.75,4.071,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1426.0,+123/-84,10.972,0.419,17.77,5.81,4.09,0.544,➖,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1420.0,+120/-99,12.618,0.371,18.18,7.33,2773.9,0.534,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1402.0,+115/-88,14.276,0.33,19.25,11.6,292.15,1.882,➖,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1401.0,+118/-76,10.185,0.371,19.27,6.01,8.05,1.864,✔️,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1399.0,+132/-82,9.42,0.38,19.4,6.06,6408.95,397.026,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1389.0,+117/-88,12.618,0.351,20.05,8.88,2217.3,1.258,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1384.0,+124/-88,13.232,0.331,20.32,10.16,3361.32,0.527,✔️,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1368.0,+131/-93,10.945,0.347,21.3,7.46,6408.95,39.606,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1334.0,+58/-48,15.024,0.186,23.52,20.03,457.0,1.064,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1333.0,+110/-76,13.112,0.293,23.58,9.54,4442.23,0.405,✔️,0.0,False,GPU +18,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1320.0,+84/-79,15.346,0.206,24.4,13.69,1618.79,0.502,✔️,0.0,False,CPU +19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1315.0,+112/-89,13.982,0.269,24.72,14.55,2217.3,0.137,✔️,0.0,False,GPU +20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1315.0,+77/-60,15.224,0.186,24.72,16.46,4.49,0.115,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1314.0,+80/-69,14.446,0.194,24.8,14.28,10.77,2.16,✔️,0.0,False,GPU +22,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1312.0,+104/-98,15.25,0.295,24.95,7.59,3.89,0.544,✔️,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1287.0,+83/-75,16.445,0.178,26.58,19.37,673.88,0.681,✔️,0.0,False,CPU +24,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1284.0,+85/-81,15.452,0.162,26.78,17.38,1103.74,0.549,✔️,0.0,False,CPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1281.0,+88/-92,15.662,0.186,27.0,18.24,516.54,1.513,✔️,0.0,False,GPU +26,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1278.0,+83/-91,16.268,0.17,27.2,10.48,6.77,0.046,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1276.0,+99/-108,15.707,0.245,27.28,16.43,4442.23,8.478,✔️,0.0,False,GPU +28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1275.0,+83/-79,15.789,0.154,27.4,11.18,1618.79,0.055,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1272.0,+120/-112,16.267,0.23,27.55,17.0,10.18,0.149,✔️,0.0,False,GPU +30,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1268.0,+136/-120,13.464,0.251,27.85,8.65,51.78,39.866,✔️,0.0,False,GPU +31,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1264.0,+74/-64,16.249,0.146,28.1,24.41,457.0,0.111,✔️,0.0,False,CPU +32,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1262.0,+76/-68,16.819,0.15,28.2,11.24,673.88,0.177,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1255.0,+98/-87,15.396,0.183,28.7,20.47,2533.55,2.301,✔️,0.0,False,CPU +34,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1252.0,+87/-54,16.217,0.133,28.9,24.07,1103.74,0.05,✔️,0.0,False,CPU +35,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1227.0,+80/-86,16.637,0.12,30.58,21.9,516.54,0.086,✔️,0.0,False,GPU +36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1212.0,+84/-118,18.467,0.139,31.52,10.49,11.78,0.312,✔️,0.0,False,GPU +37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1204.0,+96/-119,18.721,0.154,32.02,20.34,622.79,5.389,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1196.0,+97/-94,17.128,0.153,32.6,24.44,2533.55,0.11,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1157.0,+98/-138,19.935,0.13,35.05,20.0,622.79,0.398,✔️,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1141.0,+94/-87,20.989,0.076,36.08,28.32,204.47,0.853,✔️,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1121.0,+117/-112,21.645,0.109,37.3,18.33,204.47,0.091,✔️,0.0,False,CPU +42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1117.0,+81/-93,20.016,0.043,37.55,33.65,1.88,0.121,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1094.0,+98/-125,22.532,0.057,38.9,30.42,448.77,0.841,✔️,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1091.0,+80/-100,20.402,0.043,39.08,35.18,2.39,0.159,✔️,0.0,False,CPU +45,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1078.0,+103/-162,23.197,0.089,39.85,28.74,1.17,0.148,➖,0.0,False,GPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1065.0,+93/-110,23.044,0.03,40.55,34.95,448.77,0.092,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1033.0,+109/-126,23.554,0.046,42.3,34.74,7.36,0.138,✔️,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+98/-164,26.004,0.035,44.0,29.07,0.37,0.04,✔️,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),997.0,+107/-123,22.94,0.031,44.15,39.68,3.3,0.395,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),963.0,+65/-111,26.281,0.0,45.75,44.14,89.46,1.579,✔️,0.0,False,CPU +51,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),953.0,+121/-156,26.17,0.044,46.2,41.0,2.66,0.553,✔️,0.0,False,GPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),949.0,+114/-129,29.0,0.05,46.38,14.32,276.0,0.368,✔️,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),948.0,+121/-161,27.548,0.024,46.45,40.96,0.29,0.041,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),926.0,+106/-145,31.115,0.019,47.35,41.98,1.45,0.123,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),920.0,+122/-145,29.565,0.042,47.6,22.6,276.0,0.065,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),810.0,+84/-146,31.853,0.0,51.45,50.54,89.46,0.075,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),624.0,+110/-269,43.289,0.0,55.5,55.08,0.22,0.02,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip index 4b3ce9b80b951a28613ed682675de2ecafa54f3e..426b162708d9fac178f86a9881d788f58ab54fad 100644 --- a/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a0483268e20186a3da29d0fbfaf0925938bb078bde1ebc90a02ef29676184250 +oid sha256:e9e32ffc19f86c878369ffc80c8b16fc14574683c759c454886d49d73a53318c size 2250455 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index ef14fc114cbdfe7fc5d6f5c62c0360c06d06136f..b2acb4bf89d8533d7bba60a5298730af00132696 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9db303e8bb6b504a8e841c2216edef7474ab321dcc65bed34b91964a7f8a2059 +oid sha256:eacdb7b0487ae55a23dbdb8ec0de89db0fd49b74a82a0c846a78687a308cdaed size 440786 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip index db366eb1918944048ac9aba539fe786ee9e287e0..4a68495b9502c40b64a7fadf46b3a5aaf35edf65 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b1d146f3660c5a23e6b3c365e0dca9cf9c7f0578c986ab495545d3ed6df002fe -size 990599 +oid sha256:44e6896b71e279411fc6b24a157220f59a17677d5aab9b6b82dcf84ea596c516 +size 938069 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_all/tuning-impact-elo.png.zip index 0f0e86391015353b7213ce5d17195673f3eee845..fd4d6a93cf58d8298d102a664aced2420c5371ea 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:974ffb2ad6c1d0bc3fe2b63babe5d57ca926f337ff76059b538f1ac4754a50f0 +oid sha256:204f3c81d224584d0028ca0480170449e8ebc01e9e6405e6f5e92012cfdb621e size 124439 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_classification/datasets_all/website_leaderboard.csv index a94f5357046d98b5326ddf967e870e06a3119a1a..7e6ddb985f6fba4b4506a05412f5b09befe6b613 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_all/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_all/website_leaderboard.csv @@ -1,53 +1,53 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1607.0,+92/-78,0.669,8.54,3.62,6.887,521.51,7.239,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1600.0,+107/-91,0.647,8.76,2.69,4.536,2063.62,9.938,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1569.0,+100/-83,0.595,9.91,4.18,7.055,2063.62,1.087,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1555.0,+99/-65,0.579,10.45,4.35,8.539,1815.7,1.456,0.0,False,CPU -4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1550.0,+93/-73,0.571,10.62,4.56,7.411,5.62,0.771,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1508.0,+67/-58,0.49,12.42,6.88,10.213,2773.9,16.363,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1437.0,+57/-53,0.412,15.8,7.08,11.863,2773.9,0.39,0.0,False,GPU -7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1429.0,+76/-65,0.399,16.24,7.49,12.341,2461.81,1.283,0.0,False,GPU -8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1413.0,+61/-47,0.315,17.04,12.62,13.098,391.23,1.243,0.0,False,CPU -9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1386.0,+73/-54,0.335,18.47,11.49,12.614,1326.58,0.531,0.0,False,CPU -10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1386.0,+99/-64,0.355,18.5,7.28,11.346,6067.13,404.225,0.0,False,GPU -11,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1380.0,+74/-69,0.299,18.8,11.7,13.691,682.62,0.971,0.0,False,CPU -12,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1367.0,+70/-78,0.313,19.53,9.78,13.123,4853.07,0.473,0.0,False,GPU -13,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1367.0,+89/-64,0.33,19.57,10.85,13.311,2461.81,0.138,0.0,False,GPU -14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1366.0,+70/-60,0.3,19.63,13.51,13.149,1326.58,0.051,0.0,False,CPU -15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1365.0,+51/-53,0.279,19.66,10.49,13.258,5.74,0.107,0.0,False,CPU -16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1362.0,+100/-92,0.346,19.84,8.61,13.165,4853.07,9.386,0.0,False,GPU -17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1351.0,+66/-53,0.256,20.42,15.26,14.046,391.23,0.186,0.0,False,CPU -18,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1348.0,+71/-68,0.265,20.61,10.93,13.945,682.62,0.183,0.0,False,CPU -19,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1334.0,+77/-75,0.305,21.41,7.09,13.381,6067.13,39.565,0.0,False,GPU -20,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1329.0,+71/-62,0.257,21.66,12.57,15.718,250.94,2.036,0.0,False,GPU -21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1324.0,+68/-62,0.259,21.95,11.98,14.425,1053.72,2.067,0.0,False,GPU -22,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1322.0,+66/-54,0.217,22.05,13.58,15.486,2399.23,0.558,0.0,False,CPU -23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1299.0,+80/-64,0.243,23.39,14.93,15.974,10.31,0.136,0.0,False,GPU -24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1277.0,+48/-53,0.174,24.68,18.36,15.239,2480.32,2.175,0.0,False,CPU -25,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1274.0,+67/-49,0.168,24.82,16.77,15.84,10.77,1.787,0.0,False,GPU -26,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1272.0,+74/-59,0.191,24.93,12.06,16.074,1053.72,0.105,0.0,False,GPU -27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1260.0,+61/-53,0.157,25.68,14.05,16.415,2399.23,0.056,0.0,False,CPU -28,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1256.0,+63/-53,0.159,25.89,13.57,17.309,6.77,0.052,0.0,False,CPU -29,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1220.0,+82/-74,0.195,27.93,10.63,16.595,47.89,39.446,0.0,False,GPU -30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1219.0,+71/-90,0.161,28.04,18.88,18.456,622.79,5.037,0.0,False,CPU -31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1205.0,+67/-61,0.137,28.82,22.17,17.532,2480.32,0.117,0.0,False,CPU -32,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1203.0,+69/-70,0.129,28.92,15.16,17.049,1.88,0.121,0.0,False,CPU -33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1196.0,+66/-58,0.119,29.29,20.81,18.674,197.16,0.721,0.0,False,CPU -34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1172.0,+63/-66,0.099,30.66,14.84,19.122,13.35,0.353,0.0,False,GPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1172.0,+67/-77,0.106,30.68,21.08,19.605,327.31,0.708,0.0,False,CPU -36,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1159.0,+66/-81,0.118,31.38,17.52,19.975,197.16,0.083,0.0,False,CPU -37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1150.0,+60/-62,0.087,31.88,27.07,18.358,2.25,0.159,0.0,False,CPU -38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1131.0,+73/-82,0.108,32.89,19.44,20.473,622.79,0.346,0.0,False,CPU -39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1128.0,+59/-63,0.055,33.04,25.79,20.441,327.31,0.085,0.0,False,CPU -40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1028.0,+64/-78,0.035,38.03,33.9,23.172,5.73,0.148,0.0,False,CPU -41,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+68/-81,0.02,39.28,29.08,26.523,0.37,0.038,0.0,False,CPU -42,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,990.0,+78/-108,0.033,39.71,34.47,27.289,140.66,2.876,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,986.0,+71/-68,0.04,39.87,35.96,24.378,3.12,0.334,0.0,False,CPU -44,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,973.0,+91/-93,0.053,40.42,34.66,27.31,3.48,0.677,0.0,False,GPU -45,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,972.0,+86/-141,0.063,40.45,31.52,29.506,1.04,0.192,0.0,False,GPU -46,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,965.0,+89/-97,0.03,40.75,19.72,30.755,256.15,0.36,0.0,False,CPU -47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,932.0,+92/-106,0.023,42.03,27.39,31.478,256.15,0.07,0.0,False,CPU -48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,926.0,+80/-103,0.016,42.24,38.26,29.05,0.24,0.037,0.0,False,CPU -49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,871.0,+93/-114,0.013,44.11,40.88,34.085,1.32,0.117,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,809.0,+92/-127,0.013,45.89,43.66,34.791,140.66,0.188,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,553.0,+116/-146,0.0,50.42,50.13,49.426,0.18,0.028,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1607.0,+92/-78,6.887,0.669,8.54,3.62,521.51,7.239,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1600.0,+107/-91,4.536,0.647,8.76,2.69,2063.62,9.938,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1569.0,+100/-83,7.055,0.595,9.91,4.18,2063.62,1.087,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1555.0,+99/-65,8.539,0.579,10.45,4.35,1815.7,1.456,✔️,0.0,False,CPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1550.0,+93/-73,7.411,0.571,10.62,4.56,5.62,0.771,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1508.0,+67/-58,10.213,0.49,12.42,6.88,2773.9,16.363,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1437.0,+57/-53,11.863,0.412,15.8,7.08,2773.9,0.39,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1429.0,+76/-65,12.341,0.399,16.24,7.49,2461.81,1.283,✔️,0.0,False,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1413.0,+61/-47,13.098,0.315,17.04,12.62,391.23,1.243,✔️,0.0,False,CPU +9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1386.0,+73/-54,12.614,0.335,18.47,11.49,1326.58,0.531,✔️,0.0,False,CPU +10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1386.0,+99/-64,11.346,0.355,18.5,7.28,6067.13,404.225,✔️,0.0,False,GPU +11,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1380.0,+74/-69,13.691,0.299,18.8,11.7,682.62,0.971,✔️,0.0,False,CPU +12,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1367.0,+70/-78,13.123,0.313,19.53,9.78,4853.07,0.473,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1367.0,+89/-64,13.311,0.33,19.57,10.85,2461.81,0.138,✔️,0.0,False,GPU +14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1366.0,+70/-60,13.149,0.3,19.63,13.51,1326.58,0.051,✔️,0.0,False,CPU +15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1365.0,+51/-53,13.258,0.279,19.66,10.49,5.74,0.107,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1362.0,+100/-92,13.165,0.346,19.84,8.61,4853.07,9.386,✔️,0.0,False,GPU +17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1351.0,+66/-53,14.046,0.256,20.42,15.26,391.23,0.186,✔️,0.0,False,CPU +18,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1348.0,+71/-68,13.945,0.265,20.61,10.93,682.62,0.183,✔️,0.0,False,CPU +19,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1334.0,+77/-75,13.381,0.305,21.41,7.09,6067.13,39.565,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1329.0,+71/-62,15.718,0.257,21.66,12.57,250.94,2.036,➖,0.0,False,GPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1324.0,+68/-62,14.425,0.259,21.95,11.98,1053.72,2.067,✔️,0.0,False,GPU +22,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1322.0,+66/-54,15.486,0.217,22.05,13.58,2399.23,0.558,✔️,0.0,False,CPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1299.0,+80/-64,15.974,0.243,23.39,14.93,10.31,0.136,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1277.0,+48/-53,15.239,0.174,24.68,18.36,2480.32,2.175,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1274.0,+67/-49,15.84,0.168,24.82,16.77,10.77,1.787,✔️,0.0,False,GPU +26,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1272.0,+74/-59,16.074,0.191,24.93,12.06,1053.72,0.105,✔️,0.0,False,GPU +27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1260.0,+61/-53,16.415,0.157,25.68,14.05,2399.23,0.056,✔️,0.0,False,CPU +28,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1256.0,+63/-53,17.309,0.159,25.89,13.57,6.77,0.052,✔️,0.0,False,CPU +29,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1220.0,+82/-74,16.595,0.195,27.93,10.63,47.89,39.446,✔️,0.0,False,GPU +30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1219.0,+71/-90,18.456,0.161,28.04,18.88,622.79,5.037,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1205.0,+67/-61,17.532,0.137,28.82,22.17,2480.32,0.117,✔️,0.0,False,CPU +32,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1203.0,+69/-70,17.049,0.129,28.92,15.16,1.88,0.121,✔️,0.0,False,CPU +33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1196.0,+66/-58,18.674,0.119,29.29,20.81,197.16,0.721,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1172.0,+63/-66,19.122,0.099,30.66,14.84,13.35,0.353,✔️,0.0,False,GPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1172.0,+67/-77,19.605,0.106,30.68,21.08,327.31,0.708,✔️,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1159.0,+66/-81,19.975,0.118,31.38,17.52,197.16,0.083,✔️,0.0,False,CPU +37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1150.0,+60/-62,18.358,0.087,31.88,27.07,2.25,0.159,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1131.0,+73/-82,20.473,0.108,32.89,19.44,622.79,0.346,✔️,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1128.0,+59/-63,20.441,0.055,33.04,25.79,327.31,0.085,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1028.0,+64/-78,23.172,0.035,38.03,33.9,5.73,0.148,✔️,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+68/-81,26.523,0.02,39.28,29.08,0.37,0.038,✔️,0.0,False,CPU +42,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),990.0,+78/-108,27.289,0.033,39.71,34.47,140.66,2.876,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),986.0,+71/-68,24.378,0.04,39.87,35.96,3.12,0.334,✔️,0.0,False,CPU +44,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),973.0,+91/-93,27.31,0.053,40.42,34.66,3.48,0.677,✔️,0.0,False,GPU +45,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),972.0,+86/-141,29.506,0.063,40.45,31.52,1.04,0.192,➖,0.0,False,GPU +46,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),965.0,+89/-97,30.755,0.03,40.75,19.72,256.15,0.36,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),932.0,+92/-106,31.478,0.023,42.03,27.39,256.15,0.07,✔️,0.0,False,CPU +48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),926.0,+80/-103,29.05,0.016,42.24,38.26,0.24,0.037,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),871.0,+93/-114,34.085,0.013,44.11,40.88,1.32,0.117,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),809.0,+92/-127,34.791,0.013,45.89,43.66,140.66,0.188,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),553.0,+116/-146,49.426,0.0,50.42,50.13,0.18,0.028,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_all/winrate_matrix.png.zip index cb5a4c72594701283ba7295dd693886aebe55ef2..c4bded7ff986cabde8fd0fdee4fa2782d6fe9a78 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b89ecc0cbe9882f613adfae8c80280d7eb19c6519edf68ab7dbc81a01484b94f +oid sha256:e136b0209df5c0b9145ae0846786c898a888cf0d826f373f0507909688a51b16 size 1915089 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index 84a96d82d08c70daa0ff33b174446cb4948a6a90..4743aa9a7d6ce67f03dae026ee55e6274fbaf709 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6090ddcc56ae88bb63938583bb61e4456475637b908970bdff58a15ccab8deba +oid sha256:b61bf81a7d6a5629eec57fc90ba2f1f761e92d909be067339daae69b39bfd830 size 425862 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip index 65a36815256ad1bc7d77aa05b8329b8d8bcd65c7..fe4ebfc08af57daad7b07fa61afcb2a19588b101 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:71588239453cd962021aa88c7e8e91c20ec6fc2ea78d7603bb32f047dfd7a3ed -size 1011666 +oid sha256:8432010802d12587598008f141a0d5d47a8e8cf02da82317b3b7ae2af405c0b8 +size 971412 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/tuning-impact-elo.png.zip index 619f242c4b9305c5e043c946cf6565f3403587f0..05c60a838736f9301732445641c562450c87a3ef 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d302790329d0a0a6af88b80985623aa258d3c1906bf67fcb509a44e0d6794385 +oid sha256:c88396922b5f88c69931f0667c80fdc584027d9e39cc5b661e0e9a15d9517632 size 118671 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/website_leaderboard.csv index 1fcb88a2b9a74b4cd5c09c788e1c15ff2f43d5d5..c065ec55b4444ae31c21a8aa6d919d58edc19f7c 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/website_leaderboard.csv @@ -1,54 +1,54 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,2043.0,+165/-76,0.873,3.2,2.12,1.556,307.34,2.186,0.0,False,CPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1977.0,+218/-61,0.845,4.1,2.28,2.107,266.49,2.963,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1724.0,+267/-141,0.574,10.3,3.13,5.287,780.95,11.64,0.0,False,GPU -3,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1699.0,+243/-163,0.601,11.2,4.14,5.346,2335.82,14.145,0.0,False,GPU -4,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1675.0,+165/-110,0.463,12.1,8.59,7.754,202.53,0.43,0.0,False,CPU -5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1664.0,+245/-124,0.494,12.5,6.97,7.388,780.95,1.493,0.0,False,GPU -6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1659.0,+175/-97,0.517,12.7,7.86,7.039,688.25,0.192,0.0,False,CPU -7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1647.0,+189/-83,0.444,13.2,9.54,8.663,91.02,0.568,0.0,False,CPU -8,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1638.0,+171/-87,0.491,13.55,8.8,7.113,688.25,0.036,0.0,False,CPU -9,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1632.0,+128/-91,0.463,13.8,9.46,7.465,1635.73,2.146,0.0,False,GPU -10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1604.0,+180/-105,0.429,15.0,5.57,8.84,1400.65,0.571,0.0,False,GPU -11,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1603.0,+136/-69,0.455,15.05,5.96,7.228,2.2,0.044,0.0,False,CPU -12,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1599.0,+175/-116,0.418,15.2,10.54,9.77,2335.82,0.502,0.0,False,GPU -13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1588.0,+183/-107,0.409,15.7,11.05,8.072,202.53,0.055,0.0,False,CPU -14,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1570.0,+182/-110,0.352,16.5,12.53,9.516,91.02,0.074,0.0,False,CPU -15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1542.0,+159/-111,0.351,17.8,10.04,9.875,1635.73,0.082,0.0,False,GPU -16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1522.0,+209/-168,0.353,18.7,8.98,9.432,1400.65,0.065,0.0,False,GPU -17,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1514.0,+194/-127,0.329,19.1,9.49,8.469,2.3,0.818,0.0,False,GPU -18,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1499.0,+177/-140,0.294,19.8,10.12,10.117,8.46,3.879,0.0,False,GPU -19,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1450.0,+204/-130,0.249,22.2,15.55,12.904,899.24,0.221,0.0,False,CPU -20,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1446.0,+233/-158,0.29,22.4,6.94,10.304,0.47,0.047,0.0,False,CPU -21,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1420.0,+113/-70,0.158,23.7,19.71,11.014,1128.73,1.573,0.0,False,CPU -22,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1412.0,+189/-172,0.223,24.1,19.43,12.709,5.79,0.084,0.0,False,GPU -23,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1408.0,+143/-135,0.169,24.3,18.96,11.095,4546.1,406.11,0.0,False,GPU -24,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1402.0,+158/-144,0.222,24.6,16.19,11.567,713.0,2.317,0.0,False,GPU -25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1375.0,+166/-140,0.149,26.0,18.4,14.592,713.0,0.155,0.0,False,GPU -26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1349.0,+96/-101,0.071,27.3,24.64,14.558,6.53,0.294,0.0,False,GPU -27,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1326.0,+165/-163,0.123,28.45,18.52,14.827,4546.1,39.323,0.0,False,GPU -28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1321.0,+172/-131,0.136,28.7,22.41,14.536,899.24,0.015,0.0,False,CPU -29,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1315.0,+157/-123,0.104,29.0,25.99,15.96,1.95,0.02,0.0,False,CPU -30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1315.0,+106/-156,0.099,29.0,23.43,15.425,387.73,2.446,0.0,False,CPU -31,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1311.0,+135/-106,0.073,29.2,21.79,14.282,52.7,0.202,0.0,False,CPU -32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1287.0,+118/-103,0.073,30.4,27.23,14.195,1128.73,0.054,0.0,False,CPU -33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1283.0,+145/-152,0.057,30.6,27.42,13.773,38.59,0.337,0.0,False,CPU -34,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1283.0,+183/-175,0.117,30.6,20.24,15.553,82.52,0.97,0.0,False,GPU -35,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1268.0,+124/-101,0.056,31.3,26.28,14.704,52.7,0.017,0.0,False,CPU -36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1225.0,+105/-129,0.038,33.3,31.31,13.946,0.33,0.05,0.0,False,CPU -37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1212.0,+92/-115,0.009,33.9,32.53,14.785,11.03,0.489,0.0,False,GPU -38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1187.0,+152/-168,0.031,35.0,31.95,17.259,38.59,0.028,0.0,False,CPU -39,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1186.0,+136/-139,0.022,35.05,31.57,18.812,41.63,39.259,0.0,False,GPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1131.0,+103/-160,0.016,37.4,35.51,18.995,387.73,0.089,0.0,False,CPU -41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1100.0,+94/-110,0.009,38.6,37.21,17.804,1.84,0.047,0.0,False,CPU -42,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1048.0,+178/-260,0.014,40.55,36.73,28.29,43.41,0.111,0.0,False,CPU -43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1036.0,+114/-212,0.018,41.0,38.29,24.539,332.4,33.481,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1018.0,+144/-200,0.022,41.6,38.72,22.564,1.63,0.11,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+117/-257,0.0,42.2,40.81,22.603,0.11,0.012,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,998.0,+166/-272,0.002,42.25,39.61,29.261,43.41,0.02,0.0,False,CPU -47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,838.0,+248/-564,0.024,46.7,39.85,33.498,2.96,0.183,0.0,False,GPU -48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,834.0,+103/-217,0.0,46.8,46.47,30.86,0.21,0.019,0.0,False,CPU -49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,829.0,+93/-235,0.0,46.9,46.61,27.581,0.1,0.02,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,782.0,+66/-184,0.0,47.9,47.61,33.014,332.4,1.631,0.0,False,CPU -51,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,761.0,+125/-257,0.0,48.3,47.82,35.637,0.18,0.032,0.0,False,GPU -52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,426.0,+114/-399,0.0,52.2,52.17,49.883,0.12,0.094,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",2043.0,+165/-76,1.556,0.873,3.2,2.12,307.34,2.186,✔️,0.0,False,CPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1977.0,+218/-61,2.107,0.845,4.1,2.28,266.49,2.963,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1724.0,+267/-141,5.287,0.574,10.3,3.13,780.95,11.64,✔️,0.0,False,GPU +3,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1699.0,+243/-163,5.346,0.601,11.2,4.14,2335.82,14.145,✔️,0.0,False,GPU +4,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1675.0,+165/-110,7.754,0.463,12.1,8.59,202.53,0.43,✔️,0.0,False,CPU +5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1664.0,+245/-124,7.388,0.494,12.5,6.97,780.95,1.493,✔️,0.0,False,GPU +6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1659.0,+175/-97,7.039,0.517,12.7,7.86,688.25,0.192,✔️,0.0,False,CPU +7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1647.0,+189/-83,8.663,0.444,13.2,9.54,91.02,0.568,✔️,0.0,False,CPU +8,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1638.0,+171/-87,7.113,0.491,13.55,8.8,688.25,0.036,✔️,0.0,False,CPU +9,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1632.0,+128/-91,7.465,0.463,13.8,9.46,1635.73,2.146,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1604.0,+180/-105,8.84,0.429,15.0,5.57,1400.65,0.571,✔️,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1603.0,+136/-69,7.228,0.455,15.05,5.96,2.2,0.044,✔️,0.0,False,CPU +12,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1599.0,+175/-116,9.77,0.418,15.2,10.54,2335.82,0.502,✔️,0.0,False,GPU +13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1588.0,+183/-107,8.072,0.409,15.7,11.05,202.53,0.055,✔️,0.0,False,CPU +14,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1570.0,+182/-110,9.516,0.352,16.5,12.53,91.02,0.074,✔️,0.0,False,CPU +15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1542.0,+159/-111,9.875,0.351,17.8,10.04,1635.73,0.082,✔️,0.0,False,GPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1522.0,+209/-168,9.432,0.353,18.7,8.98,1400.65,0.065,✔️,0.0,False,GPU +17,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1514.0,+194/-127,8.469,0.329,19.1,9.49,2.3,0.818,✔️,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1499.0,+177/-140,10.117,0.294,19.8,10.12,8.46,3.879,✔️,0.0,False,GPU +19,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1450.0,+204/-130,12.904,0.249,22.2,15.55,899.24,0.221,✔️,0.0,False,CPU +20,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1446.0,+233/-158,10.304,0.29,22.4,6.94,0.47,0.047,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1420.0,+113/-70,11.014,0.158,23.7,19.71,1128.73,1.573,✔️,0.0,False,CPU +22,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1412.0,+189/-172,12.709,0.223,24.1,19.43,5.79,0.084,✔️,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1408.0,+143/-135,11.095,0.169,24.3,18.96,4546.1,406.11,✔️,0.0,False,GPU +24,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1402.0,+158/-144,11.567,0.222,24.6,16.19,713.0,2.317,✔️,0.0,False,GPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1375.0,+166/-140,14.592,0.149,26.0,18.4,713.0,0.155,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1349.0,+96/-101,14.558,0.071,27.3,24.64,6.53,0.294,✔️,0.0,False,GPU +27,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1326.0,+165/-163,14.827,0.123,28.45,18.52,4546.1,39.323,✔️,0.0,False,GPU +28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1321.0,+172/-131,14.536,0.136,28.7,22.41,899.24,0.015,✔️,0.0,False,CPU +29,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1315.0,+157/-123,15.96,0.104,29.0,25.99,1.95,0.02,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1315.0,+106/-156,15.425,0.099,29.0,23.43,387.73,2.446,✔️,0.0,False,CPU +31,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1311.0,+135/-106,14.282,0.073,29.2,21.79,52.7,0.202,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1287.0,+118/-103,14.195,0.073,30.4,27.23,1128.73,0.054,✔️,0.0,False,CPU +33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1283.0,+145/-152,13.773,0.057,30.6,27.42,38.59,0.337,✔️,0.0,False,CPU +34,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1283.0,+183/-175,15.553,0.117,30.6,20.24,82.52,0.97,➖,0.0,False,GPU +35,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1268.0,+124/-101,14.704,0.056,31.3,26.28,52.7,0.017,✔️,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1225.0,+105/-129,13.946,0.038,33.3,31.31,0.33,0.05,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1212.0,+92/-115,14.785,0.009,33.9,32.53,11.03,0.489,✔️,0.0,False,GPU +38,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1187.0,+152/-168,17.259,0.031,35.0,31.95,38.59,0.028,✔️,0.0,False,CPU +39,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1186.0,+136/-139,18.812,0.022,35.05,31.57,41.63,39.259,✔️,0.0,False,GPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1131.0,+103/-160,18.995,0.016,37.4,35.51,387.73,0.089,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1100.0,+94/-110,17.804,0.009,38.6,37.21,1.84,0.047,✔️,0.0,False,CPU +42,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1048.0,+178/-260,28.29,0.014,40.55,36.73,43.41,0.111,✔️,0.0,False,CPU +43,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1036.0,+114/-212,24.539,0.018,41.0,38.29,332.4,33.481,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1018.0,+144/-200,22.564,0.022,41.6,38.72,1.63,0.11,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+117/-257,22.603,0.0,42.2,40.81,0.11,0.012,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),998.0,+166/-272,29.261,0.002,42.25,39.61,43.41,0.02,✔️,0.0,False,CPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),838.0,+248/-564,33.498,0.024,46.7,39.85,2.96,0.183,✔️,0.0,False,GPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),834.0,+103/-217,30.86,0.0,46.8,46.47,0.21,0.019,✔️,0.0,False,CPU +49,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),829.0,+93/-235,27.581,0.0,46.9,46.61,0.1,0.02,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),782.0,+66/-184,33.014,0.0,47.9,47.61,332.4,1.631,✔️,0.0,False,CPU +51,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),761.0,+125/-257,35.637,0.0,48.3,47.82,0.18,0.032,➖,0.0,False,GPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),426.0,+114/-399,49.883,0.0,52.2,52.17,0.12,0.094,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/winrate_matrix.png.zip index 28b509e32fd196f36fe8d4f35754f7192712e0dd..2dd7765d53d1423f0a7bf6b750869018b56caec1 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b9f1c34da9b71ee151c8fd4d82a368efaad8540acc7987df5938ab016eb06710 +oid sha256:546c9928519cd3c31b82b8df5f1e54a5014b6f33a76a939129582cd1c3df724a size 1687839 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 1025badef7a0aca774b15f059c17c4713f4d334e..51f829c5651b5c3cc01d1c7e16426a87a6052502 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:36a6c91dd9a78d85c00e8d8b0e330ac1035ac29a581864f99acc21dcd66b64e1 +oid sha256:4a3b5ccd0756d36b97e1d5dd9f4273d0c132b28525c503d3b8e21024cfb98c83 size 437874 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip index 2573a0d877e5765f6aa675383718a60441b22456..9a2b252d794ecd8ea802a9c051d6a2eea8eeadd8 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6ace139da4a1e35dc36d52ef900bd9886308b277cf4e0ef38355cfd99fba033e -size 946898 +oid sha256:54132e5a8639d149b328088aaea5e2bb0e5284e277e5352f6c0d4d5741cb1675 +size 952618 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_small/tuning-impact-elo.png.zip index 0700a0f2a558845f0822f53b10430b993a2973db..75d4d2717b5362a87a4115f5bf12fe5d477c1aee 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:51a32b87534cb76286da6333129196bc4c0401afb15e9303db2ffd3ef09a6020 +oid sha256:d6c1ffd5f92c24a9003218763cfdb50a058332b5582ebb055d89ce02a81cde7f size 128773 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_classification/datasets_small/website_leaderboard.csv index 3526ce8863a4af890ab284d59d6547000702c35c..ffa7dda24d9167cb482dc29d6100582ec592f775 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_small/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_small/website_leaderboard.csv @@ -1,54 +1,54 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1610.0,+109/-82,0.641,7.88,3.92,7.306,6.03,0.703,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1592.0,+131/-96,0.66,8.46,2.69,4.54,2279.85,9.666,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1564.0,+130/-89,0.615,9.45,3.78,7.209,2279.85,0.968,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1535.0,+111/-88,0.589,10.57,4.69,8.868,635.11,8.569,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1487.0,+92/-69,0.473,12.61,6.55,11.474,3486.44,22.867,0.0,False,GPU -5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1468.0,+96/-69,0.458,13.5,7.44,11.301,2993.17,1.401,0.0,False,CPU -6,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1456.0,+99/-63,0.476,14.09,5.46,10.565,4.53,0.611,0.0,False,GPU -7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1421.0,+77/-75,0.404,15.84,7.35,12.856,3486.44,1.002,0.0,False,GPU -8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1394.0,+85/-74,0.412,17.29,6.14,11.685,7432.95,404.225,0.0,False,GPU -9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1391.0,+83/-79,0.367,17.43,8.78,13.865,3083.36,1.48,0.0,False,GPU -10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1358.0,+62/-49,0.257,19.27,15.23,14.947,774.51,1.455,0.0,False,CPU -11,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1356.0,+104/-78,0.298,19.36,11.43,16.037,303.47,2.324,0.0,False,GPU -12,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1349.0,+89/-90,0.365,19.79,5.93,13.112,7432.95,39.797,0.0,False,GPU -13,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1333.0,+94/-70,0.3,20.7,12.07,14.97,3083.36,0.189,0.0,False,GPU -14,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1322.0,+76/-65,0.251,21.36,14.57,14.88,1612.87,0.724,0.0,False,CPU -15,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1313.0,+86/-74,0.264,21.89,10.05,14.576,6286.73,0.473,0.0,False,GPU -16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1312.0,+82/-73,0.258,21.93,12.15,15.706,1165.19,2.067,0.0,False,GPU -17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1310.0,+68/-72,0.223,22.05,13.9,16.083,829.32,1.642,0.0,False,CPU -18,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1308.0,+58/-51,0.2,22.2,16.21,15.686,6.97,0.144,0.0,False,CPU -19,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1300.0,+67/-46,0.221,22.66,17.19,15.572,1612.87,0.061,0.0,False,CPU -20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1299.0,+68/-66,0.21,22.71,18.04,15.928,774.51,0.282,0.0,False,CPU -21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1295.0,+71/-62,0.195,22.96,13.86,16.677,2989.36,0.609,0.0,False,CPU -22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1290.0,+73/-69,0.202,23.25,11.14,16.308,829.32,0.241,0.0,False,CPU -23,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1281.0,+92/-81,0.242,23.75,14.62,16.222,6286.73,9.386,0.0,False,GPU -24,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1275.0,+82/-83,0.24,24.11,14.13,17.407,11.86,0.149,0.0,False,GPU -25,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1260.0,+70/-66,0.183,25.0,15.68,16.571,14.59,4.035,0.0,False,GPU -26,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1250.0,+62/-68,0.156,25.61,12.67,17.351,2989.36,0.063,0.0,False,CPU -27,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1249.0,+77/-84,0.194,25.7,11.39,16.867,1165.19,0.105,0.0,False,GPU -28,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1247.0,+81/-65,0.161,25.82,12.12,18.065,8.12,0.053,0.0,False,CPU -29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1243.0,+83/-70,0.172,26.04,18.42,17.009,2894.94,2.42,0.0,False,CPU -30,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1237.0,+82/-100,0.255,26.39,8.68,16.048,51.78,39.927,0.0,False,GPU -31,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1198.0,+100/-88,0.175,28.73,18.61,19.791,1425.28,7.355,0.0,False,CPU -32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1188.0,+89/-73,0.153,29.32,21.46,18.976,2894.94,0.145,0.0,False,CPU -33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1179.0,+81/-82,0.139,29.89,19.84,20.674,402.38,1.373,0.0,False,CPU -34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1167.0,+74/-90,0.119,30.57,12.6,20.919,14.22,0.353,0.0,False,GPU -35,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1156.0,+85/-85,0.147,31.2,15.48,21.194,402.38,0.163,0.0,False,CPU -36,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1137.0,+68/-74,0.068,32.32,27.34,19.723,2.85,0.19,0.0,False,CPU -37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1136.0,+77/-96,0.114,32.36,22.36,21.769,562.0,1.155,0.0,False,CPU -38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1136.0,+86/-93,0.135,32.39,19.06,21.239,1425.28,0.561,0.0,False,CPU -39,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1132.0,+66/-78,0.099,32.59,26.88,20.192,3.27,0.193,0.0,False,CPU -40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1092.0,+71/-81,0.052,34.84,27.16,22.751,562.0,0.138,0.0,False,CPU -41,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1012.0,+100/-117,0.079,38.93,29.19,27.565,2.45,0.519,0.0,False,GPU -42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1010.0,+81/-101,0.037,39.04,33.86,25.349,9.06,0.196,0.0,False,CPU -43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1001.0,+90/-124,0.058,39.43,34.33,25.341,5.43,0.836,0.0,False,GPU -44,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+83/-113,0.026,39.48,26.99,28.169,0.46,0.07,0.0,False,CPU -45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,978.0,+88/-115,0.037,40.46,34.33,28.52,123.01,1.579,0.0,False,CPU -46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,978.0,+86/-99,0.038,40.5,36.42,25.269,5.03,0.601,0.0,False,CPU -47,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,945.0,+93/-118,0.02,41.89,37.28,29.809,0.38,0.072,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,941.0,+88/-110,0.036,42.07,17.14,31.884,479.28,0.495,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,912.0,+89/-121,0.031,43.2,25.1,32.518,479.28,0.098,0.0,False,CPU -50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,877.0,+103/-132,0.018,44.46,40.34,35.489,2.34,0.132,0.0,False,CPU -51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,811.0,+102/-144,0.018,46.54,43.84,35.669,123.01,0.092,0.0,False,CPU -52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,572.0,+114/-198,0.0,51.14,50.77,49.495,0.28,0.024,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1610.0,+109/-82,7.306,0.641,7.88,3.92,6.03,0.703,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1592.0,+131/-96,4.54,0.66,8.46,2.69,2279.85,9.666,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1564.0,+130/-89,7.209,0.615,9.45,3.78,2279.85,0.968,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1535.0,+111/-88,8.868,0.589,10.57,4.69,635.11,8.569,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1487.0,+92/-69,11.474,0.473,12.61,6.55,3486.44,22.867,✔️,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1468.0,+96/-69,11.301,0.458,13.5,7.44,2993.17,1.401,✔️,0.0,False,CPU +6,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1456.0,+99/-63,10.565,0.476,14.09,5.46,4.53,0.611,➖,0.0,False,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1421.0,+77/-75,12.856,0.404,15.84,7.35,3486.44,1.002,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1394.0,+85/-74,11.685,0.412,17.29,6.14,7432.95,404.225,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1391.0,+83/-79,13.865,0.367,17.43,8.78,3083.36,1.48,✔️,0.0,False,GPU +10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1358.0,+62/-49,14.947,0.257,19.27,15.23,774.51,1.455,✔️,0.0,False,CPU +11,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1356.0,+104/-78,16.037,0.298,19.36,11.43,303.47,2.324,➖,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1349.0,+89/-90,13.112,0.365,19.79,5.93,7432.95,39.797,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1333.0,+94/-70,14.97,0.3,20.7,12.07,3083.36,0.189,✔️,0.0,False,GPU +14,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1322.0,+76/-65,14.88,0.251,21.36,14.57,1612.87,0.724,✔️,0.0,False,CPU +15,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1313.0,+86/-74,14.576,0.264,21.89,10.05,6286.73,0.473,✔️,0.0,False,GPU +16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1312.0,+82/-73,15.706,0.258,21.93,12.15,1165.19,2.067,✔️,0.0,False,GPU +17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1310.0,+68/-72,16.083,0.223,22.05,13.9,829.32,1.642,✔️,0.0,False,CPU +18,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1308.0,+58/-51,15.686,0.2,22.2,16.21,6.97,0.144,✔️,0.0,False,CPU +19,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1300.0,+67/-46,15.572,0.221,22.66,17.19,1612.87,0.061,✔️,0.0,False,CPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1299.0,+68/-66,15.928,0.21,22.71,18.04,774.51,0.282,✔️,0.0,False,CPU +21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1295.0,+71/-62,16.677,0.195,22.96,13.86,2989.36,0.609,✔️,0.0,False,CPU +22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1290.0,+73/-69,16.308,0.202,23.25,11.14,829.32,0.241,✔️,0.0,False,CPU +23,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1281.0,+92/-81,16.222,0.242,23.75,14.62,6286.73,9.386,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1275.0,+82/-83,17.407,0.24,24.11,14.13,11.86,0.149,✔️,0.0,False,GPU +25,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1260.0,+70/-66,16.571,0.183,25.0,15.68,14.59,4.035,✔️,0.0,False,GPU +26,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1250.0,+62/-68,17.351,0.156,25.61,12.67,2989.36,0.063,✔️,0.0,False,CPU +27,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1249.0,+77/-84,16.867,0.194,25.7,11.39,1165.19,0.105,✔️,0.0,False,GPU +28,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1247.0,+81/-65,18.065,0.161,25.82,12.12,8.12,0.053,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1243.0,+83/-70,17.009,0.172,26.04,18.42,2894.94,2.42,✔️,0.0,False,CPU +30,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1237.0,+82/-100,16.048,0.255,26.39,8.68,51.78,39.927,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1198.0,+100/-88,19.791,0.175,28.73,18.61,1425.28,7.355,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1188.0,+89/-73,18.976,0.153,29.32,21.46,2894.94,0.145,✔️,0.0,False,CPU +33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1179.0,+81/-82,20.674,0.139,29.89,19.84,402.38,1.373,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1167.0,+74/-90,20.919,0.119,30.57,12.6,14.22,0.353,✔️,0.0,False,GPU +35,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1156.0,+85/-85,21.194,0.147,31.2,15.48,402.38,0.163,✔️,0.0,False,CPU +36,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1137.0,+68/-74,19.723,0.068,32.32,27.34,2.85,0.19,✔️,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1136.0,+77/-96,21.769,0.114,32.36,22.36,562.0,1.155,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1136.0,+86/-93,21.239,0.135,32.39,19.06,1425.28,0.561,✔️,0.0,False,CPU +39,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1132.0,+66/-78,20.192,0.099,32.59,26.88,3.27,0.193,✔️,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1092.0,+71/-81,22.751,0.052,34.84,27.16,562.0,0.138,✔️,0.0,False,CPU +41,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1012.0,+100/-117,27.565,0.079,38.93,29.19,2.45,0.519,➖,0.0,False,GPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1010.0,+81/-101,25.349,0.037,39.04,33.86,9.06,0.196,✔️,0.0,False,CPU +43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1001.0,+90/-124,25.341,0.058,39.43,34.33,5.43,0.836,✔️,0.0,False,GPU +44,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+83/-113,28.169,0.026,39.48,26.99,0.46,0.07,✔️,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),978.0,+88/-115,28.52,0.037,40.46,34.33,123.01,1.579,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),978.0,+86/-99,25.269,0.038,40.5,36.42,5.03,0.601,✔️,0.0,False,CPU +47,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),945.0,+93/-118,29.809,0.02,41.89,37.28,0.38,0.072,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),941.0,+88/-110,31.884,0.036,42.07,17.14,479.28,0.495,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),912.0,+89/-121,32.518,0.031,43.2,25.1,479.28,0.098,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),877.0,+103/-132,35.489,0.018,44.46,40.34,2.34,0.132,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),811.0,+102/-144,35.669,0.018,46.54,43.84,123.01,0.092,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),572.0,+114/-198,49.495,0.0,51.14,50.77,0.28,0.024,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_small/winrate_matrix.png.zip index 1fca51f24d8676405ddffc53f7ff83c91d1198ed..31da0489d76be3efed0d687f3a93acc16554f3d6 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5afcee3ae2b3df2ff09415fbe6a55cc5ab59ca1052b7a50c9788900523555fb1 +oid sha256:9a4d933124aa7019f251cbb62e172808d912bf2b147595925824f1121dd78276 size 2005320 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 1066529811eaef6072ffdddad4df9a4e7d1b5961..892d67df8527f538d80b1a69b8cca880dee26bdc 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c76f2032f15bc5f3823418353bb630772d6e8c454891890127fdcdc5faf7683b +oid sha256:fa9ac2cff2a3ff15b63f761c902ed84a6d5bb7f57f670219c09383e64ea3867c size 481575 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip index c882ce462312d2bc574402e1c4d851e59231e50b..6b071a13f2bb87a17251e466802cbef17cdfc13c 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:42e415ce939a495f49002e5905cef5bd038017462a815d5cfa07d9d3744e50b9 -size 1038842 +oid sha256:ca05b464f55c0a1d5a5fbfaf3d98309387aaa3814f6fc035ee88270d24fb5071 +size 1023492 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip index df9c41b59e54e9bfa0952e60f46f3af41eda6049..f95024e4f15a828da284d45e1553c543edaf694a 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b5ce48c8471515189a0ee09615c926602bb7c5eb214e2050bb6b85ede4fe18b3 +oid sha256:1b2ea8e9fdc284d1093061fbad62f378b65118d9b815a19a5a425758003301a0 size 121062 diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/website_leaderboard.csv index c5cdb97720afb21c722cc84043868ea75819c1b0..1fe395cd30ca8ffe789f7e6c60ff47aaab3d9097 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1648.0,+88/-68,0.635,8.1,5.12,7.839,5.78,0.586,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1624.0,+124/-100,0.653,8.92,2.96,4.968,2162.76,8.93,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1601.0,+124/-104,0.61,9.81,4.39,7.727,2162.76,0.894,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1533.0,+113/-82,0.548,12.73,5.43,9.618,620.85,8.569,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1510.0,+93/-63,0.463,13.81,6.88,12.159,3159.9,21.159,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1509.0,+107/-89,0.488,13.87,6.57,11.53,3361.32,51.669,0.0,False,GPU -6,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1472.0,+108/-68,0.432,15.79,7.34,12.029,150.4,4.403,0.0,False,GPU -7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1467.0,+82/-70,0.41,16.08,8.22,12.218,2628.27,1.401,0.0,False,CPU -8,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1460.0,+100/-66,0.439,16.44,6.0,11.286,4.3,0.584,0.0,False,GPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1446.0,+89/-57,0.394,17.25,8.14,13.535,3159.9,0.646,0.0,False,GPU -10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1403.0,+84/-80,0.341,19.75,10.41,14.036,3361.32,0.527,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1403.0,+103/-85,0.381,19.77,6.66,12.433,6883.9,405.331,0.0,False,GPU -12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1392.0,+106/-73,0.335,20.42,9.82,14.604,2899.02,1.461,0.0,False,GPU -13,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1390.0,+84/-96,0.354,20.52,6.2,13.131,9.07,2.008,0.0,False,GPU -14,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1377.0,+96/-83,0.298,21.31,12.92,16.382,303.47,2.098,0.0,False,GPU -15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1360.0,+81/-99,0.341,22.42,6.69,13.867,6883.9,39.606,0.0,False,GPU -16,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1345.0,+83/-90,0.316,23.35,7.94,15.148,4.28,0.511,0.0,False,GPU -17,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1345.0,+53/-41,0.192,23.37,19.01,16.138,691.46,1.455,0.0,False,CPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1336.0,+73/-88,0.265,23.88,10.86,15.115,6020.89,0.454,0.0,False,GPU -19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1330.0,+102/-70,0.27,24.33,14.64,15.756,2899.02,0.171,0.0,False,GPU -20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1326.0,+85/-72,0.213,24.52,16.47,16.007,1394.45,0.618,0.0,False,CPU -21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1308.0,+69/-57,0.182,25.71,19.95,17.318,785.9,1.281,0.0,False,CPU -22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1306.0,+76/-53,0.169,25.83,18.07,16.868,5.74,0.139,0.0,False,CPU -23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1305.0,+73/-64,0.185,25.92,15.16,17.587,2399.23,0.565,0.0,False,CPU -24,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1300.0,+79/-73,0.2,26.27,18.27,16.877,1053.72,1.872,0.0,False,GPU -25,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1297.0,+91/-84,0.239,26.42,17.12,17.031,6020.89,8.478,0.0,False,GPU -26,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1293.0,+74/-51,0.173,26.69,21.51,16.785,1394.45,0.053,0.0,False,CPU -27,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1290.0,+72/-60,0.166,26.88,12.61,17.51,785.9,0.213,0.0,False,CPU -28,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1288.0,+71/-52,0.163,27.0,21.23,17.102,691.46,0.264,0.0,False,CPU -29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1285.0,+102/-73,0.23,27.19,16.18,18.07,11.6,0.148,0.0,False,GPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1284.0,+83/-65,0.17,27.27,16.3,17.108,12.75,3.502,0.0,False,GPU -31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1257.0,+98/-127,0.259,29.04,8.67,16.756,51.52,40.126,0.0,False,GPU -32,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1254.0,+90/-93,0.139,29.23,12.47,18.783,7.41,0.052,0.0,False,CPU -33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1254.0,+60/-73,0.136,29.27,13.11,18.357,2399.23,0.061,0.0,False,CPU -34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1253.0,+73/-74,0.169,29.31,21.75,17.668,2792.14,2.175,0.0,False,CPU -35,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1234.0,+85/-76,0.126,30.56,21.73,18.071,1053.72,0.103,0.0,False,GPU -36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1221.0,+93/-108,0.17,31.37,19.9,20.456,1350.04,6.57,0.0,False,CPU -37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1191.0,+72/-113,0.109,33.29,12.56,21.041,13.67,0.339,0.0,False,GPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1183.0,+80/-82,0.119,33.81,26.42,19.614,2792.14,0.133,0.0,False,CPU -39,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1160.0,+84/-81,0.086,35.21,27.55,22.147,378.34,1.373,0.0,False,CPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1154.0,+96/-109,0.126,35.58,21.37,21.966,1350.04,0.613,0.0,False,CPU -41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1137.0,+91/-92,0.107,36.62,19.58,22.595,378.34,0.154,0.0,False,CPU -42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1128.0,+60/-59,0.033,37.19,33.98,21.116,2.66,0.17,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1118.0,+107/-106,0.078,37.77,26.65,23.308,527.27,1.155,0.0,False,CPU -44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1103.0,+69/-69,0.047,38.63,34.62,21.73,3.01,0.177,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1079.0,+91/-89,0.045,40.04,32.03,24.138,527.27,0.128,0.0,False,CPU -46,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1029.0,+86/-152,0.069,42.69,32.05,28.036,2.28,0.454,0.0,False,GPU -47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1013.0,+91/-94,0.036,43.5,36.97,26.238,8.93,0.172,0.0,False,CPU -48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+86/-108,0.027,44.15,31.42,28.762,0.44,0.065,0.0,False,CPU -49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,999.0,+93/-122,0.051,44.19,38.38,26.76,5.0,0.753,0.0,False,GPU -50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,990.0,+94/-109,0.031,44.62,40.08,25.606,4.68,0.547,0.0,False,CPU -51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,959.0,+80/-107,0.009,46.04,43.24,29.843,118.4,1.479,0.0,False,CPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,956.0,+94/-99,0.038,46.17,16.98,31.197,429.17,0.495,0.0,False,CPU -53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,946.0,+87/-146,0.02,46.62,41.57,30.28,0.36,0.068,0.0,False,CPU -54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,930.0,+99/-111,0.032,47.27,25.54,31.807,429.17,0.081,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,897.0,+108/-120,0.015,48.58,43.99,34.214,2.16,0.129,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,766.0,+80/-151,0.0,52.58,51.76,37.314,118.4,0.082,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,576.0,+118/-291,0.0,56.08,55.73,49.46,0.27,0.024,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1648.0,+88/-68,7.839,0.635,8.1,5.12,5.78,0.586,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1624.0,+124/-100,4.968,0.653,8.92,2.96,2162.76,8.93,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1601.0,+124/-104,7.727,0.61,9.81,4.39,2162.76,0.894,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1533.0,+113/-82,9.618,0.548,12.73,5.43,620.85,8.569,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1510.0,+93/-63,12.159,0.463,13.81,6.88,3159.9,21.159,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1509.0,+107/-89,11.53,0.488,13.87,6.57,3361.32,51.669,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1472.0,+108/-68,12.029,0.432,15.79,7.34,150.4,4.403,✔️,0.0,False,GPU +7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1467.0,+82/-70,12.218,0.41,16.08,8.22,2628.27,1.401,✔️,0.0,False,CPU +8,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1460.0,+100/-66,11.286,0.439,16.44,6.0,4.3,0.584,➖,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1446.0,+89/-57,13.535,0.394,17.25,8.14,3159.9,0.646,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1403.0,+84/-80,14.036,0.341,19.75,10.41,3361.32,0.527,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1403.0,+103/-85,12.433,0.381,19.77,6.66,6883.9,405.331,✔️,0.0,False,GPU +12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1392.0,+106/-73,14.604,0.335,20.42,9.82,2899.02,1.461,✔️,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1390.0,+84/-96,13.131,0.354,20.52,6.2,9.07,2.008,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1377.0,+96/-83,16.382,0.298,21.31,12.92,303.47,2.098,➖,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1360.0,+81/-99,13.867,0.341,22.42,6.69,6883.9,39.606,✔️,0.0,False,GPU +16,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1345.0,+83/-90,15.148,0.316,23.35,7.94,4.28,0.511,✔️,0.0,False,GPU +17,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1345.0,+53/-41,16.138,0.192,23.37,19.01,691.46,1.455,✔️,0.0,False,CPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1336.0,+73/-88,15.115,0.265,23.88,10.86,6020.89,0.454,✔️,0.0,False,GPU +19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1330.0,+102/-70,15.756,0.27,24.33,14.64,2899.02,0.171,✔️,0.0,False,GPU +20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1326.0,+85/-72,16.007,0.213,24.52,16.47,1394.45,0.618,✔️,0.0,False,CPU +21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1308.0,+69/-57,17.318,0.182,25.71,19.95,785.9,1.281,✔️,0.0,False,CPU +22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1306.0,+76/-53,16.868,0.169,25.83,18.07,5.74,0.139,✔️,0.0,False,CPU +23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1305.0,+73/-64,17.587,0.185,25.92,15.16,2399.23,0.565,✔️,0.0,False,CPU +24,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1300.0,+79/-73,16.877,0.2,26.27,18.27,1053.72,1.872,✔️,0.0,False,GPU +25,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1297.0,+91/-84,17.031,0.239,26.42,17.12,6020.89,8.478,✔️,0.0,False,GPU +26,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1293.0,+74/-51,16.785,0.173,26.69,21.51,1394.45,0.053,✔️,0.0,False,CPU +27,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1290.0,+72/-60,17.51,0.166,26.88,12.61,785.9,0.213,✔️,0.0,False,CPU +28,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1288.0,+71/-52,17.102,0.163,27.0,21.23,691.46,0.264,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1285.0,+102/-73,18.07,0.23,27.19,16.18,11.6,0.148,✔️,0.0,False,GPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1284.0,+83/-65,17.108,0.17,27.27,16.3,12.75,3.502,✔️,0.0,False,GPU +31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1257.0,+98/-127,16.756,0.259,29.04,8.67,51.52,40.126,✔️,0.0,False,GPU +32,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1254.0,+90/-93,18.783,0.139,29.23,12.47,7.41,0.052,✔️,0.0,False,CPU +33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1254.0,+60/-73,18.357,0.136,29.27,13.11,2399.23,0.061,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1253.0,+73/-74,17.668,0.169,29.31,21.75,2792.14,2.175,✔️,0.0,False,CPU +35,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1234.0,+85/-76,18.071,0.126,30.56,21.73,1053.72,0.103,✔️,0.0,False,GPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1221.0,+93/-108,20.456,0.17,31.37,19.9,1350.04,6.57,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1191.0,+72/-113,21.041,0.109,33.29,12.56,13.67,0.339,✔️,0.0,False,GPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1183.0,+80/-82,19.614,0.119,33.81,26.42,2792.14,0.133,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1160.0,+84/-81,22.147,0.086,35.21,27.55,378.34,1.373,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1154.0,+96/-109,21.966,0.126,35.58,21.37,1350.04,0.613,✔️,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1137.0,+91/-92,22.595,0.107,36.62,19.58,378.34,0.154,✔️,0.0,False,CPU +42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1128.0,+60/-59,21.116,0.033,37.19,33.98,2.66,0.17,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1118.0,+107/-106,23.308,0.078,37.77,26.65,527.27,1.155,✔️,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1103.0,+69/-69,21.73,0.047,38.63,34.62,3.01,0.177,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1079.0,+91/-89,24.138,0.045,40.04,32.03,527.27,0.128,✔️,0.0,False,CPU +46,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1029.0,+86/-152,28.036,0.069,42.69,32.05,2.28,0.454,➖,0.0,False,GPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1013.0,+91/-94,26.238,0.036,43.5,36.97,8.93,0.172,✔️,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+86/-108,28.762,0.027,44.15,31.42,0.44,0.065,✔️,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),999.0,+93/-122,26.76,0.051,44.19,38.38,5.0,0.753,✔️,0.0,False,GPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),990.0,+94/-109,25.606,0.031,44.62,40.08,4.68,0.547,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),959.0,+80/-107,29.843,0.009,46.04,43.24,118.4,1.479,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),956.0,+94/-99,31.197,0.038,46.17,16.98,429.17,0.495,✔️,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),946.0,+87/-146,30.28,0.02,46.62,41.57,0.36,0.068,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),930.0,+99/-111,31.807,0.032,47.27,25.54,429.17,0.081,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),897.0,+108/-120,34.214,0.015,48.58,43.99,2.16,0.129,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),766.0,+80/-151,37.314,0.0,52.58,51.76,118.4,0.082,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),576.0,+118/-291,49.46,0.0,56.08,55.73,0.27,0.024,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip index 330841195967124859a7bb07870ddcf4005f951f..7e388b6610bb266ed5ca0621a47199775463a075 100644 --- a/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:144cce74b4d1987e758e0bd815195cf88ac08d5e25ccf6751a48e7580428d27c +oid sha256:4afa3894e7ed5e2dbc8a8d4b95fe3efbbaf117d85269c826b457149f91312eee size 2336052 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index 57003d563e2b8ad13a16cb7d0eec221a4ace58d2..26a1c306e38d539694e8881ffd80d95458aa7c89 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:29448b9639b65b8d438c36d28f34d54d4838750997e49c9f7bb756df67f0393c +oid sha256:9a3b951b5f0d70bbe82960d91ace39266362a513575e5b5735680accad76be86 size 422738 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip index b19427318370f75a74c6130ef65dd13589b8ca84..e00989f7ea7d7fbffceb6a81c8ad0c22b5d30983 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c4fe67b0531948eabf40de3116acf7d6c7be35bd7d3659852559a52a6d976bc5 -size 955458 +oid sha256:58eccea595fee2dee016f0b8fde884cf0a11d94000ce6911a50595aa8ab45dce +size 943077 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip index 6e3f254ee4f4747841063b792e2c58334bbd2d86..53e4edaf0dbb078ecc7fa68a89d247761e76a3c2 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c3a4deb9312ba0e10f0311dd9ee15bc95913eeb6d3951f2286db9d53fd1cb438 +oid sha256:c20a4e628f2f36dab93a9b3ea8627450655bdbf58d75cf117c582ccc91e6eb29 size 116890 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/website_leaderboard.csv index a944d92a6b22188f4dac96da04c94dcd01da802d..a1be6ad3fc77b2f8595469cd1d71ae1e4b9be11a 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/website_leaderboard.csv @@ -1,53 +1,53 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1720.0,+746/-199,0.786,5.62,1.64,1.808,2793.65,8.93,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1694.0,+276/-91,0.731,6.25,4.56,11.082,1008.06,9.121,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1678.0,+370/-169,0.673,6.69,3.46,9.911,2793.65,1.513,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1521.0,+345/-142,0.537,12.06,4.52,10.367,9.88,0.825,0.0,False,GPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1488.0,+229/-105,0.507,13.5,4.06,14.071,5546.65,3.268,0.0,False,CPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1454.0,+218/-127,0.426,15.12,5.15,15.484,4334.8,21.159,0.0,False,GPU -6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1448.0,+204/-96,0.378,15.38,11.91,15.024,3307.58,1.182,0.0,False,CPU -7,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1441.0,+200/-70,0.351,15.75,12.54,15.552,3307.58,0.14,0.0,False,CPU -8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1428.0,+142/-91,0.334,16.38,11.6,16.245,1280.01,4.078,0.0,False,CPU -9,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1421.0,+136/-103,0.324,16.75,10.98,16.321,1280.01,1.051,0.0,False,CPU -10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1418.0,+206/-131,0.381,16.88,12.61,15.962,4334.8,3.529,0.0,False,GPU -11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1413.0,+174/-112,0.339,17.12,12.01,18.699,5568.31,1.781,0.0,False,GPU -12,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1396.0,+154/-116,0.337,18.0,11.47,19.038,5568.31,0.372,0.0,False,GPU -13,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1394.0,+223/-132,0.377,18.12,10.3,19.449,9411.86,344.009,0.0,False,GPU -14,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1370.0,+216/-163,0.313,19.38,8.26,19.369,2079.21,3.347,0.0,False,GPU -15,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1361.0,+134/-89,0.255,19.88,18.12,16.949,2029.77,4.113,0.0,False,CPU -16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1354.0,+130/-108,0.262,20.25,18.02,16.708,2029.77,1.04,0.0,False,CPU -17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1333.0,+163/-121,0.249,21.44,16.78,18.145,6684.65,9.59,0.0,False,GPU -18,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1314.0,+218/-144,0.304,22.5,5.96,20.809,9411.86,39.507,0.0,False,GPU -19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1308.0,+161/-121,0.205,22.81,18.67,18.488,6684.65,0.752,0.0,False,GPU -20,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1305.0,+144/-127,0.236,23.0,14.3,22.14,17.09,0.145,0.0,False,GPU -21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1298.0,+155/-114,0.172,23.38,15.32,20.217,27108.48,0.742,0.0,False,CPU -22,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1295.0,+210/-158,0.243,23.56,12.64,21.612,728.32,2.441,0.0,False,CPU -23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1290.0,+104/-78,0.146,23.88,21.87,18.577,43.1,0.254,0.0,False,CPU -24,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1287.0,+249/-173,0.245,24.0,5.71,20.994,2079.21,0.319,0.0,False,GPU -25,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1280.0,+230/-206,0.265,24.38,11.89,20.771,729.17,1.833,0.0,False,CPU -26,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1268.0,+206/-145,0.213,25.06,12.59,21.742,728.32,0.356,0.0,False,CPU -27,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1267.0,+129/-128,0.181,25.12,19.54,21.941,283.57,4.104,0.0,False,GPU -28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1251.0,+131/-104,0.125,26.0,23.18,21.705,3646.83,2.162,0.0,False,CPU -29,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1233.0,+119/-130,0.153,27.0,22.32,21.727,27108.48,0.066,0.0,False,CPU -30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1228.0,+122/-124,0.131,27.25,24.61,22.231,17.86,0.099,0.0,False,CPU -31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1220.0,+145/-166,0.124,27.75,19.6,23.531,3646.83,0.194,0.0,False,CPU -32,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1208.0,+209/-184,0.179,28.38,16.27,22.297,729.17,0.325,0.0,False,CPU -33,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1195.0,+219/-198,0.186,29.12,17.06,25.651,2721.87,12.593,0.0,False,CPU -34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1195.0,+155/-153,0.136,29.12,22.73,22.982,5.12,0.444,0.0,False,CPU -35,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1188.0,+255/-173,0.216,29.5,10.16,24.06,46.36,39.812,0.0,False,GPU -36,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1188.0,+124/-124,0.069,29.5,26.9,20.547,4.93,0.592,0.0,False,CPU -37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1126.0,+151/-163,0.09,32.75,27.3,25.251,19.12,6.922,0.0,False,GPU -38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1094.0,+168/-176,0.095,34.38,25.91,27.296,2721.87,1.083,0.0,False,CPU -39,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1088.0,+162/-219,0.074,34.62,28.92,29.013,8.99,1.372,0.0,False,GPU -40,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1088.0,+126/-157,0.022,34.62,32.43,27.29,17.24,0.565,0.0,False,GPU -41,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1012.0,+267/-384,0.13,38.12,25.29,38.476,155.52,2.046,0.0,False,CPU -42,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+105/-213,0.0,38.62,36.72,35.562,0.74,0.148,0.0,False,CPU -43,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,975.0,+97/-144,0.0,39.62,38.73,30.565,14.37,0.363,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,928.0,+173/-268,0.031,41.38,36.93,34.532,8.37,0.659,0.0,False,CPU -45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,923.0,+84/-249,0.0,41.56,40.55,41.326,909.67,0.609,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,901.0,+89/-241,0.0,42.31,41.21,42.282,909.67,0.107,0.0,False,CPU -47,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,895.0,+167/-230,0.011,42.5,39.66,40.712,0.76,0.153,0.0,False,CPU -48,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,769.0,+281/-1308,0.063,46.0,38.91,50.559,155.52,0.191,0.0,False,CPU -49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,752.0,+76/-209,0.0,46.38,46.17,41.42,4.67,0.855,0.0,False,GPU -50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,710.0,+102/-321,0.0,47.25,46.92,48.104,4.62,0.135,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-292.0,+145/-1079,0.0,52.0,52.0,70.927,0.52,0.075,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1720.0,+746/-199,1.808,0.786,5.62,1.64,2793.65,8.93,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1694.0,+276/-91,11.082,0.731,6.25,4.56,1008.06,9.121,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1678.0,+370/-169,9.911,0.673,6.69,3.46,2793.65,1.513,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1521.0,+345/-142,10.367,0.537,12.06,4.52,9.88,0.825,✔️,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1488.0,+229/-105,14.071,0.507,13.5,4.06,5546.65,3.268,✔️,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1454.0,+218/-127,15.484,0.426,15.12,5.15,4334.8,21.159,✔️,0.0,False,GPU +6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1448.0,+204/-96,15.024,0.378,15.38,11.91,3307.58,1.182,✔️,0.0,False,CPU +7,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1441.0,+200/-70,15.552,0.351,15.75,12.54,3307.58,0.14,✔️,0.0,False,CPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1428.0,+142/-91,16.245,0.334,16.38,11.6,1280.01,4.078,✔️,0.0,False,CPU +9,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1421.0,+136/-103,16.321,0.324,16.75,10.98,1280.01,1.051,✔️,0.0,False,CPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1418.0,+206/-131,15.962,0.381,16.88,12.61,4334.8,3.529,✔️,0.0,False,GPU +11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1413.0,+174/-112,18.699,0.339,17.12,12.01,5568.31,1.781,✔️,0.0,False,GPU +12,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1396.0,+154/-116,19.038,0.337,18.0,11.47,5568.31,0.372,✔️,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1394.0,+223/-132,19.449,0.377,18.12,10.3,9411.86,344.009,✔️,0.0,False,GPU +14,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1370.0,+216/-163,19.369,0.313,19.38,8.26,2079.21,3.347,✔️,0.0,False,GPU +15,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1361.0,+134/-89,16.949,0.255,19.88,18.12,2029.77,4.113,✔️,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1354.0,+130/-108,16.708,0.262,20.25,18.02,2029.77,1.04,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1333.0,+163/-121,18.145,0.249,21.44,16.78,6684.65,9.59,✔️,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1314.0,+218/-144,20.809,0.304,22.5,5.96,9411.86,39.507,✔️,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1308.0,+161/-121,18.488,0.205,22.81,18.67,6684.65,0.752,✔️,0.0,False,GPU +20,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1305.0,+144/-127,22.14,0.236,23.0,14.3,17.09,0.145,✔️,0.0,False,GPU +21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1298.0,+155/-114,20.217,0.172,23.38,15.32,27108.48,0.742,✔️,0.0,False,CPU +22,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1295.0,+210/-158,21.612,0.243,23.56,12.64,728.32,2.441,✔️,0.0,False,CPU +23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1290.0,+104/-78,18.577,0.146,23.88,21.87,43.1,0.254,✔️,0.0,False,CPU +24,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1287.0,+249/-173,20.994,0.245,24.0,5.71,2079.21,0.319,✔️,0.0,False,GPU +25,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1280.0,+230/-206,20.771,0.265,24.38,11.89,729.17,1.833,✔️,0.0,False,CPU +26,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1268.0,+206/-145,21.742,0.213,25.06,12.59,728.32,0.356,✔️,0.0,False,CPU +27,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1267.0,+129/-128,21.941,0.181,25.12,19.54,283.57,4.104,➖,0.0,False,GPU +28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1251.0,+131/-104,21.705,0.125,26.0,23.18,3646.83,2.162,✔️,0.0,False,CPU +29,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1233.0,+119/-130,21.727,0.153,27.0,22.32,27108.48,0.066,✔️,0.0,False,CPU +30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1228.0,+122/-124,22.231,0.131,27.25,24.61,17.86,0.099,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1220.0,+145/-166,23.531,0.124,27.75,19.6,3646.83,0.194,✔️,0.0,False,CPU +32,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1208.0,+209/-184,22.297,0.179,28.38,16.27,729.17,0.325,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1195.0,+219/-198,25.651,0.186,29.12,17.06,2721.87,12.593,✔️,0.0,False,CPU +34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1195.0,+155/-153,22.982,0.136,29.12,22.73,5.12,0.444,✔️,0.0,False,CPU +35,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1188.0,+255/-173,24.06,0.216,29.5,10.16,46.36,39.812,✔️,0.0,False,GPU +36,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1188.0,+124/-124,20.547,0.069,29.5,26.9,4.93,0.592,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1126.0,+151/-163,25.251,0.09,32.75,27.3,19.12,6.922,✔️,0.0,False,GPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1094.0,+168/-176,27.296,0.095,34.38,25.91,2721.87,1.083,✔️,0.0,False,CPU +39,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1088.0,+162/-219,29.013,0.074,34.62,28.92,8.99,1.372,✔️,0.0,False,GPU +40,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1088.0,+126/-157,27.29,0.022,34.62,32.43,17.24,0.565,✔️,0.0,False,GPU +41,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1012.0,+267/-384,38.476,0.13,38.12,25.29,155.52,2.046,✔️,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+105/-213,35.562,0.0,38.62,36.72,0.74,0.148,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),975.0,+97/-144,30.565,0.0,39.62,38.73,14.37,0.363,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),928.0,+173/-268,34.532,0.031,41.38,36.93,8.37,0.659,✔️,0.0,False,CPU +45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),923.0,+84/-249,41.326,0.0,41.56,40.55,909.67,0.609,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),901.0,+89/-241,42.282,0.0,42.31,41.21,909.67,0.107,✔️,0.0,False,CPU +47,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),895.0,+167/-230,40.712,0.011,42.5,39.66,0.76,0.153,✔️,0.0,False,CPU +48,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),769.0,+281/-1308,50.559,0.063,46.0,38.91,155.52,0.191,✔️,0.0,False,CPU +49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),752.0,+76/-209,41.42,0.0,46.38,46.17,4.67,0.855,➖,0.0,False,GPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),710.0,+102/-321,48.104,0.0,47.25,46.92,4.62,0.135,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-292.0,+145/-1079,70.927,0.0,52.0,52.0,0.52,0.075,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/winrate_matrix.png.zip index ceb46e1d66dc1d3ab1a03bb3def9b25c3dc7981c..1fb03d9a2a03a62bf82d34a4e8dc2a6800c5f9a7 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f44c9215d321325a14ac6cf78025744cf0e741a1129e14ee12c9b24a32e46d1f +oid sha256:9f39ef4c7bf315a01d012103b114904510134a874123ca6904b3a65f8c6320af size 1446692 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index eb960811c372a4320ec6aa5e015c13174c74be3c..ab51e9d26c14388a434bbd59e9da4f1ef669f0ec 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1812b98843d341304c501dd574fdfad807a23e9ee799aec728614df19c8b890b +oid sha256:6eccc6c810acafd9438808198bdad00e6a59783fad05270f9dc318f9e33db035 size 451105 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip index d8fa81862f8c84571b8961b8df31068db4e68d1a..daa8197457d202b447e7243878e38088f348574b 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f7fee4d9b48c505eb387673c73890374048693708f19e8ea622267e8b6e84dcd -size 957364 +oid sha256:9d910235fd0dcd742fe79457585aa7841148531ce81b233c76654399523a6e13 +size 948654 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip index 1bf01fae9dfd5c96cbef9b7b9f91540a430844b0..71ea97517666cfee2471289cd5b0125c2f0f4ba2 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:96e37ecf79b85f239b72bcde25f28b50e7018d8817833f0fdea1290721e6337f +oid sha256:4019e3af637e694d1c311d16688b18e33230eeb9a7b9191ebf607827db4d33e0 size 107975 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/website_leaderboard.csv index ed900c3d3f2df0c96ed91727ee80cc17ac49c7f9..f680a6b8392e345fe931fd78494d7ac947fec6fd 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/website_leaderboard.csv @@ -1,54 +1,54 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,25228.0,+0/-0,1.0,1.0,1.0,0.0,134.59,8.308,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,23917.0,+0/-0,0.934,2.0,2.0,1.729,276.98,4.897,0.0,False,CPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,22310.0,+0/-0,0.915,3.0,3.0,2.208,276.84,3.303,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,20483.0,+0/-0,0.883,4.0,4.0,3.024,134.59,2.819,0.0,False,GPU -4,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,19432.0,+0/-0,0.576,5.0,5.0,10.141,7855.1,0.565,0.0,False,CPU -5,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,18258.0,+0/-0,0.572,6.0,6.0,10.238,61.47,0.154,0.0,False,CPU -6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,17510.0,+0/-0,0.569,7.0,7.0,10.303,157.4,4.579,0.0,False,CPU -7,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,16643.0,+0/-0,0.567,8.0,8.0,10.333,157.4,0.401,0.0,False,CPU -8,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,15885.0,+0/-0,0.551,9.0,9.0,10.67,61.47,0.015,0.0,False,CPU -9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,15137.0,+0/-0,0.525,10.0,10.0,11.231,2687.46,0.361,0.0,False,CPU -10,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,14345.0,+0/-0,0.522,11.0,11.0,11.295,7855.1,0.048,0.0,False,CPU -11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,13528.0,+0/-0,0.517,12.0,12.0,11.389,2687.46,0.793,0.0,False,CPU -12,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,12768.0,+0/-0,0.515,13.0,13.0,11.441,456.33,1.171,0.0,False,CPU -13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,12036.0,+0/-0,0.501,14.0,14.0,11.722,456.33,0.047,0.0,False,CPU -14,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,11311.0,+0/-0,0.501,15.0,15.0,11.722,0.32,0.047,0.0,False,CPU -15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,10518.0,+0/-0,0.432,16.0,16.0,13.143,8.22,0.049,0.0,False,CPU -16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,9807.0,+0/-0,0.42,17.0,17.0,13.383,4378.82,25.595,0.0,False,GPU -17,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,9111.0,+0/-0,0.387,18.0,18.0,14.026,20.73,0.079,0.0,False,CPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,8390.0,+0/-0,0.386,19.0,19.0,14.047,4378.82,2.216,0.0,False,GPU -19,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,7635.0,+0/-0,0.356,20.0,20.0,14.639,1.48,0.535,0.0,False,GPU -20,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,6953.0,+0/-0,0.292,21.0,21.0,15.857,2.79,1.057,0.0,False,GPU -21,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,6264.0,+0/-0,0.249,22.0,22.0,16.662,20.88,0.359,0.0,False,CPU -22,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,5562.0,+0/-0,0.111,23.0,23.0,19.143,20.88,0.026,0.0,False,CPU -23,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,5007.0,+0/-0,0.059,24.0,24.0,20.028,4390.96,275.957,0.0,False,GPU -24,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,4306.0,+0/-0,0.024,25.0,25.0,20.621,2341.91,0.027,0.0,False,CPU -25,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,3678.0,+0/-0,0.004,26.0,26.0,20.955,14.53,1.747,0.0,False,GPU -26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,3004.0,+0/-0,0.0,27.0,27.0,21.025,2341.91,0.172,0.0,False,CPU -27,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,2329.0,+0/-0,0.0,28.0,28.0,21.961,1993.57,0.469,0.0,False,GPU -28,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1701.0,+0/-0,0.0,29.0,29.0,22.049,1993.57,0.055,0.0,False,GPU -29,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+0/-0,0.0,30.0,30.0,24.001,0.18,0.007,0.0,False,CPU -30,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,445.0,+0/-0,0.0,31.0,31.0,24.06,4390.96,38.525,0.0,False,GPU -31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,-256.0,+0/-0,0.0,32.0,32.0,24.187,41.21,39.406,0.0,False,GPU -32,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,-946.0,+0/-0,0.0,33.0,33.0,24.591,1322.48,0.546,0.0,False,GPU -33,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,-1628.0,+0/-0,0.0,34.0,34.0,25.183,1322.48,0.055,0.0,False,GPU -34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,-2382.0,+0/-0,0.0,35.0,35.0,26.439,0.21,0.047,0.0,False,CPU -35,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,-3104.0,+0/-0,0.0,36.0,36.0,29.215,2009.11,3.767,0.0,False,GPU -36,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,-3800.0,+0/-0,0.0,37.0,37.0,30.84,2009.11,0.974,0.0,False,GPU -37,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,-4511.0,+0/-0,0.0,38.0,38.0,31.796,6.84,0.09,0.0,False,GPU -38,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,-5304.0,+0/-0,0.0,39.0,39.0,31.813,3.34,0.028,0.0,False,CPU -39,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,-6029.0,+0/-0,0.0,40.0,40.0,34.136,66.06,1.783,0.0,False,GPU -40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,-6760.0,+0/-0,0.0,41.0,41.0,38.819,6.51,0.162,0.0,False,GPU -41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,-7521.0,+0/-0,0.0,42.0,42.0,41.321,553.48,0.174,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,-8338.0,+0/-0,0.0,43.0,43.0,42.976,553.48,1.636,0.0,False,CPU -43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,-9130.0,+0/-0,0.0,44.0,44.0,50.562,0.08,0.017,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,-9878.0,+0/-0,0.0,45.0,45.0,52.296,2.73,0.083,0.0,False,CPU -45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-10636.0,+0/-0,0.0,46.0,46.0,52.871,30.19,0.134,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-11503.0,+0/-0,0.0,47.0,47.0,55.965,30.19,0.017,0.0,False,CPU -47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-12250.0,+0/-0,0.0,48.0,48.0,56.26,0.13,0.015,0.0,False,CPU -48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-13425.0,+0/-0,0.0,49.0,49.0,57.211,76.41,2.853,0.0,False,CPU -49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,-14476.0,+0/-0,0.0,50.0,50.0,57.769,3.14,0.921,0.0,False,GPU -50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,-16303.0,+0/-0,0.0,51.0,51.0,57.822,0.11,0.023,0.0,False,GPU -51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-17910.0,+0/-0,0.0,52.0,52.0,70.703,76.41,0.258,0.0,False,CPU -52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-19221.0,+0/-0,0.0,53.0,53.0,83.729,0.14,0.123,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),25228.0,+0/-0,0.0,1.0,1.0,1.0,134.59,8.308,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",23917.0,+0/-0,1.729,0.934,2.0,2.0,276.98,4.897,✔️,0.0,False,CPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",22310.0,+0/-0,2.208,0.915,3.0,3.0,276.84,3.303,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),20483.0,+0/-0,3.024,0.883,4.0,4.0,134.59,2.819,✔️,0.0,False,GPU +4,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),19432.0,+0/-0,10.141,0.576,5.0,5.0,7855.1,0.565,✔️,0.0,False,CPU +5,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),18258.0,+0/-0,10.238,0.572,6.0,6.0,61.47,0.154,✔️,0.0,False,CPU +6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),17510.0,+0/-0,10.303,0.569,7.0,7.0,157.4,4.579,✔️,0.0,False,CPU +7,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),16643.0,+0/-0,10.333,0.567,8.0,8.0,157.4,0.401,✔️,0.0,False,CPU +8,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),15885.0,+0/-0,10.67,0.551,9.0,9.0,61.47,0.015,✔️,0.0,False,CPU +9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),15137.0,+0/-0,11.231,0.525,10.0,10.0,2687.46,0.361,✔️,0.0,False,CPU +10,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),14345.0,+0/-0,11.295,0.522,11.0,11.0,7855.1,0.048,✔️,0.0,False,CPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),13528.0,+0/-0,11.389,0.517,12.0,12.0,2687.46,0.793,✔️,0.0,False,CPU +12,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),12768.0,+0/-0,11.441,0.515,13.0,13.0,456.33,1.171,✔️,0.0,False,CPU +13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),12036.0,+0/-0,11.722,0.501,14.0,14.0,456.33,0.047,✔️,0.0,False,CPU +14,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),11311.0,+0/-0,11.722,0.501,15.0,15.0,0.32,0.047,✔️,0.0,False,CPU +15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),10518.0,+0/-0,13.143,0.432,16.0,16.0,8.22,0.049,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),9807.0,+0/-0,13.383,0.42,17.0,17.0,4378.82,25.595,✔️,0.0,False,GPU +17,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),9111.0,+0/-0,14.026,0.387,18.0,18.0,20.73,0.079,✔️,0.0,False,CPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),8390.0,+0/-0,14.047,0.386,19.0,19.0,4378.82,2.216,✔️,0.0,False,GPU +19,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),7635.0,+0/-0,14.639,0.356,20.0,20.0,1.48,0.535,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),6953.0,+0/-0,15.857,0.292,21.0,21.0,2.79,1.057,✔️,0.0,False,GPU +21,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),6264.0,+0/-0,16.662,0.249,22.0,22.0,20.88,0.359,✔️,0.0,False,CPU +22,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),5562.0,+0/-0,19.143,0.111,23.0,23.0,20.88,0.026,✔️,0.0,False,CPU +23,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),5007.0,+0/-0,20.028,0.059,24.0,24.0,4390.96,275.957,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),4306.0,+0/-0,20.621,0.024,25.0,25.0,2341.91,0.027,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),3678.0,+0/-0,20.955,0.004,26.0,26.0,14.53,1.747,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),3004.0,+0/-0,21.025,0.0,27.0,27.0,2341.91,0.172,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),2329.0,+0/-0,21.961,0.0,28.0,28.0,1993.57,0.469,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1701.0,+0/-0,22.049,0.0,29.0,29.0,1993.57,0.055,✔️,0.0,False,GPU +29,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+0/-0,24.001,0.0,30.0,30.0,0.18,0.007,✔️,0.0,False,CPU +30,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),445.0,+0/-0,24.06,0.0,31.0,31.0,4390.96,38.525,✔️,0.0,False,GPU +31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),-256.0,+0/-0,24.187,0.0,32.0,32.0,41.21,39.406,✔️,0.0,False,GPU +32,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),-946.0,+0/-0,24.591,0.0,33.0,33.0,1322.48,0.546,✔️,0.0,False,GPU +33,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),-1628.0,+0/-0,25.183,0.0,34.0,34.0,1322.48,0.055,✔️,0.0,False,GPU +34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),-2382.0,+0/-0,26.439,0.0,35.0,35.0,0.21,0.047,✔️,0.0,False,CPU +35,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),-3104.0,+0/-0,29.215,0.0,36.0,36.0,2009.11,3.767,✔️,0.0,False,GPU +36,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),-3800.0,+0/-0,30.84,0.0,37.0,37.0,2009.11,0.974,✔️,0.0,False,GPU +37,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),-4511.0,+0/-0,31.796,0.0,38.0,38.0,6.84,0.09,✔️,0.0,False,GPU +38,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),-5304.0,+0/-0,31.813,0.0,39.0,39.0,3.34,0.028,✔️,0.0,False,CPU +39,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),-6029.0,+0/-0,34.136,0.0,40.0,40.0,66.06,1.783,➖,0.0,False,GPU +40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),-6760.0,+0/-0,38.819,0.0,41.0,41.0,6.51,0.162,✔️,0.0,False,GPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),-7521.0,+0/-0,41.321,0.0,42.0,42.0,553.48,0.174,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),-8338.0,+0/-0,42.976,0.0,43.0,43.0,553.48,1.636,✔️,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),-9130.0,+0/-0,50.562,0.0,44.0,44.0,0.08,0.017,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),-9878.0,+0/-0,52.296,0.0,45.0,45.0,2.73,0.083,✔️,0.0,False,CPU +45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),-10636.0,+0/-0,52.871,0.0,46.0,46.0,30.19,0.134,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),-11503.0,+0/-0,55.965,0.0,47.0,47.0,30.19,0.017,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),-12250.0,+0/-0,56.26,0.0,48.0,48.0,0.13,0.015,✔️,0.0,False,CPU +48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),-13425.0,+0/-0,57.211,0.0,49.0,49.0,76.41,2.853,✔️,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),-14476.0,+0/-0,57.769,0.0,50.0,50.0,3.14,0.921,✔️,0.0,False,GPU +50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),-16303.0,+0/-0,57.822,0.0,51.0,51.0,0.11,0.023,➖,0.0,False,GPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),-17910.0,+0/-0,70.703,0.0,52.0,52.0,76.41,0.258,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-19221.0,+0/-0,83.729,0.0,53.0,53.0,0.14,0.123,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/winrate_matrix.png.zip index 5ebf7a0908d62c24eea90413a3d4a59af16f618a..65aa98995169e5887a86a31a9f6b72d31293a378 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:eae61392bfed037d20554aa6d1ceb8064a1145d57fdd5dbdc8a5bad0d955b073 +oid sha256:c56ce20c279dd3d62a929a1594d21d3fd3a6fcffe8b78f639a8128374228941b size 1057411 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 4903597b8356b7aca05e5d3ddfa3aaff62a7cf89..91d5f965a91363e5d3c1830c26b8fb8a3b9d4074 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:81826499b7764ac326c05d7f3688e9420dcf17e68544f35787e82f2306de4e3f +oid sha256:705dc4364e8d077276487955c04c1b751470216d73571cc7152fa63925a281aa size 420132 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip index 0edf73c2f6e8be017f8cd2efbf6347113c8a09b3..1b0d221d93922aa4ab256bad065b159179bae349 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9b63aae85dfcb94f39861e8fef24396a481b7bff5b07141ce40782984093927e -size 954003 +oid sha256:d8faed2226edad7d3538f50a0c61f66f275a792ddb190fef162f3c3130356814 +size 917622 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip index cc48aae2c232d5778c0f805bace307d1fa91c7a8..98f43b263e965d57ec969c632fc000a9339f12d4 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c5a7713cdce67dd6196975757021286317e9fa53b2847523ed235fc5dc137490 +oid sha256:9cbf1c64d1d616e2e4ded898808c3673a6ed72488dba8cea473ad09325b338bf size 120826 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/website_leaderboard.csv index 0e6a76b5edd0dc868d04ab23cea3ee4478d0213d..ebfc92896694a5444c2bfe8cccf3696a7fe65bfc 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/website_leaderboard.csv @@ -1,54 +1,54 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1735.0,+490/-208,0.754,6.43,1.8,2.066,2837.96,9.546,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1714.0,+270/-90,0.701,7.0,5.03,12.349,1060.41,10.928,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1692.0,+452/-185,0.639,7.64,3.61,10.895,2837.96,1.134,0.0,False,GPU -3,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1606.0,+237/-104,0.529,10.57,6.05,10.653,6.19,1.301,0.0,False,GPU -4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1586.0,+384/-174,0.555,11.36,4.28,9.756,10.97,0.856,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1542.0,+286/-141,0.482,13.29,4.65,14.183,4655.15,21.791,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1499.0,+207/-149,0.431,15.29,12.16,14.644,4655.15,4.031,0.0,False,GPU -7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1493.0,+283/-109,0.439,15.57,4.78,15.834,6318.92,3.195,0.0,False,CPU -8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1476.0,+162/-116,0.383,16.43,11.56,18.233,5729.39,1.817,0.0,False,GPU -9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1473.0,+246/-122,0.35,16.57,12.44,15.543,3375.11,1.208,0.0,False,CPU -10,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1459.0,+174/-119,0.381,17.29,10.81,18.608,5729.39,0.373,0.0,False,GPU -11,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1459.0,+194/-76,0.315,17.29,13.39,16.169,3375.11,0.132,0.0,False,CPU -12,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1448.0,+180/-192,0.351,17.86,7.75,17.962,2149.3,2.928,0.0,False,GPU -13,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1445.0,+228/-195,0.412,18.0,9.68,19.367,10182.26,399.569,0.0,False,GPU -14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1437.0,+194/-90,0.287,18.43,13.96,17.093,1394.84,3.578,0.0,False,CPU -15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1432.0,+228/-90,0.279,18.71,13.01,17.176,1394.84,1.087,0.0,False,CPU -16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1376.0,+118/-82,0.206,21.71,20.11,17.736,2342.58,5.131,0.0,False,CPU -17,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1374.0,+204/-136,0.267,21.86,13.5,20.76,19.13,0.146,0.0,False,GPU -18,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1371.0,+119/-98,0.217,22.0,19.75,17.42,2342.58,1.065,0.0,False,CPU -19,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1369.0,+252/-210,0.345,22.14,5.38,20.345,10182.26,40.124,0.0,False,GPU -20,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1355.0,+170/-136,0.214,22.93,18.01,18.825,6994.17,9.554,0.0,False,GPU -21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1351.0,+203/-188,0.277,23.14,5.23,19.587,2149.3,0.147,0.0,False,GPU -22,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1336.0,+148/-119,0.204,24.0,19.03,20.198,326.23,4.282,0.0,False,GPU -23,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1332.0,+178/-142,0.17,24.21,19.46,19.122,6994.17,0.583,0.0,False,GPU -24,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1322.0,+208/-231,0.236,24.79,12.36,22.319,772.2,2.495,0.0,False,CPU -25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1303.0,+128/-72,0.1,25.86,23.83,19.353,57.99,0.317,0.0,False,CPU -26,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1294.0,+198/-204,0.22,26.36,12.36,22.113,772.2,0.373,0.0,False,CPU -27,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1285.0,+151/-114,0.137,26.86,23.46,21.802,3833.57,2.294,0.0,False,CPU -28,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1285.0,+127/-98,0.11,26.86,22.23,21.656,44054.5,0.843,0.0,False,CPU -29,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1268.0,+204/-281,0.216,27.86,15.62,22.276,731.01,2.021,0.0,False,CPU -30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1263.0,+270/-230,0.206,28.14,17.27,23.176,2750.22,13.569,0.0,False,CPU -31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1245.0,+189/-158,0.132,29.14,19.47,23.947,3833.57,0.22,0.0,False,CPU -32,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1242.0,+194/-213,0.154,29.29,22.35,22.488,5.15,0.526,0.0,False,CPU -33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1237.0,+117/-124,0.09,29.57,27.07,23.403,14.99,0.113,0.0,False,CPU -34,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1230.0,+256/-231,0.247,30.0,9.32,24.042,51.52,40.219,0.0,False,GPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1227.0,+129/-113,0.095,30.14,26.81,23.217,44054.5,0.068,0.0,False,CPU -36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1190.0,+137/-224,0.122,32.14,20.49,23.958,731.01,0.334,0.0,False,CPU -37,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1182.0,+76/-92,0.004,32.57,31.38,21.807,5.38,0.624,0.0,False,CPU -38,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1179.0,+159/-187,0.1,32.71,27.33,23.313,20.63,7.532,0.0,False,GPU -39,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1163.0,+154/-219,0.081,33.57,28.55,24.905,9.4,1.444,0.0,False,GPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1146.0,+234/-242,0.108,34.43,26.13,25.292,2750.22,1.239,0.0,False,CPU -41,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1096.0,+121/-168,0.015,36.86,34.67,28.196,19.64,0.478,0.0,False,GPU -42,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1080.0,+257/-419,0.147,37.57,24.17,35.799,180.38,1.24,0.0,False,CPU -43,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1004.0,+115/-178,0.0,40.86,39.86,30.387,14.87,0.391,0.0,False,CPU -44,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+112/-308,0.0,41.0,39.22,37.213,0.78,0.155,0.0,False,CPU -45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,974.0,+204/-308,0.035,42.0,37.31,31.994,8.78,0.688,0.0,False,CPU -46,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,972.0,+82/-247,0.0,42.07,41.01,39.676,913.1,0.738,0.0,False,CPU -47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,952.0,+100/-294,0.0,42.79,41.64,40.327,913.1,0.13,0.0,False,CPU -48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,934.0,+190/-297,0.01,43.43,40.35,39.305,0.77,0.156,0.0,False,CPU -49,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,836.0,+283/-1332,0.072,46.29,39.0,47.681,180.38,0.136,0.0,False,CPU -50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,813.0,+47/-188,0.0,46.86,46.67,39.077,5.48,0.91,0.0,False,GPU -51,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,743.0,+101/-303,0.0,48.29,47.92,46.939,4.65,0.14,0.0,False,CPU -52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-420.0,+334/-866,0.0,53.0,53.0,69.099,0.58,0.071,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1735.0,+490/-208,2.066,0.754,6.43,1.8,2837.96,9.546,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1714.0,+270/-90,12.349,0.701,7.0,5.03,1060.41,10.928,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1692.0,+452/-185,10.895,0.639,7.64,3.61,2837.96,1.134,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1606.0,+237/-104,10.653,0.529,10.57,6.05,6.19,1.301,➖,0.0,False,GPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1586.0,+384/-174,9.756,0.555,11.36,4.28,10.97,0.856,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1542.0,+286/-141,14.183,0.482,13.29,4.65,4655.15,21.791,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1499.0,+207/-149,14.644,0.431,15.29,12.16,4655.15,4.031,✔️,0.0,False,GPU +7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1493.0,+283/-109,15.834,0.439,15.57,4.78,6318.92,3.195,✔️,0.0,False,CPU +8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1476.0,+162/-116,18.233,0.383,16.43,11.56,5729.39,1.817,✔️,0.0,False,GPU +9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1473.0,+246/-122,15.543,0.35,16.57,12.44,3375.11,1.208,✔️,0.0,False,CPU +10,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1459.0,+174/-119,18.608,0.381,17.29,10.81,5729.39,0.373,✔️,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1459.0,+194/-76,16.169,0.315,17.29,13.39,3375.11,0.132,✔️,0.0,False,CPU +12,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1448.0,+180/-192,17.962,0.351,17.86,7.75,2149.3,2.928,✔️,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1445.0,+228/-195,19.367,0.412,18.0,9.68,10182.26,399.569,✔️,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1437.0,+194/-90,17.093,0.287,18.43,13.96,1394.84,3.578,✔️,0.0,False,CPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1432.0,+228/-90,17.176,0.279,18.71,13.01,1394.84,1.087,✔️,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1376.0,+118/-82,17.736,0.206,21.71,20.11,2342.58,5.131,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1374.0,+204/-136,20.76,0.267,21.86,13.5,19.13,0.146,✔️,0.0,False,GPU +18,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1371.0,+119/-98,17.42,0.217,22.0,19.75,2342.58,1.065,✔️,0.0,False,CPU +19,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1369.0,+252/-210,20.345,0.345,22.14,5.38,10182.26,40.124,✔️,0.0,False,GPU +20,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1355.0,+170/-136,18.825,0.214,22.93,18.01,6994.17,9.554,✔️,0.0,False,GPU +21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1351.0,+203/-188,19.587,0.277,23.14,5.23,2149.3,0.147,✔️,0.0,False,GPU +22,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1336.0,+148/-119,20.198,0.204,24.0,19.03,326.23,4.282,➖,0.0,False,GPU +23,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1332.0,+178/-142,19.122,0.17,24.21,19.46,6994.17,0.583,✔️,0.0,False,GPU +24,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1322.0,+208/-231,22.319,0.236,24.79,12.36,772.2,2.495,✔️,0.0,False,CPU +25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1303.0,+128/-72,19.353,0.1,25.86,23.83,57.99,0.317,✔️,0.0,False,CPU +26,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1294.0,+198/-204,22.113,0.22,26.36,12.36,772.2,0.373,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1285.0,+151/-114,21.802,0.137,26.86,23.46,3833.57,2.294,✔️,0.0,False,CPU +28,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1285.0,+127/-98,21.656,0.11,26.86,22.23,44054.5,0.843,✔️,0.0,False,CPU +29,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1268.0,+204/-281,22.276,0.216,27.86,15.62,731.01,2.021,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1263.0,+270/-230,23.176,0.206,28.14,17.27,2750.22,13.569,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1245.0,+189/-158,23.947,0.132,29.14,19.47,3833.57,0.22,✔️,0.0,False,CPU +32,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1242.0,+194/-213,22.488,0.154,29.29,22.35,5.15,0.526,✔️,0.0,False,CPU +33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1237.0,+117/-124,23.403,0.09,29.57,27.07,14.99,0.113,✔️,0.0,False,CPU +34,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1230.0,+256/-231,24.042,0.247,30.0,9.32,51.52,40.219,✔️,0.0,False,GPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1227.0,+129/-113,23.217,0.095,30.14,26.81,44054.5,0.068,✔️,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1190.0,+137/-224,23.958,0.122,32.14,20.49,731.01,0.334,✔️,0.0,False,CPU +37,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1182.0,+76/-92,21.807,0.004,32.57,31.38,5.38,0.624,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1179.0,+159/-187,23.313,0.1,32.71,27.33,20.63,7.532,✔️,0.0,False,GPU +39,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1163.0,+154/-219,24.905,0.081,33.57,28.55,9.4,1.444,✔️,0.0,False,GPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1146.0,+234/-242,25.292,0.108,34.43,26.13,2750.22,1.239,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1096.0,+121/-168,28.196,0.015,36.86,34.67,19.64,0.478,✔️,0.0,False,GPU +42,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1080.0,+257/-419,35.799,0.147,37.57,24.17,180.38,1.24,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1004.0,+115/-178,30.387,0.0,40.86,39.86,14.87,0.391,✔️,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+112/-308,37.213,0.0,41.0,39.22,0.78,0.155,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),974.0,+204/-308,31.994,0.035,42.0,37.31,8.78,0.688,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),972.0,+82/-247,39.676,0.0,42.07,41.01,913.1,0.738,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),952.0,+100/-294,40.327,0.0,42.79,41.64,913.1,0.13,✔️,0.0,False,CPU +48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),934.0,+190/-297,39.305,0.01,43.43,40.35,0.77,0.156,✔️,0.0,False,CPU +49,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),836.0,+283/-1332,47.681,0.072,46.29,39.0,180.38,0.136,✔️,0.0,False,CPU +50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),813.0,+47/-188,39.077,0.0,46.86,46.67,5.48,0.91,➖,0.0,False,GPU +51,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),743.0,+101/-303,46.939,0.0,48.29,47.92,4.65,0.14,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-420.0,+334/-866,69.099,0.0,53.0,53.0,0.58,0.071,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/winrate_matrix.png.zip index 99737fefb923bf56f55481c7efa8a571d6891eae..9110ac5ac9f85cc25d49d31d88b8f8cc9ce0513c 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0d79f0882c3673bddbf59fa975801a9b5738d58afa79d94afacd8da62d257867 +oid sha256:784a04ce1ab4e79da2595b241e40589d9f11239b53278bcf609d587414fb2793 size 1435771 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index d3f437b39ade30f566216b7f3409472792a8c106..3aef60bac3c162343db2666bd53abe84855be7f9 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0450671d94f605a39f56c84a18e00dfc6e05049a9d39d4fa7ab252223399b5be +oid sha256:31238b7a81d2723b92a5f60639c05ac16a8333347a1fd57a48f685e3c733e955 size 456644 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip index 3cbf67d18acbd9071304c5fcf41261ea1b4c9014..3a3da247bcca7b19a9762a040b54317624ed24f5 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4b95440aa8b1ca7c2e94a4d13048ceb5de5d9978ceea4f3f797981bdff278c21 -size 1036337 +oid sha256:f5601473ef5b4871a921fc6ee01869603bee1a7bc35b68edd6401e0b30418689 +size 975573 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip index 4a59fe4c4720204207629ad652826339cb880fd2..5e04931cf7095d7842e8bf36d548403db41676fb 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9c63e9680d6e6ae47a09e56b4b23f175296c0f4fe43ce347e00f7de2a91db0eb +oid sha256:c6e00cef3f52b349a4f96f6f2bad5d782c604e4c52e8a8036ea79fcda149c108 size 125861 diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv index 4f05fb5c88ee5a61d5225392ab49c2f9a4cf43ae..a4c2ec6e6121067ab7b4a9e628750c998dfa2ed1 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1997.0,+809/-100,0.875,3.17,1.58,2.191,2793.65,8.93,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1857.0,+337/-82,0.729,5.42,4.12,12.495,2793.65,1.02,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1746.0,+356/-135,0.689,8.17,5.69,14.332,1295.16,14.354,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1716.0,+553/-144,0.632,9.08,4.46,11.083,9.88,0.825,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1688.0,+378/-116,0.596,10.0,5.22,14.698,3561.2,34.413,0.0,False,GPU -5,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1634.0,+358/-88,0.506,12.0,6.72,12.332,5.93,0.989,0.0,False,GPU -6,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1626.0,+204/-38,0.513,12.33,10.85,15.748,206.59,4.778,0.0,False,GPU -7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1622.0,+272/-96,0.534,12.5,4.43,16.24,4334.8,21.159,0.0,False,GPU -8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1583.0,+233/-63,0.472,14.17,12.85,16.59,4334.8,3.529,0.0,False,GPU -9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1522.0,+224/-109,0.384,17.0,14.03,17.857,3307.58,1.182,0.0,False,CPU -10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1506.0,+327/-108,0.373,17.83,11.35,16.716,3561.2,0.612,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1502.0,+371/-116,0.388,18.0,9.36,14.808,5.48,0.434,0.0,False,GPU -12,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1489.0,+378/-152,0.401,18.67,4.6,18.377,6700.69,3.268,0.0,False,CPU -13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1476.0,+201/-74,0.307,19.33,15.87,18.68,3307.58,0.108,0.0,False,CPU -14,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1445.0,+344/-241,0.382,21.0,9.92,22.477,9411.86,433.416,0.0,False,GPU -15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1433.0,+268/-147,0.283,21.67,15.18,21.223,5568.31,1.781,0.0,False,GPU -16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1418.0,+175/-44,0.219,22.5,21.18,19.816,2029.77,1.073,0.0,False,CPU -17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1412.0,+142/-39,0.197,22.83,22.15,20.228,2029.77,6.631,0.0,False,CPU -18,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1412.0,+262/-84,0.213,22.83,16.26,19.85,1280.01,5.479,0.0,False,CPU -19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1409.0,+244/-141,0.272,23.0,14.96,21.671,5568.31,0.372,0.0,False,GPU -20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1403.0,+285/-92,0.219,23.33,14.81,19.946,1280.01,1.143,0.0,False,CPU -21,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1398.0,+205/-127,0.219,23.58,19.93,21.447,6829.34,7.818,0.0,False,GPU -22,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1394.0,+268/-198,0.246,23.83,18.35,20.929,2059.45,3.769,0.0,False,GPU -23,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1379.0,+437/-228,0.297,24.67,6.92,22.95,13.87,2.152,0.0,False,GPU -24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1375.0,+264/-171,0.169,24.92,20.15,21.793,6829.34,0.53,0.0,False,GPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1356.0,+255/-176,0.229,26.0,13.94,24.08,17.09,0.145,0.0,False,GPU -26,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1353.0,+340/-277,0.32,26.17,4.97,23.608,9411.86,42.924,0.0,False,GPU -27,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1319.0,+250/-164,0.191,28.17,20.83,23.403,410.08,4.104,0.0,False,GPU -28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1302.0,+373/-354,0.223,29.17,18.56,26.238,2721.87,14.595,0.0,False,CPU -29,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1296.0,+161/-125,0.109,29.5,26.85,22.348,155.57,0.254,0.0,False,CPU -30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1278.0,+292/-226,0.145,30.5,21.21,22.852,2059.45,0.127,0.0,False,GPU -31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1270.0,+202/-155,0.115,31.0,23.64,25.058,27108.48,0.742,0.0,False,CPU -32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1264.0,+222/-200,0.119,31.33,27.49,25.243,4370.77,2.162,0.0,False,CPU -33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1246.0,+253/-279,0.12,32.33,25.27,26.005,728.32,2.548,0.0,False,CPU -34,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1235.0,+397/-370,0.284,33.0,8.74,27.727,42.75,45.46,0.0,False,GPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1217.0,+342/-353,0.148,34.0,18.85,25.893,729.17,2.154,0.0,False,CPU -36,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1211.0,+276/-254,0.102,34.33,25.35,25.765,728.32,0.356,0.0,False,CPU -37,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1190.0,+175/-206,0.075,35.5,30.79,26.917,27108.48,0.066,0.0,False,CPU -38,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1190.0,+176/-212,0.088,35.5,30.85,25.981,19.2,6.922,0.0,False,GPU -39,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1181.0,+89/-95,0.0,36.0,35.1,24.783,4.93,0.671,0.0,False,CPU -40,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1181.0,+170/-206,0.035,36.0,34.18,27.166,11.78,0.099,0.0,False,CPU -41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1159.0,+197/-173,0.06,37.17,32.88,26.155,5.12,0.444,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1156.0,+276/-281,0.113,37.33,27.67,28.736,2721.87,1.334,0.0,False,CPU -43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1153.0,+228/-281,0.078,37.5,31.64,28.727,8.99,1.498,0.0,False,GPU -44,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1146.0,+149/-137,0.006,37.83,36.17,27.899,4370.77,0.22,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1137.0,+257/-308,0.097,38.33,25.06,27.785,729.17,0.337,0.0,False,CPU -46,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1120.0,+154/-209,0.01,39.17,36.91,29.623,17.24,0.42,0.0,False,GPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+143/-432,0.0,44.67,42.99,37.954,0.74,0.148,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,979.0,+86/-362,0.0,45.5,44.67,38.521,909.67,0.893,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,962.0,+100/-303,0.0,46.17,45.13,39.28,909.67,0.107,0.0,False,CPU -50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,962.0,+232/-508,0.032,46.17,41.47,34.491,8.37,0.685,0.0,False,CPU -51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,939.0,+300/-564,0.037,47.0,40.48,41.716,155.52,1.033,0.0,False,CPU -52,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,934.0,+277/-430,0.004,47.17,43.75,39.387,0.76,0.153,0.0,False,CPU -53,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,924.0,+96/-253,0.0,47.5,47.01,35.184,14.57,0.363,0.0,False,CPU -54,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,748.0,+71/-303,0.0,52.17,51.98,44.164,4.67,0.855,0.0,False,GPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,723.0,+130/-462,0.0,52.67,52.34,44.545,4.62,0.135,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,372.0,+132/-1205,0.0,56.33,56.29,55.52,155.52,0.12,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-548.0,+343/-1220,0.0,58.0,58.0,70.032,0.52,0.058,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1997.0,+809/-100,2.191,0.875,3.17,1.58,2793.65,8.93,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1857.0,+337/-82,12.495,0.729,5.42,4.12,2793.65,1.02,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1746.0,+356/-135,14.332,0.689,8.17,5.69,1295.16,14.354,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1716.0,+553/-144,11.083,0.632,9.08,4.46,9.88,0.825,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1688.0,+378/-116,14.698,0.596,10.0,5.22,3561.2,34.413,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1634.0,+358/-88,12.332,0.506,12.0,6.72,5.93,0.989,➖,0.0,False,GPU +6,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1626.0,+204/-38,15.748,0.513,12.33,10.85,206.59,4.778,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1622.0,+272/-96,16.24,0.534,12.5,4.43,4334.8,21.159,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1583.0,+233/-63,16.59,0.472,14.17,12.85,4334.8,3.529,✔️,0.0,False,GPU +9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1522.0,+224/-109,17.857,0.384,17.0,14.03,3307.58,1.182,✔️,0.0,False,CPU +10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1506.0,+327/-108,16.716,0.373,17.83,11.35,3561.2,0.612,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1502.0,+371/-116,14.808,0.388,18.0,9.36,5.48,0.434,✔️,0.0,False,GPU +12,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1489.0,+378/-152,18.377,0.401,18.67,4.6,6700.69,3.268,✔️,0.0,False,CPU +13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1476.0,+201/-74,18.68,0.307,19.33,15.87,3307.58,0.108,✔️,0.0,False,CPU +14,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1445.0,+344/-241,22.477,0.382,21.0,9.92,9411.86,433.416,✔️,0.0,False,GPU +15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1433.0,+268/-147,21.223,0.283,21.67,15.18,5568.31,1.781,✔️,0.0,False,GPU +16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1418.0,+175/-44,19.816,0.219,22.5,21.18,2029.77,1.073,✔️,0.0,False,CPU +17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1412.0,+142/-39,20.228,0.197,22.83,22.15,2029.77,6.631,✔️,0.0,False,CPU +18,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1412.0,+262/-84,19.85,0.213,22.83,16.26,1280.01,5.479,✔️,0.0,False,CPU +19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1409.0,+244/-141,21.671,0.272,23.0,14.96,5568.31,0.372,✔️,0.0,False,GPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1403.0,+285/-92,19.946,0.219,23.33,14.81,1280.01,1.143,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1398.0,+205/-127,21.447,0.219,23.58,19.93,6829.34,7.818,✔️,0.0,False,GPU +22,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1394.0,+268/-198,20.929,0.246,23.83,18.35,2059.45,3.769,✔️,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1379.0,+437/-228,22.95,0.297,24.67,6.92,13.87,2.152,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1375.0,+264/-171,21.793,0.169,24.92,20.15,6829.34,0.53,✔️,0.0,False,GPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1356.0,+255/-176,24.08,0.229,26.0,13.94,17.09,0.145,✔️,0.0,False,GPU +26,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1353.0,+340/-277,23.608,0.32,26.17,4.97,9411.86,42.924,✔️,0.0,False,GPU +27,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1319.0,+250/-164,23.403,0.191,28.17,20.83,410.08,4.104,➖,0.0,False,GPU +28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1302.0,+373/-354,26.238,0.223,29.17,18.56,2721.87,14.595,✔️,0.0,False,CPU +29,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1296.0,+161/-125,22.348,0.109,29.5,26.85,155.57,0.254,✔️,0.0,False,CPU +30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1278.0,+292/-226,22.852,0.145,30.5,21.21,2059.45,0.127,✔️,0.0,False,GPU +31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1270.0,+202/-155,25.058,0.115,31.0,23.64,27108.48,0.742,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1264.0,+222/-200,25.243,0.119,31.33,27.49,4370.77,2.162,✔️,0.0,False,CPU +33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1246.0,+253/-279,26.005,0.12,32.33,25.27,728.32,2.548,✔️,0.0,False,CPU +34,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1235.0,+397/-370,27.727,0.284,33.0,8.74,42.75,45.46,✔️,0.0,False,GPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1217.0,+342/-353,25.893,0.148,34.0,18.85,729.17,2.154,✔️,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1211.0,+276/-254,25.765,0.102,34.33,25.35,728.32,0.356,✔️,0.0,False,CPU +37,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1190.0,+175/-206,26.917,0.075,35.5,30.79,27108.48,0.066,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1190.0,+176/-212,25.981,0.088,35.5,30.85,19.2,6.922,✔️,0.0,False,GPU +39,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1181.0,+89/-95,24.783,0.0,36.0,35.1,4.93,0.671,✔️,0.0,False,CPU +40,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1181.0,+170/-206,27.166,0.035,36.0,34.18,11.78,0.099,✔️,0.0,False,CPU +41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1159.0,+197/-173,26.155,0.06,37.17,32.88,5.12,0.444,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1156.0,+276/-281,28.736,0.113,37.33,27.67,2721.87,1.334,✔️,0.0,False,CPU +43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1153.0,+228/-281,28.727,0.078,37.5,31.64,8.99,1.498,✔️,0.0,False,GPU +44,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1146.0,+149/-137,27.899,0.006,37.83,36.17,4370.77,0.22,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1137.0,+257/-308,27.785,0.097,38.33,25.06,729.17,0.337,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1120.0,+154/-209,29.623,0.01,39.17,36.91,17.24,0.42,✔️,0.0,False,GPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+143/-432,37.954,0.0,44.67,42.99,0.74,0.148,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),979.0,+86/-362,38.521,0.0,45.5,44.67,909.67,0.893,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),962.0,+100/-303,39.28,0.0,46.17,45.13,909.67,0.107,✔️,0.0,False,CPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),962.0,+232/-508,34.491,0.032,46.17,41.47,8.37,0.685,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),939.0,+300/-564,41.716,0.037,47.0,40.48,155.52,1.033,✔️,0.0,False,CPU +52,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),934.0,+277/-430,39.387,0.004,47.17,43.75,0.76,0.153,✔️,0.0,False,CPU +53,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),924.0,+96/-253,35.184,0.0,47.5,47.01,14.57,0.363,✔️,0.0,False,CPU +54,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),748.0,+71/-303,44.164,0.0,52.17,51.98,4.67,0.855,➖,0.0,False,GPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),723.0,+130/-462,44.545,0.0,52.67,52.34,4.62,0.135,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),372.0,+132/-1205,55.52,0.0,56.33,56.29,155.52,0.12,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-548.0,+343/-1220,70.032,0.0,58.0,58.0,0.52,0.058,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip index 4195401ed2e3c411095ea85f129b36da9c298469..34217f66ba01051a307bf91c9176dc41140c7fcd 100644 --- a/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b9cc20b46186e388ec4f8e64098422e8c9a5863198eedea5cf9c6c3cfa75da69 +oid sha256:ae9c565c6ed73c863db8835fda0bf33788261e08b2320fc445f66973a7e33ce6 size 1681180 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index 8356129e13c41110fb5ab63217afee34c710a881..989b36f8b2eee94195794a42653c785b816eff5e 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:20388c0baf853058e971237856e1383b9ac660890ac805957c3626bc24ea2fa3 +oid sha256:dae14c6585f07ef55aa457d9b4e25458905605227b9e9124a927920b396f756d size 420357 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip index f234b68253948c7b3193bf93f67fd5160ebf8b92..1afadd659e2a78aa651947db4b14d4e42f201992 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3432fa60b73e63af1eb071844a8b53cfc7975d8ec716677ea9ae2ff31c22c397 -size 969672 +oid sha256:8c8bd8991b471e68b6234d723ce86b5fd25ade404293f508e8db540dfc88de1e +size 957317 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_all/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_all/tuning-impact-elo.png.zip index da4d17838d0f838724630e575104e792d27a6fba..70c20091fb93d6a9f52c52792e6fd04e32a0c4ae 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:10a7ddda84904aafba026cdc42b8a620d4ee2389d91a254598aee95555eaf54c +oid sha256:c3e4fcb6b06589974a27c10b5bcd4dda673df75b64ecf4d9be106a212f66e4cd size 120084 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_all/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_regression/datasets_all/website_leaderboard.csv index 0f8135132c4febae355137c293131396a8d62555..706f8247a907c2d73593e8c437c9f1c694cd6007 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_all/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_all/website_leaderboard.csv @@ -1,52 +1,52 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1932.0,+314/-174,0.799,5.15,2.41,1.161,2043.44,8.369,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1857.0,+142/-62,0.647,6.85,3.93,2.562,616.43,4.041,0.0,False,GPU -2,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1854.0,+258/-95,0.729,6.92,2.28,2.472,7125.58,336.907,0.0,False,GPU -3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1819.0,+190/-85,0.677,7.85,3.79,2.973,7125.58,38.573,0.0,False,GPU -4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1808.0,+228/-164,0.673,8.15,3.27,2.101,2043.44,0.781,0.0,False,GPU -5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1760.0,+101/-62,0.537,9.62,5.51,4.699,1489.94,5.79,0.0,False,CPU -6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1755.0,+118/-66,0.571,9.77,8.06,4.174,4057.12,10.478,0.0,False,GPU -7,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1692.0,+182/-93,0.541,11.85,6.26,4.157,44.65,38.298,0.0,False,GPU -8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1690.0,+216/-178,0.532,11.92,6.71,3.576,6.94,0.519,0.0,False,GPU -9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1663.0,+157/-141,0.505,12.92,5.46,6.669,3811.43,7.581,0.0,False,GPU -10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1616.0,+147/-131,0.433,14.69,10.42,5.331,4057.12,0.337,0.0,False,GPU -11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1611.0,+92/-59,0.328,14.92,12.75,7.3,686.46,5.477,0.0,False,CPU -12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1582.0,+93/-70,0.29,16.08,12.31,7.083,2895.38,1.317,0.0,False,CPU -13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1564.0,+127/-84,0.358,16.85,13.31,6.477,708.95,1.973,0.0,False,GPU -14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1551.0,+132/-103,0.311,17.38,13.53,5.748,4228.53,1.188,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1542.0,+123/-97,0.271,17.77,9.88,7.207,2895.38,0.073,0.0,False,CPU -16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1516.0,+104/-90,0.23,18.85,15.39,8.098,686.46,0.743,0.0,False,CPU -17,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1512.0,+232/-213,0.404,19.0,4.97,13.045,5.49,0.754,0.0,False,GPU -18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1503.0,+118/-74,0.257,19.38,16.18,7.042,708.95,0.069,0.0,False,GPU -19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1472.0,+71/-57,0.174,20.69,19.28,8.021,848.99,0.475,0.0,False,CPU -20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1472.0,+60/-49,0.173,20.69,19.7,8.036,848.99,2.378,0.0,False,CPU -21,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1460.0,+109/-92,0.202,21.23,18.6,9.327,16.07,0.29,0.0,False,GPU -22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1444.0,+134/-95,0.191,21.92,18.71,6.582,4228.53,0.125,0.0,False,GPU -23,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1434.0,+150/-147,0.259,22.38,13.93,9.198,3811.43,0.452,0.0,False,GPU -24,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1430.0,+117/-136,0.173,22.54,18.16,9.171,8.35,0.087,0.0,False,CPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1357.0,+136/-132,0.129,25.77,22.76,8.245,13.9,0.123,0.0,False,GPU -26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1309.0,+141/-114,0.087,27.85,23.61,10.088,9.17,1.416,0.0,False,GPU -27,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1296.0,+108/-102,0.078,28.38,25.93,9.416,4452.11,0.852,0.0,False,CPU -28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1295.0,+134/-114,0.093,28.46,23.52,12.273,161.73,0.784,0.0,False,CPU -29,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1271.0,+135/-118,0.105,29.46,23.46,12.586,161.73,0.12,0.0,False,CPU -30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1241.0,+130/-136,0.064,30.69,27.38,10.034,4452.11,0.089,0.0,False,CPU -31,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1241.0,+179/-168,0.076,30.69,14.6,12.917,2.47,0.755,0.0,False,GPU -32,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1224.0,+76/-101,0.013,31.38,30.46,10.917,2.03,0.296,0.0,False,CPU -33,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1214.0,+159/-194,0.12,31.77,21.26,13.421,3017.28,0.248,0.0,False,CPU -34,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1191.0,+97/-118,0.053,32.69,30.4,11.634,2.15,0.18,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1182.0,+83/-122,0.018,33.08,31.12,13.249,526.17,0.77,0.0,False,CPU -36,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1166.0,+155/-206,0.119,33.69,15.28,13.982,3017.28,0.027,0.0,False,CPU -37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1138.0,+95/-134,0.016,34.73,32.85,14.003,526.17,0.119,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1119.0,+104/-116,0.014,35.46,33.28,14.183,0.42,0.061,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1084.0,+75/-108,0.0,36.69,35.72,14.282,527.21,2.831,0.0,False,CPU -40,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1039.0,+118/-167,0.049,38.15,35.84,15.082,10.04,0.03,0.0,False,CPU -41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1037.0,+92/-138,0.0,38.23,36.79,14.125,20.5,0.081,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1034.0,+97/-114,0.0,38.31,37.3,14.768,527.21,0.311,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+73/-90,0.0,39.35,38.95,15.317,0.63,0.064,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,887.0,+94/-166,0.0,42.31,41.7,18.821,3.08,0.293,0.0,False,CPU -45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,877.0,+134/-183,0.0,42.54,41.76,20.298,90.87,0.906,0.0,False,CPU -46,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,713.0,+109/-192,0.0,45.54,45.39,22.447,90.87,0.055,0.0,False,CPU -47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,572.0,+205/-344,0.0,47.31,46.87,29.125,0.25,0.044,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,439.0,+113/-319,0.0,48.58,48.52,36.211,177.04,0.14,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,351.0,+147/-403,0.0,49.27,49.19,36.396,177.04,0.059,0.0,False,CPU -50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,199.0,+138/-328,0.0,50.23,50.22,38.856,0.88,0.109,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1932.0,+314/-174,1.161,0.799,5.15,2.41,2043.44,8.369,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1857.0,+142/-62,2.562,0.647,6.85,3.93,616.43,4.041,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1854.0,+258/-95,2.472,0.729,6.92,2.28,7125.58,336.907,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1819.0,+190/-85,2.973,0.677,7.85,3.79,7125.58,38.573,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1808.0,+228/-164,2.101,0.673,8.15,3.27,2043.44,0.781,✔️,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1760.0,+101/-62,4.699,0.537,9.62,5.51,1489.94,5.79,✔️,0.0,False,CPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1755.0,+118/-66,4.174,0.571,9.77,8.06,4057.12,10.478,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1692.0,+182/-93,4.157,0.541,11.85,6.26,44.65,38.298,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1690.0,+216/-178,3.576,0.532,11.92,6.71,6.94,0.519,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1663.0,+157/-141,6.669,0.505,12.92,5.46,3811.43,7.581,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1616.0,+147/-131,5.331,0.433,14.69,10.42,4057.12,0.337,✔️,0.0,False,GPU +11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1611.0,+92/-59,7.3,0.328,14.92,12.75,686.46,5.477,✔️,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1582.0,+93/-70,7.083,0.29,16.08,12.31,2895.38,1.317,✔️,0.0,False,CPU +13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1564.0,+127/-84,6.477,0.358,16.85,13.31,708.95,1.973,✔️,0.0,False,GPU +14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1551.0,+132/-103,5.748,0.311,17.38,13.53,4228.53,1.188,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1542.0,+123/-97,7.207,0.271,17.77,9.88,2895.38,0.073,✔️,0.0,False,CPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1516.0,+104/-90,8.098,0.23,18.85,15.39,686.46,0.743,✔️,0.0,False,CPU +17,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1512.0,+232/-213,13.045,0.404,19.0,4.97,5.49,0.754,➖,0.0,False,GPU +18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1503.0,+118/-74,7.042,0.257,19.38,16.18,708.95,0.069,✔️,0.0,False,GPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1472.0,+71/-57,8.021,0.174,20.69,19.28,848.99,0.475,✔️,0.0,False,CPU +20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1472.0,+60/-49,8.036,0.173,20.69,19.7,848.99,2.378,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1460.0,+109/-92,9.327,0.202,21.23,18.6,16.07,0.29,✔️,0.0,False,GPU +22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1444.0,+134/-95,6.582,0.191,21.92,18.71,4228.53,0.125,✔️,0.0,False,GPU +23,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1434.0,+150/-147,9.198,0.259,22.38,13.93,3811.43,0.452,✔️,0.0,False,GPU +24,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1430.0,+117/-136,9.171,0.173,22.54,18.16,8.35,0.087,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1357.0,+136/-132,8.245,0.129,25.77,22.76,13.9,0.123,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1309.0,+141/-114,10.088,0.087,27.85,23.61,9.17,1.416,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1296.0,+108/-102,9.416,0.078,28.38,25.93,4452.11,0.852,✔️,0.0,False,CPU +28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1295.0,+134/-114,12.273,0.093,28.46,23.52,161.73,0.784,✔️,0.0,False,CPU +29,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1271.0,+135/-118,12.586,0.105,29.46,23.46,161.73,0.12,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1241.0,+130/-136,10.034,0.064,30.69,27.38,4452.11,0.089,✔️,0.0,False,CPU +31,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1241.0,+179/-168,12.917,0.076,30.69,14.6,2.47,0.755,✔️,0.0,False,GPU +32,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1224.0,+76/-101,10.917,0.013,31.38,30.46,2.03,0.296,✔️,0.0,False,CPU +33,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1214.0,+159/-194,13.421,0.12,31.77,21.26,3017.28,0.248,✔️,0.0,False,CPU +34,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1191.0,+97/-118,11.634,0.053,32.69,30.4,2.15,0.18,✔️,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1182.0,+83/-122,13.249,0.018,33.08,31.12,526.17,0.77,✔️,0.0,False,CPU +36,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1166.0,+155/-206,13.982,0.119,33.69,15.28,3017.28,0.027,✔️,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1138.0,+95/-134,14.003,0.016,34.73,32.85,526.17,0.119,✔️,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1119.0,+104/-116,14.183,0.014,35.46,33.28,0.42,0.061,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1084.0,+75/-108,14.282,0.0,36.69,35.72,527.21,2.831,✔️,0.0,False,CPU +40,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1039.0,+118/-167,15.082,0.049,38.15,35.84,10.04,0.03,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1037.0,+92/-138,14.125,0.0,38.23,36.79,20.5,0.081,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1034.0,+97/-114,14.768,0.0,38.31,37.3,527.21,0.311,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+73/-90,15.317,0.0,39.35,38.95,0.63,0.064,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),887.0,+94/-166,18.821,0.0,42.31,41.7,3.08,0.293,✔️,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),877.0,+134/-183,20.298,0.0,42.54,41.76,90.87,0.906,✔️,0.0,False,CPU +46,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),713.0,+109/-192,22.447,0.0,45.54,45.39,90.87,0.055,✔️,0.0,False,CPU +47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),572.0,+205/-344,29.125,0.0,47.31,46.87,0.25,0.044,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),439.0,+113/-319,36.211,0.0,48.58,48.52,177.04,0.14,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),351.0,+147/-403,36.396,0.0,49.27,49.19,177.04,0.059,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),199.0,+138/-328,38.856,0.0,50.23,50.22,0.88,0.109,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_all/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_all/winrate_matrix.png.zip index 3605d8815e1f3db9d61adea12b7ff4158061bf12..0abb271ab9d7ba43da34fae1da3a0b95a275b467 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:08be8afdd0f0e20bbeaa8eebb468ab0b46eb4440077e325ddc238397dd7422b8 +oid sha256:598843ddce77f6ae141eb8630c255d75e9b89d14e418fd31e79d4e3ad5ec8a85 size 1471479 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index c545c74b873eb874e16fc2bcbd62c1825a8bd1ab..61cbdac8d75d0cd016e9cbc7b60a48b4f55c3b49 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:192d6f20c3bba8a089a3a2c83ec318ab168ac24a87e6057400e9b97750ada931 +oid sha256:63229fad1bc6bafe6268582cc8e98ed6125dfe6157959ccd584ee7696b629589 size 430652 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip index 50b76ee5db1aed2c310523115b56412d78d347be..fabc2244a3562be0d60e00c4f6cfdd7377095f5d 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a0ed1a7b0aec7f79576fc6e35813f42a3c1c5a2eccf9907f5caa0f571a57e592 -size 984135 +oid sha256:64c603bd1b32c2e569f74c054f5f1acc2c14278c003610e5c45847a8ad8b9ab3 +size 917540 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/tuning-impact-elo.png.zip index d690b23148f0a4984f305cf26c6420ae62bd70c9..5ff55a105373740553892bce0532bcf597f48421 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cc182a07eaa944d15afc1708dd15d412fd261840ba3fa70bf62871f5f947ce52 +oid sha256:af93e2729b962c9d1a179621367a88891484681549768209c815f05db2032e9f size 119314 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/website_leaderboard.csv index dbc54b0b2b4f8c3dd4bc8feaa1c2728697a1133a..46a7e63e483c8172dd9737f1097129e0df9c843a 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/website_leaderboard.csv @@ -1,52 +1,52 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,2061.0,+1014/-71,0.832,4.4,3.24,1.163,739.1,11.65,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,2027.0,+1086/-121,0.835,5.0,2.74,1.262,739.1,0.689,0.0,False,GPU -2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1950.0,+850/-36,0.699,6.6,5.71,2.284,2552.28,4.522,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1916.0,+658/-8,0.655,7.4,3.4,3.209,474.94,11.837,0.0,False,CPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1885.0,+615/-13,0.612,8.2,6.09,3.513,400.37,2.16,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1870.0,+930/-101,0.637,8.6,6.91,2.851,2068.17,1.355,0.0,False,GPU -6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1822.0,+1355/-177,0.687,10.0,2.25,2.083,4529.88,357.209,0.0,False,GPU -7,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1790.0,+1289/-174,0.65,11.0,4.14,2.39,4529.88,39.544,0.0,False,GPU -8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1783.0,+1058/-258,0.548,11.2,6.42,2.625,1.59,0.506,0.0,False,GPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1688.0,+928/-198,0.485,14.4,10.08,4.112,2068.17,0.068,0.0,False,GPU -10,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1677.0,+780/-243,0.471,14.8,11.42,4.058,2552.28,0.201,0.0,False,GPU -11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1677.0,+503/-139,0.377,14.8,11.88,5.836,223.05,9.707,0.0,False,CPU -12,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1671.0,+1088/-218,0.526,15.0,6.19,3.203,41.5,38.298,0.0,False,GPU -13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1660.0,+548/-97,0.349,15.4,12.03,6.007,223.05,0.793,0.0,False,CPU -14,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1649.0,+524/-91,0.418,15.8,13.49,4.481,438.55,0.52,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1644.0,+510/-102,0.3,16.0,10.62,6.235,1696.82,0.538,0.0,False,CPU -16,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1617.0,+754/-226,0.336,17.0,12.43,5.187,3456.17,0.611,0.0,False,GPU -17,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1590.0,+500/-169,0.28,18.0,7.25,6.475,1696.82,0.055,0.0,False,CPU -18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1559.0,+428/-97,0.253,19.2,17.73,5.477,438.55,0.035,0.0,False,GPU -19,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1553.0,+728/-163,0.287,19.4,16.95,5.3,6.87,0.098,0.0,False,GPU -20,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1537.0,+750/-208,0.306,20.0,4.21,4.7,2.07,0.754,0.0,False,GPU -21,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1521.0,+561/-275,0.27,20.6,16.66,7.061,4.99,0.047,0.0,False,CPU -22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1505.0,+528/-187,0.183,21.2,19.63,6.188,3456.17,0.037,0.0,False,GPU -23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1494.0,+356/-58,0.189,21.6,20.89,6.853,289.35,1.798,0.0,False,CPU -24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1484.0,+386/-70,0.181,22.0,21.09,6.921,289.35,0.36,0.0,False,CPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1435.0,+648/-216,0.129,23.8,21.42,7.402,5.37,0.025,0.0,False,GPU -26,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1353.0,+354/-228,0.137,26.8,24.63,8.025,0.57,0.074,0.0,False,CPU -27,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1302.0,+357/-139,0.002,28.6,27.72,8.909,2.32,0.093,0.0,False,GPU -28,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1297.0,+375/-186,0.018,28.8,28.08,8.225,0.66,0.213,0.0,False,CPU -29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1274.0,+225/-245,0.064,29.6,27.65,8.161,4108.24,0.479,0.0,False,CPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1250.0,+508/-284,0.028,30.4,28.51,9.603,6.54,0.284,0.0,False,GPU -31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1221.0,+204/-296,0.029,31.4,29.47,9.253,4108.24,0.037,0.0,False,CPU -32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1172.0,+353/-451,0.023,33.0,31.22,10.288,35.47,0.502,0.0,False,CPU -33,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1147.0,+417/-890,0.144,33.8,25.8,13.625,1820.58,0.133,0.0,False,CPU -34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1140.0,+314/-411,0.006,34.0,32.83,10.454,35.47,0.054,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1134.0,+107/-401,0.0,34.2,33.96,10.053,79.96,0.571,0.0,False,CPU -36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1124.0,+128/-390,0.0,34.5,34.09,10.217,79.96,0.053,0.0,False,CPU -37,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1061.0,+165/-350,0.0,36.4,35.28,13.469,14.05,0.029,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1033.0,+387/-674,0.006,37.2,35.12,12.12,0.18,0.054,0.0,False,CPU -39,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1026.0,+316/-953,0.1,37.4,34.8,14.242,1820.58,0.012,0.0,False,CPU -40,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+78/-318,0.0,38.1,37.96,11.35,0.42,0.053,0.0,False,CPU -41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,996.0,+363/-1057,0.12,38.2,34.58,14.681,3.47,0.01,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,871.0,+51/-752,0.0,41.2,40.95,14.269,317.48,0.719,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,842.0,+43/-769,0.0,41.8,41.54,14.827,317.48,0.127,0.0,False,CPU -44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,800.0,+94/-889,0.0,42.6,42.42,18.666,142.18,1.027,0.0,False,CPU -45,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,662.0,+74/-902,0.0,44.8,44.73,21.083,142.18,0.057,0.0,False,CPU -46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,662.0,+107/-831,0.0,44.8,44.56,18.199,1.33,0.126,0.0,False,CPU -47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,361.0,+132/-1452,0.0,47.6,47.49,28.294,0.06,0.036,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,145.0,+92/-1782,0.0,48.8,48.8,39.482,48.35,0.054,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-119.0,+197/-2129,0.0,50.0,49.99,39.829,48.35,0.013,0.0,False,CPU -50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-276.0,+185/-2881,0.0,50.6,50.6,42.789,0.13,0.02,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),2061.0,+1014/-71,1.163,0.832,4.4,3.24,739.1,11.65,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),2027.0,+1086/-121,1.262,0.835,5.0,2.74,739.1,0.689,✔️,0.0,False,GPU +2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1950.0,+850/-36,2.284,0.699,6.6,5.71,2552.28,4.522,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1916.0,+658/-8,3.209,0.655,7.4,3.4,474.94,11.837,✔️,0.0,False,CPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1885.0,+615/-13,3.513,0.612,8.2,6.09,400.37,2.16,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1870.0,+930/-101,2.851,0.637,8.6,6.91,2068.17,1.355,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1822.0,+1355/-177,2.083,0.687,10.0,2.25,4529.88,357.209,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1790.0,+1289/-174,2.39,0.65,11.0,4.14,4529.88,39.544,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1783.0,+1058/-258,2.625,0.548,11.2,6.42,1.59,0.506,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1688.0,+928/-198,4.112,0.485,14.4,10.08,2068.17,0.068,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1677.0,+780/-243,4.058,0.471,14.8,11.42,2552.28,0.201,✔️,0.0,False,GPU +11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1677.0,+503/-139,5.836,0.377,14.8,11.88,223.05,9.707,✔️,0.0,False,CPU +12,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1671.0,+1088/-218,3.203,0.526,15.0,6.19,41.5,38.298,✔️,0.0,False,GPU +13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1660.0,+548/-97,6.007,0.349,15.4,12.03,223.05,0.793,✔️,0.0,False,CPU +14,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1649.0,+524/-91,4.481,0.418,15.8,13.49,438.55,0.52,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1644.0,+510/-102,6.235,0.3,16.0,10.62,1696.82,0.538,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1617.0,+754/-226,5.187,0.336,17.0,12.43,3456.17,0.611,✔️,0.0,False,GPU +17,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1590.0,+500/-169,6.475,0.28,18.0,7.25,1696.82,0.055,✔️,0.0,False,CPU +18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1559.0,+428/-97,5.477,0.253,19.2,17.73,438.55,0.035,✔️,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1553.0,+728/-163,5.3,0.287,19.4,16.95,6.87,0.098,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1537.0,+750/-208,4.7,0.306,20.0,4.21,2.07,0.754,➖,0.0,False,GPU +21,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1521.0,+561/-275,7.061,0.27,20.6,16.66,4.99,0.047,✔️,0.0,False,CPU +22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1505.0,+528/-187,6.188,0.183,21.2,19.63,3456.17,0.037,✔️,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1494.0,+356/-58,6.853,0.189,21.6,20.89,289.35,1.798,✔️,0.0,False,CPU +24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1484.0,+386/-70,6.921,0.181,22.0,21.09,289.35,0.36,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1435.0,+648/-216,7.402,0.129,23.8,21.42,5.37,0.025,✔️,0.0,False,GPU +26,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1353.0,+354/-228,8.025,0.137,26.8,24.63,0.57,0.074,✔️,0.0,False,CPU +27,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1302.0,+357/-139,8.909,0.002,28.6,27.72,2.32,0.093,✔️,0.0,False,GPU +28,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1297.0,+375/-186,8.225,0.018,28.8,28.08,0.66,0.213,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1274.0,+225/-245,8.161,0.064,29.6,27.65,4108.24,0.479,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1250.0,+508/-284,9.603,0.028,30.4,28.51,6.54,0.284,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1221.0,+204/-296,9.253,0.029,31.4,29.47,4108.24,0.037,✔️,0.0,False,CPU +32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1172.0,+353/-451,10.288,0.023,33.0,31.22,35.47,0.502,✔️,0.0,False,CPU +33,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1147.0,+417/-890,13.625,0.144,33.8,25.8,1820.58,0.133,✔️,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1140.0,+314/-411,10.454,0.006,34.0,32.83,35.47,0.054,✔️,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1134.0,+107/-401,10.053,0.0,34.2,33.96,79.96,0.571,✔️,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1124.0,+128/-390,10.217,0.0,34.5,34.09,79.96,0.053,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1061.0,+165/-350,13.469,0.0,36.4,35.28,14.05,0.029,✔️,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1033.0,+387/-674,12.12,0.006,37.2,35.12,0.18,0.054,✔️,0.0,False,CPU +39,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1026.0,+316/-953,14.242,0.1,37.4,34.8,1820.58,0.012,✔️,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+78/-318,11.35,0.0,38.1,37.96,0.42,0.053,✔️,0.0,False,CPU +41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),996.0,+363/-1057,14.681,0.12,38.2,34.58,3.47,0.01,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),871.0,+51/-752,14.269,0.0,41.2,40.95,317.48,0.719,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),842.0,+43/-769,14.827,0.0,41.8,41.54,317.48,0.127,✔️,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),800.0,+94/-889,18.666,0.0,42.6,42.42,142.18,1.027,✔️,0.0,False,CPU +45,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),662.0,+74/-902,21.083,0.0,44.8,44.73,142.18,0.057,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),662.0,+107/-831,18.199,0.0,44.8,44.56,1.33,0.126,✔️,0.0,False,CPU +47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),361.0,+132/-1452,28.294,0.0,47.6,47.49,0.06,0.036,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),145.0,+92/-1782,39.482,0.0,48.8,48.8,48.35,0.054,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),-119.0,+197/-2129,39.829,0.0,50.0,49.99,48.35,0.013,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),-276.0,+185/-2881,42.789,0.0,50.6,50.6,0.13,0.02,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/winrate_matrix.png.zip index e4459272d5fce82cd0bb6ce910b37184b692a967..b8231e7b9275afd25fb88d64a28b618780991445 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:12f84f0e255219eee537db0bbdfacfbc33d4ea9a62c5bdbd4fb9ae959e328421 +oid sha256:ba0514e7743c7311010aa6fec04e9d1b34a7d7154407f7b7426e411ef6a8997a size 1244143 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 392f7e5eede4ec3a476d1160ddaa4cc4feb524d2..47d852ab8d1d8ae7a9407552208074dbffe7a97d 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:afe4cf82d363d970a15d4718bb0f6eff0aba9ae74c699faa13cd6c4c8eb0b2a9 +oid sha256:bd0ee128f6b06efeaa559396818531af3b65b31b97991095ecaf88916cfe3631 size 423977 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip index 119b0ed1096e9deb67f08d09d959a1351202434d..621e4db931809c245018e8d83905782d02c14aa4 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7d6d6266d85db43f2f511fa3d2082e6336a9458060c1c3c45dab013c0774083a -size 992662 +oid sha256:7740a2d9aa2c96259cfec19b7fcb846aca999e815c9b3ca168ad332318f22aec +size 987556 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_small/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_small/tuning-impact-elo.png.zip index 4c22de815c91feebb6293253ba476d33c81d8319..97bd3e8b02f71b53dc1b7fa8762ce63099d2dbcd 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cb9ccab10fe6092a3ab6687834dbe6002d793df2b0a8ab4cf8208746f2ca3165 +oid sha256:873e422985707e325535b349258bf21efa0f50f79b57eb29f98dff677fd2122a size 118758 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_small/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_regression/datasets_small/website_leaderboard.csv index c7312197bd2c110c2c16f513bce785df8b4b7818..fc1524f33ce240dba0cf26c627b509cb6c741e77 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_small/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_small/website_leaderboard.csv @@ -1,52 +1,52 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1952.0,+273/-132,0.755,5.0,2.3,2.715,9830.47,232.266,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1921.0,+505/-228,0.779,5.62,2.08,1.16,2355.62,7.55,0.0,False,GPU -2,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1910.0,+208/-53,0.694,5.88,3.59,3.338,9830.47,22.221,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1904.0,+318/-109,0.67,6.0,3.22,1.968,716.17,4.181,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1759.0,+168/-71,0.55,9.88,6.3,4.753,51.27,43.085,0.0,False,GPU -5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1751.0,+359/-200,0.571,10.12,3.72,2.625,2355.62,0.879,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1740.0,+162/-78,0.529,10.5,9.0,5.0,4918.07,35.607,0.0,False,GPU -7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1725.0,+173/-65,0.463,11.0,8.97,5.631,4919.72,5.372,0.0,False,CPU -8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1686.0,+347/-216,0.521,12.38,6.9,4.17,8.24,0.532,0.0,False,GPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1619.0,+208/-123,0.4,14.88,10.65,6.092,4918.07,0.949,0.0,False,GPU -10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1616.0,+120/-74,0.297,15.0,13.37,8.215,1206.7,4.573,0.0,False,CPU -11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1588.0,+151/-78,0.285,16.12,13.67,7.613,3495.99,1.462,0.0,False,CPU -12,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1569.0,+199/-136,0.384,16.88,5.32,9.41,9006.35,8.016,0.0,False,GPU -13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1554.0,+166/-105,0.32,17.5,13.2,7.724,1672.82,2.447,0.0,False,GPU -14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1551.0,+231/-103,0.265,17.62,12.79,7.664,3495.99,0.148,0.0,False,CPU -15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1551.0,+184/-110,0.295,17.62,14.33,6.098,4322.16,1.598,0.0,False,GPU -16,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1534.0,+445/-282,0.466,18.38,5.6,18.26,5.86,0.71,0.0,False,GPU -17,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1509.0,+143/-86,0.259,19.5,15.34,8.02,1672.82,0.097,0.0,False,GPU -18,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1500.0,+108/-62,0.169,19.88,18.29,8.708,1277.27,0.648,0.0,False,CPU -19,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1495.0,+102/-52,0.163,20.12,19.02,8.776,1277.27,3.746,0.0,False,CPU -20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1476.0,+129/-93,0.156,21.0,18.63,9.405,1206.7,0.689,0.0,False,CPU -21,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1446.0,+144/-126,0.149,22.38,19.81,11.844,25.13,0.294,0.0,False,GPU -22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1446.0,+182/-115,0.196,22.38,18.18,6.829,4322.16,0.198,0.0,False,GPU -23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1415.0,+158/-132,0.112,23.75,19.24,10.49,10.03,0.115,0.0,False,CPU -24,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1372.0,+166/-175,0.136,25.62,20.38,13.514,644.11,0.953,0.0,False,CPU -25,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1358.0,+178/-127,0.124,26.25,21.31,10.391,22.41,7.82,0.0,False,GPU -26,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1350.0,+189/-198,0.167,26.62,19.91,13.918,644.11,0.187,0.0,False,CPU -27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1342.0,+194/-144,0.13,27.0,23.69,8.773,25.53,0.154,0.0,False,GPU -28,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1339.0,+195/-178,0.127,27.12,16.15,12.41,9006.35,0.527,0.0,False,GPU -29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1328.0,+144/-144,0.087,27.62,24.95,10.201,5194.73,1.75,0.0,False,CPU -30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1269.0,+207/-194,0.086,30.25,26.22,10.523,5194.73,0.121,0.0,False,CPU -31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1263.0,+232/-223,0.105,30.5,19.16,13.293,3991.28,0.402,0.0,False,CPU -32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1243.0,+248/-243,0.131,31.38,11.32,13.82,3991.28,0.034,0.0,False,CPU -33,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1228.0,+219/-289,0.121,32.0,11.27,15.422,7.0,1.362,0.0,False,GPU -34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1219.0,+112/-184,0.029,32.38,29.57,15.247,657.99,1.135,0.0,False,CPU -35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1204.0,+91/-98,0.009,33.0,32.16,12.599,4.54,0.374,0.0,False,CPU -36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1182.0,+69/-127,0.0,33.88,33.07,14.291,2150.84,7.995,0.0,False,CPU -37,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1169.0,+101/-179,0.019,34.38,32.23,15.473,0.46,0.117,0.0,False,CPU -38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1155.0,+114/-244,0.027,34.88,32.12,16.37,657.99,0.147,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1121.0,+95/-151,0.0,36.12,35.07,14.731,2150.84,0.566,0.0,False,CPU -40,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1114.0,+92/-103,0.0,36.38,35.62,13.89,4.08,0.255,0.0,False,CPU -41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1064.0,+125/-209,0.005,38.12,36.68,15.332,12.71,0.039,0.0,False,CPU -42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1025.0,+148/-218,0.0,39.38,37.8,14.535,22.25,0.131,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+88/-203,0.0,40.12,39.6,17.797,0.69,0.125,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,978.0,+89/-211,0.0,40.75,40.09,19.209,6.77,0.511,0.0,False,CPU -45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,909.0,+148/-323,0.0,42.5,41.35,21.318,88.79,0.572,0.0,False,CPU -46,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,724.0,+133/-310,0.0,46.0,45.81,23.299,88.79,0.052,0.0,False,CPU -47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,642.0,+210/-603,0.0,47.12,46.49,29.645,0.43,0.047,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,525.0,+94/-492,0.0,48.44,48.34,34.167,742.99,0.364,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,485.0,+125/-572,0.0,48.81,48.71,34.251,742.99,0.097,0.0,False,CPU -50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,334.0,+100/-501,0.0,50.0,49.99,36.397,3.98,0.12,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1952.0,+273/-132,2.715,0.755,5.0,2.3,9830.47,232.266,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1921.0,+505/-228,1.16,0.779,5.62,2.08,2355.62,7.55,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1910.0,+208/-53,3.338,0.694,5.88,3.59,9830.47,22.221,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1904.0,+318/-109,1.968,0.67,6.0,3.22,716.17,4.181,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1759.0,+168/-71,4.753,0.55,9.88,6.3,51.27,43.085,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1751.0,+359/-200,2.625,0.571,10.12,3.72,2355.62,0.879,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1740.0,+162/-78,5.0,0.529,10.5,9.0,4918.07,35.607,✔️,0.0,False,GPU +7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1725.0,+173/-65,5.631,0.463,11.0,8.97,4919.72,5.372,✔️,0.0,False,CPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1686.0,+347/-216,4.17,0.521,12.38,6.9,8.24,0.532,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1619.0,+208/-123,6.092,0.4,14.88,10.65,4918.07,0.949,✔️,0.0,False,GPU +10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1616.0,+120/-74,8.215,0.297,15.0,13.37,1206.7,4.573,✔️,0.0,False,CPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1588.0,+151/-78,7.613,0.285,16.12,13.67,3495.99,1.462,✔️,0.0,False,CPU +12,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1569.0,+199/-136,9.41,0.384,16.88,5.32,9006.35,8.016,✔️,0.0,False,GPU +13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1554.0,+166/-105,7.724,0.32,17.5,13.2,1672.82,2.447,✔️,0.0,False,GPU +14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1551.0,+231/-103,7.664,0.265,17.62,12.79,3495.99,0.148,✔️,0.0,False,CPU +15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1551.0,+184/-110,6.098,0.295,17.62,14.33,4322.16,1.598,✔️,0.0,False,GPU +16,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1534.0,+445/-282,18.26,0.466,18.38,5.6,5.86,0.71,➖,0.0,False,GPU +17,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1509.0,+143/-86,8.02,0.259,19.5,15.34,1672.82,0.097,✔️,0.0,False,GPU +18,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1500.0,+108/-62,8.708,0.169,19.88,18.29,1277.27,0.648,✔️,0.0,False,CPU +19,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1495.0,+102/-52,8.776,0.163,20.12,19.02,1277.27,3.746,✔️,0.0,False,CPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1476.0,+129/-93,9.405,0.156,21.0,18.63,1206.7,0.689,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1446.0,+144/-126,11.844,0.149,22.38,19.81,25.13,0.294,✔️,0.0,False,GPU +22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1446.0,+182/-115,6.829,0.196,22.38,18.18,4322.16,0.198,✔️,0.0,False,GPU +23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1415.0,+158/-132,10.49,0.112,23.75,19.24,10.03,0.115,✔️,0.0,False,CPU +24,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1372.0,+166/-175,13.514,0.136,25.62,20.38,644.11,0.953,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1358.0,+178/-127,10.391,0.124,26.25,21.31,22.41,7.82,✔️,0.0,False,GPU +26,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1350.0,+189/-198,13.918,0.167,26.62,19.91,644.11,0.187,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1342.0,+194/-144,8.773,0.13,27.0,23.69,25.53,0.154,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1339.0,+195/-178,12.41,0.127,27.12,16.15,9006.35,0.527,✔️,0.0,False,GPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1328.0,+144/-144,10.201,0.087,27.62,24.95,5194.73,1.75,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1269.0,+207/-194,10.523,0.086,30.25,26.22,5194.73,0.121,✔️,0.0,False,CPU +31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1263.0,+232/-223,13.293,0.105,30.5,19.16,3991.28,0.402,✔️,0.0,False,CPU +32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1243.0,+248/-243,13.82,0.131,31.38,11.32,3991.28,0.034,✔️,0.0,False,CPU +33,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1228.0,+219/-289,15.422,0.121,32.0,11.27,7.0,1.362,✔️,0.0,False,GPU +34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1219.0,+112/-184,15.247,0.029,32.38,29.57,657.99,1.135,✔️,0.0,False,CPU +35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1204.0,+91/-98,12.599,0.009,33.0,32.16,4.54,0.374,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1182.0,+69/-127,14.291,0.0,33.88,33.07,2150.84,7.995,✔️,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1169.0,+101/-179,15.473,0.019,34.38,32.23,0.46,0.117,✔️,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1155.0,+114/-244,16.37,0.027,34.88,32.12,657.99,0.147,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1121.0,+95/-151,14.731,0.0,36.12,35.07,2150.84,0.566,✔️,0.0,False,CPU +40,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1114.0,+92/-103,13.89,0.0,36.38,35.62,4.08,0.255,✔️,0.0,False,CPU +41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1064.0,+125/-209,15.332,0.005,38.12,36.68,12.71,0.039,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1025.0,+148/-218,14.535,0.0,39.38,37.8,22.25,0.131,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+88/-203,17.797,0.0,40.12,39.6,0.69,0.125,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),978.0,+89/-211,19.209,0.0,40.75,40.09,6.77,0.511,✔️,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),909.0,+148/-323,21.318,0.0,42.5,41.35,88.79,0.572,✔️,0.0,False,CPU +46,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),724.0,+133/-310,23.299,0.0,46.0,45.81,88.79,0.052,✔️,0.0,False,CPU +47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),642.0,+210/-603,29.645,0.0,47.12,46.49,0.43,0.047,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),525.0,+94/-492,34.167,0.0,48.44,48.34,742.99,0.364,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),485.0,+125/-572,34.251,0.0,48.81,48.71,742.99,0.097,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),334.0,+100/-501,36.397,0.0,50.0,49.99,3.98,0.12,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_small/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_small/winrate_matrix.png.zip index c000af8ee0fa595c822a87dd5b914f44ac50e4c1..1410ef502feefb27094e9ff11e77211a0336bf57 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:760a144dc636ad9cbedacadb367fc542d053b2d3aabf5db118c774165a9d7d46 +oid sha256:f826ab381f065d3248028b0bbc3552b60f5a3bf71f7c92f76362bfc782c5db42 size 1315140 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 658b4d07378d5e822279a6541d3551fa7773f26e..5ad182eeb9a05fa150081417b4ac7eb07afe2e02 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:76f2750091c95fe15aa2a30e669a674d3b1d75c52e0f3398e4f07c0f900e083b +oid sha256:8c4aec408b169814300b4c22cc8e56d15b4f3aa39ed1e28467b0b3767b055186 size 451328 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip index 6b3aa5e3350e472ba0452b48be92f3bbf19ace3b..a5ded5f8320c08bcdd63b908c95e1140690f16d3 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9eae04c352a81993c5962bf6f2f5ab3cdc6b8c4a9dad5387f7ecae7050753c4f -size 1044019 +oid sha256:0607851f50acc1a266062ff0463e747f9b7cfa0c714039908a98e1b563a844bc +size 1042022 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip index b1d2618be1823f634605470d59c87d2937ef924d..e00559d02df73399558fb66d087e66ee329ac937 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5fdc6da47a0c36cda8163cf28974b2b607495d7413170b6bc7ed042598a0f372 +oid sha256:31036bfbeb65c3cd11f53cce5994d4e1400b5e4b2f5fa02bbbd3b31becb1aa77 size 120889 diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/website_leaderboard.csv b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/website_leaderboard.csv index a69a092ddc81da31abce4a88c97bc436b3de2008..601be121811987b8fb57ec8c68320a67aca39762 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/website_leaderboard.csv @@ -1,56 +1,56 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1906.0,+304/-113,0.726,6.43,2.55,3.095,10054.02,249.652,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1900.0,+513/-229,0.772,6.57,2.05,1.229,2279.61,6.732,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1884.0,+400/-86,0.683,7.0,3.48,1.908,669.82,4.041,0.0,False,GPU -3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1873.0,+180/-50,0.671,7.29,4.98,3.69,10054.02,28.583,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1780.0,+499/-189,0.597,10.14,3.32,2.584,6601.21,73.561,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1772.0,+224/-44,0.565,10.43,6.19,4.944,47.81,37.998,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1756.0,+146/-30,0.551,11.0,9.95,5.167,4840.44,33.436,0.0,False,GPU -7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1729.0,+403/-218,0.545,12.0,3.79,2.847,2279.61,0.781,0.0,False,GPU -8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1714.0,+373/-213,0.554,12.57,7.37,4.191,7.82,0.519,0.0,False,GPU -9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1693.0,+192/-49,0.444,13.43,10.94,6.056,5390.56,5.054,0.0,False,CPU -10,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1658.0,+220/-118,0.398,14.86,7.71,6.637,105.28,2.433,0.0,False,GPU -11,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1624.0,+410/-299,0.518,16.29,5.15,19.429,5.73,0.638,0.0,False,GPU -12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1605.0,+159/-48,0.287,17.14,15.34,7.956,3301.07,1.317,0.0,False,CPU -13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1605.0,+257/-127,0.39,17.14,12.18,6.481,4840.44,0.706,0.0,False,GPU -14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1574.0,+114/-57,0.246,18.57,17.2,9.035,997.12,3.787,0.0,False,CPU -15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1568.0,+226/-106,0.266,18.86,14.48,8.008,3301.07,0.125,0.0,False,CPU -16,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1558.0,+418/-227,0.406,19.29,10.76,5.384,6601.21,0.588,0.0,False,GPU -17,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1544.0,+165/-105,0.295,20.0,14.25,8.299,1709.03,2.724,0.0,False,GPU -18,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1529.0,+188/-118,0.267,20.71,17.7,6.513,4228.53,1.447,0.0,False,GPU -19,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1515.0,+180/-138,0.267,21.43,15.3,10.909,9360.97,7.658,0.0,False,GPU -20,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1512.0,+142/-77,0.251,21.57,16.24,8.482,1709.03,0.097,0.0,False,GPU -21,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1469.0,+405/-336,0.352,23.71,13.17,7.531,8.97,0.837,0.0,False,GPU -22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1466.0,+83/-44,0.098,23.86,23.13,9.543,911.22,0.527,0.0,False,CPU -23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1463.0,+78/-38,0.1,24.0,23.39,9.588,911.22,3.532,0.0,False,CPU -24,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1434.0,+126/-68,0.105,25.43,23.65,10.356,997.12,0.636,0.0,False,CPU -25,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1425.0,+196/-137,0.16,25.86,21.99,7.312,4228.53,0.173,0.0,False,GPU -26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1422.0,+220/-139,0.117,26.0,21.17,11.174,9.39,0.105,0.0,False,CPU -27,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1407.0,+139/-152,0.121,26.71,24.25,13.106,27.26,0.29,0.0,False,GPU -28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1392.0,+209/-236,0.158,27.43,21.57,14.555,613.95,0.952,0.0,False,CPU -29,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1384.0,+236/-250,0.193,27.86,20.58,14.881,613.95,0.135,0.0,False,CPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1366.0,+155/-135,0.129,28.71,23.97,11.015,22.96,8.006,0.0,False,GPU -31,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1325.0,+195/-187,0.114,30.71,27.44,9.406,17.97,0.145,0.0,False,GPU -32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1290.0,+275/-208,0.111,32.43,20.76,14.054,3778.28,0.323,0.0,False,CPU -33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1282.0,+124/-160,0.026,32.86,31.52,11.205,5937.36,1.406,0.0,False,CPU -34,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1276.0,+260/-286,0.138,33.14,10.38,16.185,7.12,1.138,0.0,False,GPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1272.0,+275/-228,0.14,33.29,14.11,14.636,3778.28,0.028,0.0,False,CPU -36,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1272.0,+146/-174,0.037,33.29,30.98,14.048,9360.97,0.522,0.0,False,GPU -37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1229.0,+155/-260,0.03,35.29,32.39,16.471,628.51,0.812,0.0,False,CPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1222.0,+185/-280,0.031,35.57,32.35,11.535,5937.36,0.119,0.0,False,CPU -39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1175.0,+138/-220,0.02,37.57,35.46,16.642,0.43,0.117,0.0,False,CPU -40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1173.0,+176/-289,0.029,37.64,34.8,17.646,628.51,0.149,0.0,False,CPU -41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1172.0,+59/-100,0.0,37.71,37.36,13.734,4.41,0.363,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1150.0,+55/-106,0.0,38.57,38.08,15.611,2163.39,7.622,0.0,False,CPU -43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1114.0,+109/-169,0.0,40.0,39.21,14.895,3.97,0.24,0.0,False,CPU -44,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1095.0,+89/-158,0.0,40.71,39.47,15.956,10.54,0.036,0.0,False,CPU -45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1095.0,+106/-175,0.0,40.71,39.78,16.062,2163.39,0.713,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1002.0,+135/-301,0.0,43.86,42.36,15.699,20.5,0.13,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+131/-217,0.0,43.93,43.18,19.276,0.67,0.118,0.0,False,CPU -48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,943.0,+77/-213,0.0,45.57,45.12,21.065,6.88,0.452,0.0,False,CPU -49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,937.0,+178/-381,0.0,45.71,44.31,22.177,86.71,0.53,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,757.0,+168/-358,0.0,49.57,49.27,24.152,86.71,0.05,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,712.0,+245/-524,0.0,50.29,49.48,30.287,0.4,0.044,0.0,False,CPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,533.0,+85/-893,0.0,52.5,52.4,36.2,753.48,0.227,0.0,False,CPU -53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,504.0,+117/-984,0.0,52.79,52.68,36.249,753.48,0.091,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,352.0,+97/-674,0.0,54.0,53.99,37.99,4.35,0.119,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1906.0,+304/-113,3.095,0.726,6.43,2.55,10054.02,249.652,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1900.0,+513/-229,1.229,0.772,6.57,2.05,2279.61,6.732,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1884.0,+400/-86,1.908,0.683,7.0,3.48,669.82,4.041,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1873.0,+180/-50,3.69,0.671,7.29,4.98,10054.02,28.583,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1780.0,+499/-189,2.584,0.597,10.14,3.32,6601.21,73.561,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1772.0,+224/-44,4.944,0.565,10.43,6.19,47.81,37.998,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1756.0,+146/-30,5.167,0.551,11.0,9.95,4840.44,33.436,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1729.0,+403/-218,2.847,0.545,12.0,3.79,2279.61,0.781,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1714.0,+373/-213,4.191,0.554,12.57,7.37,7.82,0.519,✔️,0.0,False,GPU +9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1693.0,+192/-49,6.056,0.444,13.43,10.94,5390.56,5.054,✔️,0.0,False,CPU +10,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1658.0,+220/-118,6.637,0.398,14.86,7.71,105.28,2.433,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1624.0,+410/-299,19.429,0.518,16.29,5.15,5.73,0.638,➖,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1605.0,+159/-48,7.956,0.287,17.14,15.34,3301.07,1.317,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1605.0,+257/-127,6.481,0.39,17.14,12.18,4840.44,0.706,✔️,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1574.0,+114/-57,9.035,0.246,18.57,17.2,997.12,3.787,✔️,0.0,False,CPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1568.0,+226/-106,8.008,0.266,18.86,14.48,3301.07,0.125,✔️,0.0,False,CPU +16,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1558.0,+418/-227,5.384,0.406,19.29,10.76,6601.21,0.588,✔️,0.0,False,GPU +17,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1544.0,+165/-105,8.299,0.295,20.0,14.25,1709.03,2.724,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1529.0,+188/-118,6.513,0.267,20.71,17.7,4228.53,1.447,✔️,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1515.0,+180/-138,10.909,0.267,21.43,15.3,9360.97,7.658,✔️,0.0,False,GPU +20,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1512.0,+142/-77,8.482,0.251,21.57,16.24,1709.03,0.097,✔️,0.0,False,GPU +21,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1469.0,+405/-336,7.531,0.352,23.71,13.17,8.97,0.837,✔️,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1466.0,+83/-44,9.543,0.098,23.86,23.13,911.22,0.527,✔️,0.0,False,CPU +23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1463.0,+78/-38,9.588,0.1,24.0,23.39,911.22,3.532,✔️,0.0,False,CPU +24,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1434.0,+126/-68,10.356,0.105,25.43,23.65,997.12,0.636,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1425.0,+196/-137,7.312,0.16,25.86,21.99,4228.53,0.173,✔️,0.0,False,GPU +26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1422.0,+220/-139,11.174,0.117,26.0,21.17,9.39,0.105,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1407.0,+139/-152,13.106,0.121,26.71,24.25,27.26,0.29,✔️,0.0,False,GPU +28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1392.0,+209/-236,14.555,0.158,27.43,21.57,613.95,0.952,✔️,0.0,False,CPU +29,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1384.0,+236/-250,14.881,0.193,27.86,20.58,613.95,0.135,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1366.0,+155/-135,11.015,0.129,28.71,23.97,22.96,8.006,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1325.0,+195/-187,9.406,0.114,30.71,27.44,17.97,0.145,✔️,0.0,False,GPU +32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1290.0,+275/-208,14.054,0.111,32.43,20.76,3778.28,0.323,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1282.0,+124/-160,11.205,0.026,32.86,31.52,5937.36,1.406,✔️,0.0,False,CPU +34,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1276.0,+260/-286,16.185,0.138,33.14,10.38,7.12,1.138,✔️,0.0,False,GPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1272.0,+275/-228,14.636,0.14,33.29,14.11,3778.28,0.028,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1272.0,+146/-174,14.048,0.037,33.29,30.98,9360.97,0.522,✔️,0.0,False,GPU +37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1229.0,+155/-260,16.471,0.03,35.29,32.39,628.51,0.812,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1222.0,+185/-280,11.535,0.031,35.57,32.35,5937.36,0.119,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1175.0,+138/-220,16.642,0.02,37.57,35.46,0.43,0.117,✔️,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1173.0,+176/-289,17.646,0.029,37.64,34.8,628.51,0.149,✔️,0.0,False,CPU +41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1172.0,+59/-100,13.734,0.0,37.71,37.36,4.41,0.363,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1150.0,+55/-106,15.611,0.0,38.57,38.08,2163.39,7.622,✔️,0.0,False,CPU +43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1114.0,+109/-169,14.895,0.0,40.0,39.21,3.97,0.24,✔️,0.0,False,CPU +44,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1095.0,+89/-158,15.956,0.0,40.71,39.47,10.54,0.036,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1095.0,+106/-175,16.062,0.0,40.71,39.78,2163.39,0.713,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1002.0,+135/-301,15.699,0.0,43.86,42.36,20.5,0.13,✔️,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+131/-217,19.276,0.0,43.93,43.18,0.67,0.118,✔️,0.0,False,CPU +48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),943.0,+77/-213,21.065,0.0,45.57,45.12,6.88,0.452,✔️,0.0,False,CPU +49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),937.0,+178/-381,22.177,0.0,45.71,44.31,86.71,0.53,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),757.0,+168/-358,24.152,0.0,49.57,49.27,86.71,0.05,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),712.0,+245/-524,30.287,0.0,50.29,49.48,0.4,0.044,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),533.0,+85/-893,36.2,0.0,52.5,52.4,753.48,0.227,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),504.0,+117/-984,36.249,0.0,52.79,52.68,753.48,0.091,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),352.0,+97/-674,37.99,0.0,54.0,53.99,4.35,0.119,✔️,0.0,False,CPU diff --git a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip index 335f4191942278d85ffd727dfbdc8053305ace32..76812a3f8fcb0d889d0a241484d2848ca61baa76 100644 --- a/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_no/splits_lite/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2b9c1483354240a504c08272434a27f08f7f0b455819c9efd8cb70f781c81736 +oid sha256:a856b5adb7cb96846edc8c57da779beb279e2fa34af2caa3ebc687d8e33f0f98 size 1396925 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index 4d93e73eb79a44adc38730e690a696d5c9c3c713..f705ae9b51d59fd19bfe4a382b3898f3b120ceab 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:982bafdb0a62d7e0fc6a59d5ba4026375aaad014b56e5c39cae1285276f16fb3 +oid sha256:fa4f1a079bb2f2d0d3d8caf839b427701975844b55b41d11bf3932389880707e size 496056 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_all/pareto_n_configs_imp.png.zip index a95fa2732be5193eef2338f814be87c4c9a0cf7d..37a7682dd0fcf096a309a796ca41d538689b0453 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e6b35e58032a829038a7005caace50d3d0481328787ae5b37d20e3d5ef993660 -size 1083898 +oid sha256:3af29ee92648265e44296ccd3884f79e0d994bf6d8218bb80b183ac19610586e +size 1071506 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_all/tuning-impact-elo.png.zip index 880f47d6d3445bdd99a64035889ad0a40c7d6fd5..da19d9f921a2af8c53c7ecf269ec0492c8041a34 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6643ee5c7740c444bfd741f033411ba831e09ea49b60c8f84dbb2f43ce36ae7f +oid sha256:89d2d49b05e0c90ecf46b2eab93457b446865302c70503e7e4c1648beb725fe4 size 177418 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_all/datasets_all/website_leaderboard.csv index f159f35ad3210aaa1f9d4e8a6baf0c56b642d441..133eba95641b4653bc2ec2a9002114e2d00af0b3 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_all/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_all/datasets_all/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1636.0,+60/-48,0.753,8.48,3.84,5.839,582.21,6.116,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1599.0,+78/-57,0.716,9.88,3.37,5.088,2040.22,8.92,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1554.0,+67/-50,0.658,11.83,4.65,6.062,2040.22,1.218,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1523.0,+66/-46,0.628,13.31,6.19,6.766,5.81,0.645,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1499.0,+56/-46,0.522,14.53,8.14,9.055,2950.72,11.988,0.0,False,GPU -5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1499.0,+55/-47,0.517,14.56,5.99,9.005,1735.72,2.559,0.0,False,CPU -6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1451.0,+62/-49,0.496,17.21,5.14,9.138,4907.64,286.652,0.0,False,GPU -7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1435.0,+50/-38,0.399,18.15,9.57,10.535,3285.87,1.472,0.0,False,GPU -8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1419.0,+32/-29,0.35,19.13,12.78,11.754,416.98,2.639,0.0,False,CPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1416.0,+47/-46,0.365,19.32,10.24,10.535,2950.72,0.663,0.0,False,GPU -10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1403.0,+42/-39,0.334,20.12,11.5,11.264,1658.41,0.653,0.0,False,CPU -11,🧠🔁,Neural Network,[LimiX (default) [11.98% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1402.0,+67/-55,0.464,20.2,6.37,12.97,3.68,0.598,11.98,True,GPU -12,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1396.0,+57/-53,0.402,20.54,7.13,10.533,4907.64,39.96,0.0,False,GPU -13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1392.0,+40/-41,0.324,20.8,10.82,11.447,1658.41,0.081,0.0,False,CPU -14,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1382.0,+75/-52,0.371,21.43,8.49,11.746,4621.67,8.149,0.0,False,GPU -15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1379.0,+48/-44,0.316,21.65,11.04,11.448,3285.87,0.173,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1374.0,+31/-29,0.265,21.92,15.54,12.443,416.98,0.334,0.0,False,CPU -17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1364.0,+33/-38,0.28,22.57,14.36,12.518,693.49,1.69,0.0,False,CPU -18,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1354.0,+38/-43,0.29,23.24,12.84,12.323,6.83,0.08,0.0,False,CPU -19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1348.0,+40/-37,0.245,23.61,12.01,12.287,4621.67,0.475,0.0,False,GPU -20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1343.0,+51/-39,0.253,23.97,14.02,12.886,846.88,2.552,0.0,False,GPU -21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1342.0,+32/-34,0.23,24.04,16.93,12.838,693.49,0.308,0.0,False,CPU -22,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1338.0,+65/-65,0.382,24.27,8.06,12.658,3030.15,21.444,35.29,True,GPU -23,🧠🔁,Neural Network,[Mitra (default) [35.29% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1326.0,+62/-62,0.36,25.11,8.78,13.408,87.65,2.505,35.29,True,GPU -24,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1320.0,+68/-55,0.326,25.48,8.32,13.232,47.62,43.739,0.0,False,GPU -25,🧠⚡,Foundation Model,[TabICL (default) [29.41% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1317.0,+51/-57,0.318,25.66,9.42,12.932,6.63,1.48,29.41,True,GPU -26,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1295.0,+42/-37,0.188,27.14,16.97,14.176,846.88,0.13,0.0,False,GPU -27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1287.0,+47/-44,0.219,27.69,16.45,13.952,10.49,0.132,0.0,False,GPU -28,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1281.0,+62/-58,0.285,28.06,9.86,14.316,3030.15,0.462,35.29,True,GPU -29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1280.0,+47/-45,0.153,28.17,19.25,13.735,2874.67,1.952,0.0,False,CPU -30,🧠⚡,Foundation Model,[BetaTabPFN (default) [25.49% IMPUTED]](https://arxiv.org/abs/2502.02527),➖,1274.0,+56/-50,0.232,28.59,15.28,15.319,205.87,1.343,25.49,True,GPU -31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1259.0,+40/-38,0.149,29.58,19.57,15.485,2929.85,0.422,0.0,False,CPU -32,🧠⚡,Foundation Model,[TabPFNv2 (default) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1254.0,+66/-61,0.259,29.86,13.15,15.267,3.36,0.313,35.29,True,GPU -33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1246.0,+39/-40,0.122,30.46,19.1,15.853,14.87,0.308,0.0,False,GPU -34,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1226.0,+35/-38,0.082,31.77,22.77,15.245,10.06,1.689,0.0,False,GPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1223.0,+44/-45,0.106,31.95,20.88,16.161,2929.85,0.051,0.0,False,CPU -36,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1210.0,+38/-40,0.091,32.81,24.48,15.708,1.94,0.123,0.0,False,CPU -37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1205.0,+43/-44,0.08,33.15,23.23,15.593,2874.67,0.131,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1205.0,+42/-41,0.094,33.16,22.18,17.165,183.02,0.761,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1202.0,+48/-50,0.12,33.37,20.04,16.636,593.24,4.467,0.0,False,CPU -40,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1192.0,+44/-47,0.097,34.0,22.56,17.32,7.32,0.05,0.0,False,CPU -41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1183.0,+31/-30,0.058,34.57,29.77,16.331,1.96,0.142,0.0,False,CPU -42,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1177.0,+50/-49,0.074,34.94,22.57,18.028,373.18,0.771,0.0,False,CPU -43,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1168.0,+43/-46,0.07,35.51,22.57,18.131,183.02,0.091,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1141.0,+48/-52,0.05,37.19,24.44,18.128,593.24,0.306,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1138.0,+49/-40,0.049,37.41,22.15,18.798,373.18,0.085,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1072.0,+36/-46,0.009,41.25,33.48,19.966,9.99,0.126,0.0,False,CPU -47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1039.0,+59/-64,0.034,42.99,34.35,23.528,3.23,0.919,0.0,False,GPU -48,🧠⚡,Foundation Model,[TabFlex (default) [25.49% IMPUTED]](https://arxiv.org/abs/2506.05584),➖,1008.0,+61/-75,0.045,44.54,31.75,25.129,0.79,0.123,25.49,True,GPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1001.0,+53/-65,0.009,44.88,33.0,22.986,2.86,0.373,0.0,False,CPU -50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+44/-40,0.004,44.93,40.01,23.738,0.43,0.054,0.0,False,CPU -51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,993.0,+62/-80,0.018,45.25,31.89,25.602,129.01,1.802,0.0,False,CPU -52,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,978.0,+58/-62,0.011,45.94,40.23,25.248,0.25,0.05,0.0,False,CPU -53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,961.0,+64/-98,0.031,46.69,35.2,31.513,237.58,0.424,0.0,False,CPU -54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,933.0,+72/-114,0.02,47.87,37.08,32.121,237.58,0.085,0.0,False,CPU -55,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,882.0,+62/-98,0.007,49.77,38.19,30.831,129.01,0.18,0.0,False,CPU -56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,851.0,+70/-120,0.012,50.8,40.6,34.849,1.19,0.12,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,627.0,+89/-100,0.0,55.71,50.58,44.451,0.19,0.036,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1636.0,+60/-48,5.839,0.753,8.48,3.84,582.21,6.116,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1599.0,+78/-57,5.088,0.716,9.88,3.37,2040.22,8.92,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1554.0,+67/-50,6.062,0.658,11.83,4.65,2040.22,1.218,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1523.0,+66/-46,6.766,0.628,13.31,6.19,5.81,0.645,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1499.0,+56/-46,9.055,0.522,14.53,8.14,2950.72,11.988,✔️,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1499.0,+55/-47,9.005,0.517,14.56,5.99,1735.72,2.559,✔️,0.0,False,CPU +6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1451.0,+62/-49,9.138,0.496,17.21,5.14,4907.64,286.652,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1435.0,+50/-38,10.535,0.399,18.15,9.57,3285.87,1.472,✔️,0.0,False,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1419.0,+32/-29,11.754,0.35,19.13,12.78,416.98,2.639,✔️,0.0,False,CPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1416.0,+47/-46,10.535,0.365,19.32,10.24,2950.72,0.663,✔️,0.0,False,GPU +10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1403.0,+42/-39,11.264,0.334,20.12,11.5,1658.41,0.653,✔️,0.0,False,CPU +11,🧠⚡,Foundation Model,[LimiX (default) [11.98% IMPUTED]](https://arxiv.org/abs/2509.03505),1402.0,+67/-55,12.97,0.464,20.2,6.37,3.68,0.598,➖,11.98,True,GPU +12,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1396.0,+57/-53,10.533,0.402,20.54,7.13,4907.64,39.96,✔️,0.0,False,GPU +13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1392.0,+40/-41,11.447,0.324,20.8,10.82,1658.41,0.081,✔️,0.0,False,CPU +14,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1382.0,+75/-52,11.746,0.371,21.43,8.49,4621.67,8.149,✔️,0.0,False,GPU +15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1379.0,+48/-44,11.448,0.316,21.65,11.04,3285.87,0.173,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1374.0,+31/-29,12.443,0.265,21.92,15.54,416.98,0.334,✔️,0.0,False,CPU +17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1364.0,+33/-38,12.518,0.28,22.57,14.36,693.49,1.69,✔️,0.0,False,CPU +18,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1354.0,+38/-43,12.323,0.29,23.24,12.84,6.83,0.08,✔️,0.0,False,CPU +19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1348.0,+40/-37,12.287,0.245,23.61,12.01,4621.67,0.475,✔️,0.0,False,GPU +20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1343.0,+51/-39,12.886,0.253,23.97,14.02,846.88,2.552,✔️,0.0,False,GPU +21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1342.0,+32/-34,12.838,0.23,24.04,16.93,693.49,0.308,✔️,0.0,False,CPU +22,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1338.0,+65/-65,12.658,0.382,24.27,8.06,3030.15,21.444,✔️,35.29,True,GPU +23,🧠⚡,Foundation Model,[Mitra (default) [35.29% IMPUTED]](https://arxiv.org/abs/2510.21204),1326.0,+62/-62,13.408,0.36,25.11,8.78,87.65,2.505,✔️,35.29,True,GPU +24,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1320.0,+68/-55,13.232,0.326,25.48,8.32,47.62,43.739,✔️,0.0,False,GPU +25,🧠⚡,Foundation Model,[TabICL (default) [29.41% IMPUTED]](https://arxiv.org/abs/2502.05564),1317.0,+51/-57,12.932,0.318,25.66,9.42,6.63,1.48,✔️,29.41,True,GPU +26,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1295.0,+42/-37,14.176,0.188,27.14,16.97,846.88,0.13,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1287.0,+47/-44,13.952,0.219,27.69,16.45,10.49,0.132,✔️,0.0,False,GPU +28,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1281.0,+62/-58,14.316,0.285,28.06,9.86,3030.15,0.462,✔️,35.29,True,GPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1280.0,+47/-45,13.735,0.153,28.17,19.25,2874.67,1.952,✔️,0.0,False,CPU +30,🧠⚡,Foundation Model,[BetaTabPFN (default) [25.49% IMPUTED]](https://arxiv.org/abs/2502.02527),1274.0,+56/-50,15.319,0.232,28.59,15.28,205.87,1.343,➖,25.49,True,GPU +31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1259.0,+40/-38,15.485,0.149,29.58,19.57,2929.85,0.422,✔️,0.0,False,CPU +32,🧠⚡,Foundation Model,[TabPFNv2 (default) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1254.0,+66/-61,15.267,0.259,29.86,13.15,3.36,0.313,✔️,35.29,True,GPU +33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1246.0,+39/-40,15.853,0.122,30.46,19.1,14.87,0.308,✔️,0.0,False,GPU +34,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1226.0,+35/-38,15.245,0.082,31.77,22.77,10.06,1.689,✔️,0.0,False,GPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1223.0,+44/-45,16.161,0.106,31.95,20.88,2929.85,0.051,✔️,0.0,False,CPU +36,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1210.0,+38/-40,15.708,0.091,32.81,24.48,1.94,0.123,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1205.0,+43/-44,15.593,0.08,33.15,23.23,2874.67,0.131,✔️,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1205.0,+42/-41,17.165,0.094,33.16,22.18,183.02,0.761,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1202.0,+48/-50,16.636,0.12,33.37,20.04,593.24,4.467,✔️,0.0,False,CPU +40,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1192.0,+44/-47,17.32,0.097,34.0,22.56,7.32,0.05,✔️,0.0,False,CPU +41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1183.0,+31/-30,16.331,0.058,34.57,29.77,1.96,0.142,✔️,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1177.0,+50/-49,18.028,0.074,34.94,22.57,373.18,0.771,✔️,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1168.0,+43/-46,18.131,0.07,35.51,22.57,183.02,0.091,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1141.0,+48/-52,18.128,0.05,37.19,24.44,593.24,0.306,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1138.0,+49/-40,18.798,0.049,37.41,22.15,373.18,0.085,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1072.0,+36/-46,19.966,0.009,41.25,33.48,9.99,0.126,✔️,0.0,False,CPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1039.0,+59/-64,23.528,0.034,42.99,34.35,3.23,0.919,✔️,0.0,False,GPU +48,🧠⚡,Foundation Model,[TabFlex (default) [25.49% IMPUTED]](https://arxiv.org/abs/2506.05584),1008.0,+61/-75,25.129,0.045,44.54,31.75,0.79,0.123,➖,25.49,True,GPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1001.0,+53/-65,22.986,0.009,44.88,33.0,2.86,0.373,✔️,0.0,False,CPU +50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+44/-40,23.738,0.004,44.93,40.01,0.43,0.054,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),993.0,+62/-80,25.602,0.018,45.25,31.89,129.01,1.802,✔️,0.0,False,CPU +52,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),978.0,+58/-62,25.248,0.011,45.94,40.23,0.25,0.05,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),961.0,+64/-98,31.513,0.031,46.69,35.2,237.58,0.424,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),933.0,+72/-114,32.121,0.02,47.87,37.08,237.58,0.085,✔️,0.0,False,CPU +55,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),882.0,+62/-98,30.831,0.007,49.77,38.19,129.01,0.18,✔️,0.0,False,CPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),851.0,+70/-120,34.849,0.012,50.8,40.6,1.19,0.12,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),627.0,+89/-100,44.451,0.0,55.71,50.58,0.19,0.036,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_all/winrate_matrix.png.zip index d3671ed2f00756f7e528ecf1ff3d6fc1773b3065..0d257c5d2ab9327363b765ad4577222c487c3971 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a83087f41cd3f2d63c07da50e88a4a1f2bb72da876688260a7d5b520458be1e7 +oid sha256:bd478c61c99e2a708ec628c9c588c5486306797858b1ec77f23f659c86f64b46 size 2772353 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index 0d152925db546fc2930aa020788ca71dafef3758..93f96399fb45917145243941d1f60bc69d16ba4b 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c8e1d1ef26fb63a3b6e6f8091565858b8c8022e852a8f37545521d1399e8fe83 +oid sha256:50ddacad8d527054ff4c75b7c0ecf25903a0849b377fd321379506a41e66c76f size 450547 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip index f1896523e83a58c6c0b71bb8d74323dc78f48ae3..868ab6237a6adedd8d092bc8b567cb3e0140444f 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9d4de3950bd3972519bdf86de3563d6e37d70a460e34997d95ae9c68b613c21e -size 1020382 +oid sha256:1a868df62aa7c9168d0b2c36f553073e7588803430d89b3b1b245832771a9470 +size 999272 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_medium/tuning-impact-elo.png.zip index 419fd1b694999f1f9d60061e7a8c38504eeb6465..6bc982968639695a5feb05cc43f5eb3a07849a74 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:33325882d580d36b0dcd96c58e65741298e025ae4644afded040522c1d79cb85 +oid sha256:b847cc585de762cd70bc6171886abdb1ebfab1e9b4043a14d2fc4c42b1cd33a7 size 142000 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_all/datasets_medium/website_leaderboard.csv index 101ae0c2851daed494536c592b2742064b15fae9..1c8a716163e8f3cfcba6d39b9ab23880e483014a 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_medium/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_all/datasets_medium/website_leaderboard.csv @@ -1,55 +1,55 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1809.0,+94/-49,0.822,6.04,2.88,2.62,289.53,3.187,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1784.0,+87/-53,0.795,6.69,3.13,2.723,432.35,4.085,0.0,False,CPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1662.0,+163/-119,0.649,10.68,3.35,5.438,735.58,11.736,0.0,False,GPU -3,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1632.0,+104/-90,0.576,11.83,7.36,5.74,1719.82,1.675,0.0,False,GPU -4,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1620.0,+120/-113,0.593,12.33,6.83,5.81,2526.28,6.013,0.0,False,GPU -5,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1607.0,+90/-64,0.536,12.89,6.9,5.642,777.59,0.25,0.0,False,CPU -6,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1606.0,+146/-108,0.582,12.92,5.31,6.152,735.58,1.386,0.0,False,GPU -7,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1598.0,+97/-65,0.538,13.3,5.73,5.838,777.59,0.052,0.0,False,CPU -8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1590.0,+75/-59,0.488,13.64,9.18,7.405,131.56,2.639,0.0,False,CPU -9,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1557.0,+110/-95,0.521,15.15,7.44,6.028,3.24,0.031,0.0,False,CPU -10,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1549.0,+87/-56,0.442,15.51,9.32,7.531,282.13,0.563,0.0,False,CPU -11,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1536.0,+146/-95,0.471,16.17,7.91,6.964,1.88,0.645,0.0,False,GPU -12,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1533.0,+114/-88,0.406,16.29,9.19,7.07,1719.82,0.081,0.0,False,GPU -13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1531.0,+149/-89,0.424,16.41,7.54,7.466,1993.14,0.619,0.0,False,GPU -14,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1519.0,+130/-121,0.482,16.97,4.24,7.078,4786.55,444.544,0.0,False,GPU -15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1502.0,+78/-60,0.346,17.84,12.32,8.625,131.56,0.132,0.0,False,CPU -16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1501.0,+70/-49,0.359,17.86,13.6,8.115,282.13,0.066,0.0,False,CPU -17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1479.0,+145/-95,0.358,18.99,8.67,8.222,1993.14,0.061,0.0,False,GPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1473.0,+104/-87,0.33,19.32,13.86,8.204,2526.28,0.349,0.0,False,GPU -19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1427.0,+112/-93,0.282,21.7,13.1,9.726,566.74,0.951,0.0,False,GPU -20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1397.0,+123/-120,0.296,23.29,8.88,9.704,4786.55,42.641,0.0,False,GPU -21,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1375.0,+70/-65,0.177,24.49,17.88,9.473,1770.56,1.069,0.0,False,CPU -22,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1374.0,+96/-92,0.206,24.56,19.3,11.424,566.74,0.06,0.0,False,GPU -23,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1366.0,+101/-111,0.228,24.99,16.75,9.978,0.49,0.053,0.0,False,CPU -24,🧠⚡,Foundation Model,[TabICL (default) [33.33% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1341.0,+131/-109,0.276,26.37,12.29,10.315,3.97,1.596,33.33,True,GPU -25,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1328.0,+120/-136,0.225,27.08,10.23,12.499,46.62,43.739,0.0,False,GPU -26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1322.0,+132/-105,0.181,27.38,17.94,11.062,5.16,0.074,0.0,False,GPU -27,🧠🔁,Neural Network,[LimiX (default) [40.00% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1322.0,+138/-128,0.25,27.39,8.86,14.019,1.08,0.32,40.0,True,GPU -28,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1320.0,+118/-126,0.193,27.51,18.45,12.974,1517.65,0.134,0.0,False,CPU -29,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1296.0,+76/-79,0.111,28.81,24.71,11.512,8.06,0.25,0.0,False,GPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1274.0,+85/-73,0.05,29.99,26.82,12.26,6.75,0.235,0.0,False,GPU -31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1269.0,+78/-77,0.064,30.26,25.57,11.256,1770.56,0.051,0.0,False,CPU -32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1259.0,+112/-138,0.136,30.76,23.35,13.864,1517.65,0.015,0.0,False,CPU -33,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1257.0,+57/-67,0.045,30.9,28.62,11.913,0.29,0.045,0.0,False,CPU -34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1249.0,+82/-72,0.041,31.29,25.01,12.464,66.0,0.341,0.0,False,CPU -35,🧠⚡,Foundation Model,[BetaTabPFN (default) [33.33% IMPUTED]](https://arxiv.org/abs/2502.02527),➖,1225.0,+107/-93,0.081,32.56,25.25,14.429,66.06,0.652,33.33,True,GPU -36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1213.0,+90/-83,0.042,33.14,26.81,13.175,66.0,0.024,0.0,False,CPU -37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1212.0,+101/-142,0.096,33.22,28.28,15.447,3.12,0.016,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1208.0,+70/-83,0.025,33.41,30.98,12.591,36.48,0.329,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1200.0,+110/-138,0.075,33.83,26.33,14.414,351.94,1.658,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1163.0,+61/-79,0.011,35.65,33.8,13.815,36.48,0.037,0.0,False,CPU -41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1118.0,+69/-72,0.001,37.78,36.08,15.577,2.18,0.039,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1093.0,+87/-116,0.022,38.9,34.23,16.687,351.94,0.114,0.0,False,CPU -43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1049.0,+139/-182,0.02,40.74,35.28,23.725,2.49,0.145,0.0,False,GPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1014.0,+135/-175,0.026,42.13,37.5,20.379,1.51,0.111,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+58/-88,0.0,42.67,41.79,18.741,0.2,0.021,0.0,False,CPU -46,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,989.0,+88/-119,0.006,43.08,36.37,22.293,150.3,10.071,0.0,False,CPU -47,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,948.0,+151/-218,0.005,44.5,40.65,31.147,43.64,0.081,0.0,False,CPU -48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,945.0,+111/-138,0.003,44.62,42.76,21.857,0.11,0.028,0.0,False,CPU -49,🧠⚡,Foundation Model,[TabFlex (default) [33.33% IMPUTED]](https://arxiv.org/abs/2506.05584),➖,926.0,+72/-121,0.0,45.24,44.31,25.797,0.25,0.045,33.33,True,GPU -50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,899.0,+155/-243,0.003,46.06,43.0,32.089,43.64,0.018,0.0,False,CPU -51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,804.0,+62/-101,0.0,48.55,48.25,28.282,150.3,0.482,0.0,False,CPU -52,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,787.0,+131/-231,0.0,48.93,47.58,34.193,0.19,0.019,0.0,False,CPU -53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,560.0,+100/-154,0.0,52.4,52.31,42.236,0.1,0.072,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1809.0,+94/-49,2.62,0.822,6.04,2.88,289.53,3.187,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1784.0,+87/-53,2.723,0.795,6.69,3.13,432.35,4.085,✔️,0.0,False,CPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1662.0,+163/-119,5.438,0.649,10.68,3.35,735.58,11.736,✔️,0.0,False,GPU +3,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1632.0,+104/-90,5.74,0.576,11.83,7.36,1719.82,1.675,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1620.0,+120/-113,5.81,0.593,12.33,6.83,2526.28,6.013,✔️,0.0,False,GPU +5,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1607.0,+90/-64,5.642,0.536,12.89,6.9,777.59,0.25,✔️,0.0,False,CPU +6,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1606.0,+146/-108,6.152,0.582,12.92,5.31,735.58,1.386,✔️,0.0,False,GPU +7,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1598.0,+97/-65,5.838,0.538,13.3,5.73,777.59,0.052,✔️,0.0,False,CPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1590.0,+75/-59,7.405,0.488,13.64,9.18,131.56,2.639,✔️,0.0,False,CPU +9,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1557.0,+110/-95,6.028,0.521,15.15,7.44,3.24,0.031,✔️,0.0,False,CPU +10,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1549.0,+87/-56,7.531,0.442,15.51,9.32,282.13,0.563,✔️,0.0,False,CPU +11,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1536.0,+146/-95,6.964,0.471,16.17,7.91,1.88,0.645,✔️,0.0,False,GPU +12,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1533.0,+114/-88,7.07,0.406,16.29,9.19,1719.82,0.081,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1531.0,+149/-89,7.466,0.424,16.41,7.54,1993.14,0.619,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1519.0,+130/-121,7.078,0.482,16.97,4.24,4786.55,444.544,✔️,0.0,False,GPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1502.0,+78/-60,8.625,0.346,17.84,12.32,131.56,0.132,✔️,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1501.0,+70/-49,8.115,0.359,17.86,13.6,282.13,0.066,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1479.0,+145/-95,8.222,0.358,18.99,8.67,1993.14,0.061,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1473.0,+104/-87,8.204,0.33,19.32,13.86,2526.28,0.349,✔️,0.0,False,GPU +19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1427.0,+112/-93,9.726,0.282,21.7,13.1,566.74,0.951,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1397.0,+123/-120,9.704,0.296,23.29,8.88,4786.55,42.641,✔️,0.0,False,GPU +21,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1375.0,+70/-65,9.473,0.177,24.49,17.88,1770.56,1.069,✔️,0.0,False,CPU +22,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1374.0,+96/-92,11.424,0.206,24.56,19.3,566.74,0.06,✔️,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1366.0,+101/-111,9.978,0.228,24.99,16.75,0.49,0.053,✔️,0.0,False,CPU +24,🧠⚡,Foundation Model,[TabICL (default) [33.33% IMPUTED]](https://arxiv.org/abs/2502.05564),1341.0,+131/-109,10.315,0.276,26.37,12.29,3.97,1.596,✔️,33.33,True,GPU +25,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1328.0,+120/-136,12.499,0.225,27.08,10.23,46.62,43.739,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1322.0,+132/-105,11.062,0.181,27.38,17.94,5.16,0.074,✔️,0.0,False,GPU +27,🧠⚡,Foundation Model,[LimiX (default) [40.00% IMPUTED]](https://arxiv.org/abs/2509.03505),1322.0,+138/-128,14.019,0.25,27.39,8.86,1.08,0.32,➖,40.0,True,GPU +28,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1320.0,+118/-126,12.974,0.193,27.51,18.45,1517.65,0.134,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1296.0,+76/-79,11.512,0.111,28.81,24.71,8.06,0.25,✔️,0.0,False,GPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1274.0,+85/-73,12.26,0.05,29.99,26.82,6.75,0.235,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1269.0,+78/-77,11.256,0.064,30.26,25.57,1770.56,0.051,✔️,0.0,False,CPU +32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1259.0,+112/-138,13.864,0.136,30.76,23.35,1517.65,0.015,✔️,0.0,False,CPU +33,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1257.0,+57/-67,11.913,0.045,30.9,28.62,0.29,0.045,✔️,0.0,False,CPU +34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1249.0,+82/-72,12.464,0.041,31.29,25.01,66.0,0.341,✔️,0.0,False,CPU +35,🧠⚡,Foundation Model,[BetaTabPFN (default) [33.33% IMPUTED]](https://arxiv.org/abs/2502.02527),1225.0,+107/-93,14.429,0.081,32.56,25.25,66.06,0.652,➖,33.33,True,GPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1213.0,+90/-83,13.175,0.042,33.14,26.81,66.0,0.024,✔️,0.0,False,CPU +37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1212.0,+101/-142,15.447,0.096,33.22,28.28,3.12,0.016,✔️,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1208.0,+70/-83,12.591,0.025,33.41,30.98,36.48,0.329,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1200.0,+110/-138,14.414,0.075,33.83,26.33,351.94,1.658,✔️,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1163.0,+61/-79,13.815,0.011,35.65,33.8,36.48,0.037,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1118.0,+69/-72,15.577,0.001,37.78,36.08,2.18,0.039,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1093.0,+87/-116,16.687,0.022,38.9,34.23,351.94,0.114,✔️,0.0,False,CPU +43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1049.0,+139/-182,23.725,0.02,40.74,35.28,2.49,0.145,✔️,0.0,False,GPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1014.0,+135/-175,20.379,0.026,42.13,37.5,1.51,0.111,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+58/-88,18.741,0.0,42.67,41.79,0.2,0.021,✔️,0.0,False,CPU +46,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),989.0,+88/-119,22.293,0.006,43.08,36.37,150.3,10.071,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),948.0,+151/-218,31.147,0.005,44.5,40.65,43.64,0.081,✔️,0.0,False,CPU +48,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),945.0,+111/-138,21.857,0.003,44.62,42.76,0.11,0.028,✔️,0.0,False,CPU +49,🧠⚡,Foundation Model,[TabFlex (default) [33.33% IMPUTED]](https://arxiv.org/abs/2506.05584),926.0,+72/-121,25.797,0.0,45.24,44.31,0.25,0.045,➖,33.33,True,GPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),899.0,+155/-243,32.089,0.003,46.06,43.0,43.64,0.018,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),804.0,+62/-101,28.282,0.0,48.55,48.25,150.3,0.482,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),787.0,+131/-231,34.193,0.0,48.93,47.58,0.19,0.019,✔️,0.0,False,CPU +53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),560.0,+100/-154,42.236,0.0,52.4,52.31,0.1,0.072,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_medium/winrate_matrix.png.zip index 1eb665bf59487259ef7bbc7f6874c0ac35a32ff4..6a4fb7e0c7374ec76c77fad73ea32c688ff37f00 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e02322996075c246616c322456716eb4a3afbafd841188c04023641dbb4204b1 +oid sha256:48d4cf58c01eec79b7a840a77e8291c7df43374e8496dc8d674d46a2d966f2c7 size 2309578 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 222f946d89c203c74b5d408ad24e453330126aac..d6078810a67604e37c20493bd883852e38e713f8 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8ec2b217756c92ca695ebbaaf7715d86fd41c5b3713ceaa55f6948eb9420eaf6 +oid sha256:a68bc3372dc9aa20198862597f333cf01a54677a31598ff040d2875c48a8e5a1 size 464248 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_small/pareto_n_configs_imp.png.zip index c12f78c5f2026f234a8236bc5ac70816afe237bc..4448b456ceed313000383be8752cbca53972025c 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cc17aa3a51db65b96144eba85fff725a3d07d62e55b6ba8bb8eeb8d91608d3d6 -size 1073532 +oid sha256:b4e4c03de480d60e809f5e16dddc004ad6d4765bf8628c84fcb5fab0a4b18a7c +size 1048136 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_small/tuning-impact-elo.png.zip index aac6bef2da2044c94f7aef7873d6b1a9d0c3d041..c1c09209e32a7bb9c59e22ee32e6f172d2c49df8 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0862ad8e7e235e7e06d99016fd865cf0926b8d46b0b44e074d6ab4bc721cffec +oid sha256:1013d6f95f51046efd9af64816ef80526d76a6afd0b3e0981194a9169102cf37 size 193409 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_all/datasets_small/website_leaderboard.csv index 4acb141c6eaaa174f17ddc5eebabb529c726a2e8..fc19183ddeb66d8f23a3f265ac756bbd4cf5fd31 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_small/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_all/datasets_small/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1612.0,+63/-54,0.715,9.5,4.46,7.181,659.63,7.444,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1610.0,+87/-62,0.738,9.55,3.38,4.943,2289.05,8.049,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1567.0,+71/-55,0.681,11.37,4.42,6.024,2289.05,1.143,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1551.0,+81/-54,0.676,12.09,5.68,6.684,6.76,0.636,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1482.0,+56/-48,0.487,15.58,8.52,10.436,3770.75,21.902,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1480.0,+89/-64,0.542,15.73,6.02,10.121,3527.3,48.6,8.33,True,GPU -6,🧠🔁,Neural Network,[LimiX (default) [0.31% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1459.0,+86/-69,0.544,16.89,5.69,12.533,5.08,0.645,0.31,True,GPU -7,🧠🔁,Neural Network,[Mitra (default) [8.33% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1458.0,+68/-56,0.51,16.97,6.59,11.186,126.84,3.094,8.33,True,GPU -8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1452.0,+80/-59,0.486,17.3,5.65,9.997,5119.36,218.711,0.0,False,GPU -9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1442.0,+59/-46,0.391,17.84,9.65,11.622,3716.07,2.547,0.0,False,CPU -10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1426.0,+52/-46,0.373,18.86,10.78,11.814,3553.12,1.741,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1418.0,+73/-57,0.435,19.3,6.59,10.878,5119.36,28.351,0.0,False,GPU -12,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1399.0,+52/-41,0.331,20.5,10.75,11.979,3770.75,1.778,0.0,False,GPU -13,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1389.0,+77/-62,0.403,21.1,7.45,12.469,3527.3,0.817,8.33,True,GPU -14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1384.0,+33/-36,0.273,21.41,15.27,13.566,892.41,2.574,0.0,False,CPU -15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1364.0,+51/-49,0.282,22.68,12.45,12.792,3553.12,0.241,0.0,False,GPU -16,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1358.0,+40/-44,0.23,23.13,15.92,13.606,2476.51,0.814,0.0,False,CPU -17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1350.0,+34/-30,0.207,23.6,17.44,14.034,892.41,0.346,0.0,False,CPU -18,🧠⚡,Foundation Model,[TabPFNv2 (default) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1349.0,+76/-63,0.367,23.7,10.18,13.819,4.54,0.443,8.33,True,GPU -19,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1346.0,+45/-37,0.215,23.91,17.16,13.784,2476.51,0.101,0.0,False,CPU -20,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1337.0,+72/-68,0.35,24.45,7.71,13.538,50.32,43.709,0.0,False,GPU -21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1332.0,+44/-41,0.228,24.79,14.42,14.202,1126.71,2.941,0.0,False,GPU -22,🧠⚡,Foundation Model,[TabICL (default) [27.78% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1328.0,+79/-106,0.327,25.08,8.57,14.023,7.15,1.47,27.78,True,GPU -23,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1326.0,+62/-62,0.266,25.22,9.46,14.22,6521.96,8.396,0.0,False,GPU -24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1324.0,+41/-41,0.193,25.36,11.38,13.989,6521.96,0.506,0.0,False,GPU -25,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1321.0,+38/-39,0.19,25.51,18.55,14.596,884.18,2.368,0.0,False,CPU -26,🧠⚡,Foundation Model,[BetaTabPFN (default) [22.22% IMPUTED]](https://arxiv.org/abs/2502.02527),➖,1308.0,+58/-61,0.286,26.4,13.07,15.69,296.89,1.835,22.22,True,GPU -27,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1305.0,+37/-38,0.175,26.6,18.42,14.946,9.64,0.128,0.0,False,CPU -28,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1305.0,+36/-40,0.151,26.61,18.85,14.805,884.18,0.393,0.0,False,CPU -29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1290.0,+54/-48,0.219,27.59,15.87,15.156,13.18,0.17,0.0,False,GPU -30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1284.0,+41/-41,0.164,27.99,16.12,15.323,1126.71,0.163,0.0,False,GPU -31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1260.0,+42/-48,0.13,29.6,19.86,15.511,3865.95,2.158,0.0,False,CPU -32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1255.0,+43/-43,0.117,29.96,19.99,16.532,3351.28,0.47,0.0,False,CPU -33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1238.0,+44/-50,0.109,31.08,17.44,17.662,16.27,0.321,0.0,False,GPU -34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1226.0,+44/-44,0.086,31.89,19.9,17.118,3351.28,0.056,0.0,False,CPU -35,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1221.0,+39/-42,0.082,32.22,21.35,16.489,15.69,4.691,0.0,False,GPU -36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1216.0,+55/-59,0.134,32.51,18.12,17.561,1678.92,8.069,0.0,False,CPU -37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1210.0,+65/-64,0.119,32.92,19.81,19.071,499.14,1.466,0.0,False,CPU -38,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1197.0,+48/-53,0.087,33.75,20.7,18.1,9.08,0.057,0.0,False,CPU -39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1192.0,+43/-48,0.079,34.11,22.31,17.401,3865.95,0.148,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1174.0,+61/-64,0.091,35.22,19.78,19.929,499.14,0.207,0.0,False,CPU -41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1167.0,+50/-51,0.06,35.7,21.68,18.728,1678.92,0.826,0.0,False,CPU -42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1162.0,+40/-40,0.042,35.97,30.21,18.171,3.61,0.17,0.0,False,CPU -43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1162.0,+30/-36,0.019,35.98,30.28,18.096,3.29,0.253,0.0,False,CPU -44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1156.0,+57/-68,0.086,36.41,21.67,20.347,590.04,1.461,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1114.0,+55/-70,0.049,38.94,20.59,21.141,590.04,0.15,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1057.0,+44/-51,0.01,42.17,32.23,21.794,13.69,0.155,0.0,False,CPU -47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1041.0,+70/-88,0.032,43.0,33.58,23.446,5.44,1.098,0.0,False,GPU -48,🧠⚡,Foundation Model,[TabFlex (default) [22.22% IMPUTED]](https://arxiv.org/abs/2506.05584),➖,1035.0,+80/-81,0.064,43.32,28.07,24.85,1.58,0.15,22.22,True,GPU -49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+58/-77,0.006,45.05,38.62,25.82,0.51,0.089,0.0,False,CPU -50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1000.0,+51/-63,0.0,45.05,31.06,24.072,5.48,0.685,0.0,False,CPU -51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,995.0,+71/-103,0.023,45.29,30.0,26.981,111.84,1.215,0.0,False,CPU -52,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,995.0,+71/-73,0.013,45.29,38.52,26.662,0.47,0.092,0.0,False,CPU -53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,967.0,+84/-94,0.039,46.55,32.92,31.665,520.95,0.647,0.0,False,CPU -54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,944.0,+82/-100,0.026,47.53,34.6,32.134,520.95,0.113,0.0,False,CPU -55,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,907.0,+74/-97,0.009,49.0,34.54,31.894,111.84,0.111,0.0,False,CPU -56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,866.0,+98/-119,0.017,50.45,37.69,35.122,2.59,0.138,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,654.0,+112/-150,0.0,55.42,48.52,45.374,0.37,0.033,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1612.0,+63/-54,7.181,0.715,9.5,4.46,659.63,7.444,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1610.0,+87/-62,4.943,0.738,9.55,3.38,2289.05,8.049,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1567.0,+71/-55,6.024,0.681,11.37,4.42,2289.05,1.143,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1551.0,+81/-54,6.684,0.676,12.09,5.68,6.76,0.636,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1482.0,+56/-48,10.436,0.487,15.58,8.52,3770.75,21.902,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1480.0,+89/-64,10.121,0.542,15.73,6.02,3527.3,48.6,✔️,8.33,True,GPU +6,🧠⚡,Foundation Model,[LimiX (default) [0.31% IMPUTED]](https://arxiv.org/abs/2509.03505),1459.0,+86/-69,12.533,0.544,16.89,5.69,5.08,0.645,➖,0.31,True,GPU +7,🧠⚡,Foundation Model,[Mitra (default) [8.33% IMPUTED]](https://arxiv.org/abs/2510.21204),1458.0,+68/-56,11.186,0.51,16.97,6.59,126.84,3.094,✔️,8.33,True,GPU +8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1452.0,+80/-59,9.997,0.486,17.3,5.65,5119.36,218.711,✔️,0.0,False,GPU +9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1442.0,+59/-46,11.622,0.391,17.84,9.65,3716.07,2.547,✔️,0.0,False,CPU +10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1426.0,+52/-46,11.814,0.373,18.86,10.78,3553.12,1.741,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1418.0,+73/-57,10.878,0.435,19.3,6.59,5119.36,28.351,✔️,0.0,False,GPU +12,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1399.0,+52/-41,11.979,0.331,20.5,10.75,3770.75,1.778,✔️,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1389.0,+77/-62,12.469,0.403,21.1,7.45,3527.3,0.817,✔️,8.33,True,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1384.0,+33/-36,13.566,0.273,21.41,15.27,892.41,2.574,✔️,0.0,False,CPU +15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1364.0,+51/-49,12.792,0.282,22.68,12.45,3553.12,0.241,✔️,0.0,False,GPU +16,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1358.0,+40/-44,13.606,0.23,23.13,15.92,2476.51,0.814,✔️,0.0,False,CPU +17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1350.0,+34/-30,14.034,0.207,23.6,17.44,892.41,0.346,✔️,0.0,False,CPU +18,🧠⚡,Foundation Model,[TabPFNv2 (default) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1349.0,+76/-63,13.819,0.367,23.7,10.18,4.54,0.443,✔️,8.33,True,GPU +19,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1346.0,+45/-37,13.784,0.215,23.91,17.16,2476.51,0.101,✔️,0.0,False,CPU +20,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1337.0,+72/-68,13.538,0.35,24.45,7.71,50.32,43.709,✔️,0.0,False,GPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1332.0,+44/-41,14.202,0.228,24.79,14.42,1126.71,2.941,✔️,0.0,False,GPU +22,🧠⚡,Foundation Model,[TabICL (default) [27.78% IMPUTED]](https://arxiv.org/abs/2502.05564),1328.0,+79/-106,14.023,0.327,25.08,8.57,7.15,1.47,✔️,27.78,True,GPU +23,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1326.0,+62/-62,14.22,0.266,25.22,9.46,6521.96,8.396,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1324.0,+41/-41,13.989,0.193,25.36,11.38,6521.96,0.506,✔️,0.0,False,GPU +25,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1321.0,+38/-39,14.596,0.19,25.51,18.55,884.18,2.368,✔️,0.0,False,CPU +26,🧠⚡,Foundation Model,[BetaTabPFN (default) [22.22% IMPUTED]](https://arxiv.org/abs/2502.02527),1308.0,+58/-61,15.69,0.286,26.4,13.07,296.89,1.835,➖,22.22,True,GPU +27,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1305.0,+37/-38,14.946,0.175,26.6,18.42,9.64,0.128,✔️,0.0,False,CPU +28,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1305.0,+36/-40,14.805,0.151,26.61,18.85,884.18,0.393,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1290.0,+54/-48,15.156,0.219,27.59,15.87,13.18,0.17,✔️,0.0,False,GPU +30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1284.0,+41/-41,15.323,0.164,27.99,16.12,1126.71,0.163,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1260.0,+42/-48,15.511,0.13,29.6,19.86,3865.95,2.158,✔️,0.0,False,CPU +32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1255.0,+43/-43,16.532,0.117,29.96,19.99,3351.28,0.47,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1238.0,+44/-50,17.662,0.109,31.08,17.44,16.27,0.321,✔️,0.0,False,GPU +34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1226.0,+44/-44,17.118,0.086,31.89,19.9,3351.28,0.056,✔️,0.0,False,CPU +35,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1221.0,+39/-42,16.489,0.082,32.22,21.35,15.69,4.691,✔️,0.0,False,GPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1216.0,+55/-59,17.561,0.134,32.51,18.12,1678.92,8.069,✔️,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1210.0,+65/-64,19.071,0.119,32.92,19.81,499.14,1.466,✔️,0.0,False,CPU +38,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1197.0,+48/-53,18.1,0.087,33.75,20.7,9.08,0.057,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1192.0,+43/-48,17.401,0.079,34.11,22.31,3865.95,0.148,✔️,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1174.0,+61/-64,19.929,0.091,35.22,19.78,499.14,0.207,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1167.0,+50/-51,18.728,0.06,35.7,21.68,1678.92,0.826,✔️,0.0,False,CPU +42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1162.0,+40/-40,18.171,0.042,35.97,30.21,3.61,0.17,✔️,0.0,False,CPU +43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1162.0,+30/-36,18.096,0.019,35.98,30.28,3.29,0.253,✔️,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1156.0,+57/-68,20.347,0.086,36.41,21.67,590.04,1.461,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1114.0,+55/-70,21.141,0.049,38.94,20.59,590.04,0.15,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1057.0,+44/-51,21.794,0.01,42.17,32.23,13.69,0.155,✔️,0.0,False,CPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1041.0,+70/-88,23.446,0.032,43.0,33.58,5.44,1.098,✔️,0.0,False,GPU +48,🧠⚡,Foundation Model,[TabFlex (default) [22.22% IMPUTED]](https://arxiv.org/abs/2506.05584),1035.0,+80/-81,24.85,0.064,43.32,28.07,1.58,0.15,➖,22.22,True,GPU +49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+58/-77,25.82,0.006,45.05,38.62,0.51,0.089,✔️,0.0,False,CPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1000.0,+51/-63,24.072,0.0,45.05,31.06,5.48,0.685,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),995.0,+71/-103,26.981,0.023,45.29,30.0,111.84,1.215,✔️,0.0,False,CPU +52,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),995.0,+71/-73,26.662,0.013,45.29,38.52,0.47,0.092,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),967.0,+84/-94,31.665,0.039,46.55,32.92,520.95,0.647,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),944.0,+82/-100,32.134,0.026,47.53,34.6,520.95,0.113,✔️,0.0,False,CPU +55,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),907.0,+74/-97,31.894,0.009,49.0,34.54,111.84,0.111,✔️,0.0,False,CPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),866.0,+98/-119,35.122,0.017,50.45,37.69,2.59,0.138,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),654.0,+112/-150,45.374,0.0,55.42,48.52,0.37,0.033,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_small/winrate_matrix.png.zip index e6c4037bd2ac41ca1af68e0086b37d916c64b16e..c440b334f001b575d568cd42839ebc6479e0da53 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c735852e7df29e1a141497f6979b5b60056595d6ed85043353ca3c7972118264 +oid sha256:85e0b9600387668d2e326b6a29b3bfb48e9abd2ee98864500a52cb39ccdcb66f size 2746120 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 7b379bf0d52703b2cf8f3000978208fb4fbaafb4..4914366b2a28d3777590e98b2bc5232ccfd66701 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2f23b821023904354c3b2b82f2c3646bb1d36a6d02d839464f02c49eea670d91 +oid sha256:c0e71ebe67203c7780a5e6b9ad321d726ac480167864a139ea182620c10d90a2 size 464205 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip index a5e633138abccdc67d7523a35ef1fe6f7c29e20c..0eb344392fddacc64325ef2c9759e552201eb4df 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9f3140f501f22776903203d8b1b05f4531d477070f85261de6441c97a3265a8f -size 1109426 +oid sha256:f98ba33aed96dbeb4c223c1e82609a4ee363341edd42c5c4620fe0ff8c95bcfc +size 1085967 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip index 29adf2838e145c9561aac98e7751b64cc787b976..b38ea9763f0f7afd08e71288b381c8eb1d865331 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ba620d1a5dd20e832c577c3be996153f35c34f0f676d17e115d80a6806239ce9 +oid sha256:4b128b80a98f144a4b9c30f631608ad8d94bdfd251dcdba4e53481d2562472c0 size 145426 diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/website_leaderboard.csv index 4ebf7f3adb7cf16d83ba4a97a4e4a8f3e2c782bd..2282a6b130e7072a65a5583bec2a5d6423438f49 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1632.0,+93/-70,0.74,9.53,3.46,5.292,2166.18,7.945,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1628.0,+76/-55,0.711,9.71,4.55,7.717,611.55,7.01,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1594.0,+66/-62,0.714,11.11,5.37,6.928,6.14,0.586,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1589.0,+71/-61,0.686,11.29,4.35,6.39,2166.18,0.993,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1540.0,+74/-48,0.586,13.62,5.6,9.556,3899.42,55.833,0.0,False,GPU -5,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1511.0,+72/-47,0.551,15.1,6.15,10.725,136.59,3.391,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1505.0,+56/-46,0.499,15.41,8.34,11.115,3547.04,20.963,0.0,False,GPU -7,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1498.0,+74/-70,0.576,15.78,5.57,13.136,4.85,0.629,0.0,False,GPU -8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1461.0,+74/-61,0.47,17.9,5.77,10.724,4980.83,207.372,0.0,False,GPU -9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1453.0,+48/-47,0.383,18.36,10.23,12.387,3376.59,2.54,0.0,False,CPU -10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1441.0,+54/-50,0.369,19.08,10.6,12.586,3372.56,1.658,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1436.0,+78/-68,0.435,19.38,6.39,11.521,4980.83,23.583,0.0,False,GPU -12,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1434.0,+82/-68,0.435,19.47,6.97,12.117,3899.42,0.983,0.0,False,GPU -13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1417.0,+57/-46,0.339,20.53,10.64,12.689,3547.04,1.57,0.0,False,GPU -14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1392.0,+38/-36,0.252,22.1,15.71,14.551,771.57,2.49,0.0,False,CPU -15,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1386.0,+68/-69,0.396,22.44,9.6,13.598,4.22,0.554,0.0,False,GPU -16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1379.0,+55/-51,0.278,22.92,12.22,13.598,3372.56,0.208,0.0,False,GPU -17,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1377.0,+43/-38,0.237,23.04,15.6,14.476,2034.85,0.805,0.0,False,CPU -18,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1363.0,+38/-34,0.219,23.94,17.08,14.675,2034.85,0.097,0.0,False,CPU -19,🧠⚡,Foundation Model,[TabICL (default) [21.21% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1363.0,+77/-84,0.352,23.95,8.04,13.82,7.27,1.64,21.21,True,GPU -20,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1361.0,+91/-63,0.373,24.06,7.26,14.241,48.9,43.804,0.0,False,GPU -21,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1358.0,+34/-30,0.186,24.26,17.76,15.026,771.57,0.322,0.0,False,CPU -22,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1338.0,+52/-46,0.208,25.57,15.07,15.196,1105.59,2.552,0.0,False,GPU -23,🧠⚡,Foundation Model,[BetaTabPFN (default) [21.21% IMPUTED]](https://arxiv.org/abs/2502.02527),➖,1338.0,+64/-77,0.291,25.61,12.57,16.243,303.08,1.792,21.21,True,GPU -24,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1334.0,+37/-36,0.182,25.82,19.14,15.598,828.74,2.312,0.0,False,CPU -25,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1334.0,+45/-48,0.172,25.85,12.27,14.832,6147.69,0.485,0.0,False,GPU -26,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1334.0,+73/-66,0.252,25.86,11.65,15.247,6147.69,8.149,0.0,False,GPU -27,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1322.0,+48/-44,0.177,26.64,18.03,15.903,8.51,0.123,0.0,False,CPU -28,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1316.0,+34/-34,0.142,26.99,19.24,15.81,828.74,0.336,0.0,False,CPU -29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1309.0,+55/-53,0.22,27.51,15.35,15.975,12.24,0.154,0.0,False,GPU -30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1293.0,+45/-41,0.154,28.52,16.9,16.334,1105.59,0.13,0.0,False,GPU -31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1278.0,+47/-44,0.114,29.53,19.36,17.376,2929.85,0.433,0.0,False,CPU -32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1271.0,+45/-54,0.122,30.01,21.32,16.4,3704.3,2.074,0.0,False,CPU -33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1266.0,+46/-41,0.117,30.36,16.68,17.996,16.16,0.307,0.0,False,GPU -34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1248.0,+51/-48,0.081,31.51,19.23,17.977,2929.85,0.054,0.0,False,CPU -35,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1243.0,+40/-38,0.081,31.85,20.77,17.096,15.48,4.535,0.0,False,GPU -36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1234.0,+51/-60,0.137,32.42,17.55,18.45,1459.62,8.056,0.0,False,CPU -37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1216.0,+49/-51,0.072,33.57,20.75,18.863,8.47,0.054,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1211.0,+58/-55,0.1,33.91,22.7,20.343,416.39,1.393,0.0,False,CPU -39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1201.0,+42/-46,0.065,34.56,22.3,18.313,3704.3,0.143,0.0,False,CPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1180.0,+54/-59,0.06,35.83,21.57,19.638,1459.62,0.89,0.0,False,CPU -41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1177.0,+57/-56,0.069,36.06,23.58,21.176,416.39,0.179,0.0,False,CPU -42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1172.0,+34/-37,0.015,36.37,31.15,19.218,3.05,0.241,0.0,False,CPU -43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1158.0,+35/-41,0.021,37.24,32.62,19.446,3.39,0.158,0.0,False,CPU -44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1143.0,+54/-53,0.051,38.11,26.04,21.823,572.67,1.421,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1103.0,+45/-45,0.018,40.42,26.25,22.6,572.67,0.143,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1067.0,+50/-59,0.011,42.4,31.83,22.778,11.82,0.147,0.0,False,CPU -47,🧠⚡,Foundation Model,[TabFlex (default) [21.21% IMPUTED]](https://arxiv.org/abs/2506.05584),➖,1056.0,+82/-88,0.069,43.0,27.1,25.63,1.19,0.15,21.21,True,GPU -48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1054.0,+70/-86,0.035,43.09,33.03,24.777,5.39,1.034,0.0,False,GPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1009.0,+51/-66,0.0,45.3,30.44,24.915,5.18,0.652,0.0,False,CPU -50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1006.0,+82/-85,0.014,45.46,38.22,27.446,0.42,0.078,0.0,False,CPU -51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1000.0,+72/-88,0.014,45.71,33.26,28.256,100.58,1.106,0.0,False,CPU -52,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+53/-69,0.002,45.73,39.57,26.69,0.47,0.069,0.0,False,CPU -53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1000.0,+71/-104,0.043,45.75,31.75,31.374,504.17,0.629,0.0,False,CPU -54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,977.0,+72/-104,0.028,46.76,33.44,31.806,504.17,0.103,0.0,False,CPU -55,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,904.0,+78/-101,0.0,49.65,35.57,33.41,100.58,0.101,0.0,False,CPU -56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,899.0,+80/-125,0.018,49.8,36.54,34.29,2.51,0.137,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,677.0,+96/-161,0.0,55.29,47.88,45.738,0.29,0.028,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1632.0,+93/-70,5.292,0.74,9.53,3.46,2166.18,7.945,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1628.0,+76/-55,7.717,0.711,9.71,4.55,611.55,7.01,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1594.0,+66/-62,6.928,0.714,11.11,5.37,6.14,0.586,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1589.0,+71/-61,6.39,0.686,11.29,4.35,2166.18,0.993,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1540.0,+74/-48,9.556,0.586,13.62,5.6,3899.42,55.833,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1511.0,+72/-47,10.725,0.551,15.1,6.15,136.59,3.391,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1505.0,+56/-46,11.115,0.499,15.41,8.34,3547.04,20.963,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1498.0,+74/-70,13.136,0.576,15.78,5.57,4.85,0.629,➖,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1461.0,+74/-61,10.724,0.47,17.9,5.77,4980.83,207.372,✔️,0.0,False,GPU +9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1453.0,+48/-47,12.387,0.383,18.36,10.23,3376.59,2.54,✔️,0.0,False,CPU +10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1441.0,+54/-50,12.586,0.369,19.08,10.6,3372.56,1.658,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1436.0,+78/-68,11.521,0.435,19.38,6.39,4980.83,23.583,✔️,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1434.0,+82/-68,12.117,0.435,19.47,6.97,3899.42,0.983,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1417.0,+57/-46,12.689,0.339,20.53,10.64,3547.04,1.57,✔️,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1392.0,+38/-36,14.551,0.252,22.1,15.71,771.57,2.49,✔️,0.0,False,CPU +15,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1386.0,+68/-69,13.598,0.396,22.44,9.6,4.22,0.554,✔️,0.0,False,GPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1379.0,+55/-51,13.598,0.278,22.92,12.22,3372.56,0.208,✔️,0.0,False,GPU +17,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1377.0,+43/-38,14.476,0.237,23.04,15.6,2034.85,0.805,✔️,0.0,False,CPU +18,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1363.0,+38/-34,14.675,0.219,23.94,17.08,2034.85,0.097,✔️,0.0,False,CPU +19,🧠⚡,Foundation Model,[TabICL (default) [21.21% IMPUTED]](https://arxiv.org/abs/2502.05564),1363.0,+77/-84,13.82,0.352,23.95,8.04,7.27,1.64,✔️,21.21,True,GPU +20,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1361.0,+91/-63,14.241,0.373,24.06,7.26,48.9,43.804,✔️,0.0,False,GPU +21,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1358.0,+34/-30,15.026,0.186,24.26,17.76,771.57,0.322,✔️,0.0,False,CPU +22,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1338.0,+52/-46,15.196,0.208,25.57,15.07,1105.59,2.552,✔️,0.0,False,GPU +23,🧠⚡,Foundation Model,[BetaTabPFN (default) [21.21% IMPUTED]](https://arxiv.org/abs/2502.02527),1338.0,+64/-77,16.243,0.291,25.61,12.57,303.08,1.792,➖,21.21,True,GPU +24,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1334.0,+37/-36,15.598,0.182,25.82,19.14,828.74,2.312,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1334.0,+45/-48,14.832,0.172,25.85,12.27,6147.69,0.485,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1334.0,+73/-66,15.247,0.252,25.86,11.65,6147.69,8.149,✔️,0.0,False,GPU +27,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1322.0,+48/-44,15.903,0.177,26.64,18.03,8.51,0.123,✔️,0.0,False,CPU +28,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1316.0,+34/-34,15.81,0.142,26.99,19.24,828.74,0.336,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1309.0,+55/-53,15.975,0.22,27.51,15.35,12.24,0.154,✔️,0.0,False,GPU +30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1293.0,+45/-41,16.334,0.154,28.52,16.9,1105.59,0.13,✔️,0.0,False,GPU +31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1278.0,+47/-44,17.376,0.114,29.53,19.36,2929.85,0.433,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1271.0,+45/-54,16.4,0.122,30.01,21.32,3704.3,2.074,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1266.0,+46/-41,17.996,0.117,30.36,16.68,16.16,0.307,✔️,0.0,False,GPU +34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1248.0,+51/-48,17.977,0.081,31.51,19.23,2929.85,0.054,✔️,0.0,False,CPU +35,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1243.0,+40/-38,17.096,0.081,31.85,20.77,15.48,4.535,✔️,0.0,False,GPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1234.0,+51/-60,18.45,0.137,32.42,17.55,1459.62,8.056,✔️,0.0,False,CPU +37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1216.0,+49/-51,18.863,0.072,33.57,20.75,8.47,0.054,✔️,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1211.0,+58/-55,20.343,0.1,33.91,22.7,416.39,1.393,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1201.0,+42/-46,18.313,0.065,34.56,22.3,3704.3,0.143,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1180.0,+54/-59,19.638,0.06,35.83,21.57,1459.62,0.89,✔️,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1177.0,+57/-56,21.176,0.069,36.06,23.58,416.39,0.179,✔️,0.0,False,CPU +42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1172.0,+34/-37,19.218,0.015,36.37,31.15,3.05,0.241,✔️,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1158.0,+35/-41,19.446,0.021,37.24,32.62,3.39,0.158,✔️,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1143.0,+54/-53,21.823,0.051,38.11,26.04,572.67,1.421,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1103.0,+45/-45,22.6,0.018,40.42,26.25,572.67,0.143,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1067.0,+50/-59,22.778,0.011,42.4,31.83,11.82,0.147,✔️,0.0,False,CPU +47,🧠⚡,Foundation Model,[TabFlex (default) [21.21% IMPUTED]](https://arxiv.org/abs/2506.05584),1056.0,+82/-88,25.63,0.069,43.0,27.1,1.19,0.15,➖,21.21,True,GPU +48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1054.0,+70/-86,24.777,0.035,43.09,33.03,5.39,1.034,✔️,0.0,False,GPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1009.0,+51/-66,24.915,0.0,45.3,30.44,5.18,0.652,✔️,0.0,False,CPU +50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1006.0,+82/-85,27.446,0.014,45.46,38.22,0.42,0.078,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1000.0,+72/-88,28.256,0.014,45.71,33.26,100.58,1.106,✔️,0.0,False,CPU +52,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+53/-69,26.69,0.002,45.73,39.57,0.47,0.069,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1000.0,+71/-104,31.374,0.043,45.75,31.75,504.17,0.629,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),977.0,+72/-104,31.806,0.028,46.76,33.44,504.17,0.103,✔️,0.0,False,CPU +55,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),904.0,+78/-101,33.41,0.0,49.65,35.57,100.58,0.101,✔️,0.0,False,CPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),899.0,+80/-125,34.29,0.018,49.8,36.54,2.51,0.137,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),677.0,+96/-161,45.738,0.0,55.29,47.88,0.29,0.028,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/winrate_matrix.png.zip index c6e00583dfd438eb74da387bc25b5e147919370d..f48a2908ffceb98098d756a8723595572c59c836 100644 --- a/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_all/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f59494c728d28acb933accda1130a6b956b207a31cfef8a288dd026817f0250b +oid sha256:9d3b10a349a171a8cc51b9e1fbbf6310568b5b656b4f33c6dc82f0ca7cb5a6dc size 2779156 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index 4ea30a35f86dcf66d868b7729478dc04cae1b258..f7b50c516c1e2a80e80eb514d62623dd6d72d60e 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0a12cee638f020d19b1326743905f4ba5529c3a8d480d72b1037f9c5a532f2d1 +oid sha256:ca1a4a5000190b6609c6289e0d55e78e4e8a9e616eaf69dac0a38c6f912c4ee5 size 475350 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip index 125770bbc31eaf9384d5754030aa580a40a2e94f..f4a28cc5820061ead99710faeacdcb3e8e2cffcb 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fafb3ed8788863cc447dfcbc24e9001df833f53b05dca56183099f8dd9d06ec0 -size 1066951 +oid sha256:72b0f5f36dc4f78e8d6104ee66a3adeb5372df60a2424c994ac53e98125cd778 +size 1054118 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_all/tuning-impact-elo.png.zip index 082324ec425eea51d3e5ab0825c8e47b9094aa20..72c3816066eb22d8a5c2a167d8faefeec4953e7a 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fc4b850d6eaa2e2db758ecfc98ad7cc5ed4d941920627357bb1b246398a2e048 +oid sha256:56559afa1e3f99d8c9d1778046fb174af7c451ba01a34032ba8c801fcb37b099 size 176724 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_binary/datasets_all/website_leaderboard.csv index 95e32d1de558cee7178f6d93013845274b3f60f9..5461becbda8817cebaf2b4803e70c2d0cce327cf 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_all/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_all/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1623.0,+89/-65,0.747,8.95,3.5,6.145,442.09,6.381,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1554.0,+100/-58,0.642,11.87,4.35,6.413,1923.88,9.103,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1529.0,+90/-51,0.639,13.04,6.02,7.496,4.91,0.694,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1521.0,+94/-54,0.594,13.46,5.24,7.523,1923.88,1.329,0.0,False,GPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1484.0,+71/-60,0.511,15.42,5.71,9.411,1152.95,1.784,0.0,False,CPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1472.0,+48/-54,0.455,16.12,10.09,10.519,2775.44,8.849,0.0,False,GPU -6,🧠⚡,Foundation Model,[TabICL (default) [3.33% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1448.0,+65/-54,0.468,17.52,6.71,9.774,8.06,1.743,3.33,True,GPU -7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1448.0,+66/-50,0.407,17.54,8.17,11.17,2229.7,1.387,0.0,False,GPU -8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1414.0,+51/-37,0.337,19.61,12.21,12.663,328.96,1.102,0.0,False,CPU -9,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1405.0,+66/-55,0.391,20.18,6.96,9.517,4982.66,358.753,0.0,False,GPU -10,🧠🔁,Neural Network,[LimiX (default) [17.04% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1398.0,+82/-76,0.469,20.62,6.79,13.11,1.92,0.489,17.04,True,GPU -11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1393.0,+72/-56,0.326,20.99,9.69,12.081,2229.7,0.133,0.0,False,GPU -12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1386.0,+60/-51,0.309,21.41,11.06,12.442,1055.6,0.482,0.0,False,CPU -13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1382.0,+61/-52,0.317,21.68,10.08,12.536,1055.6,0.063,0.0,False,CPU -14,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1380.0,+54/-51,0.327,21.79,10.5,12.413,4.27,0.064,0.0,False,CPU -15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1379.0,+49/-54,0.271,21.9,11.37,12.311,2775.44,0.415,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1365.0,+38/-30,0.234,22.8,15.34,13.52,328.96,0.126,0.0,False,CPU -17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1364.0,+62/-51,0.28,22.87,12.42,13.566,472.99,0.621,0.0,False,CPU -18,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1362.0,+62/-55,0.323,22.98,11.51,14.431,250.36,1.708,0.0,False,GPU -19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1349.0,+46/-56,0.236,23.86,11.46,13.197,3383.91,0.51,0.0,False,GPU -20,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1349.0,+90/-92,0.348,23.88,9.54,13.238,3383.91,8.396,0.0,False,GPU -21,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1343.0,+59/-52,0.272,24.26,8.57,11.206,4982.66,42.199,0.0,False,GPU -22,🧠🔁,Neural Network,[Mitra (default) [33.33% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1336.0,+87/-81,0.395,24.76,8.22,14.039,103.86,2.616,33.33,True,GPU -23,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1332.0,+47/-40,0.206,24.99,15.8,14.081,472.99,0.13,0.0,False,CPU -24,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1330.0,+86/-76,0.346,25.18,8.69,14.004,3031.01,27.044,33.33,True,GPU -25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1315.0,+56/-53,0.181,26.15,15.32,14.2,642.33,2.767,0.0,False,GPU -26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1302.0,+49/-40,0.175,27.06,18.1,13.82,2241.65,2.415,0.0,False,CPU -27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1298.0,+64/-63,0.226,27.36,14.59,14.773,8.72,0.126,0.0,False,GPU -28,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1293.0,+48/-43,0.152,27.68,18.0,15.686,1560.02,0.404,0.0,False,CPU -29,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1288.0,+46/-56,0.142,28.06,16.42,15.473,642.33,0.146,0.0,False,GPU -30,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1282.0,+80/-75,0.276,28.42,8.94,15.828,3031.01,0.514,33.33,True,GPU -31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1269.0,+62/-70,0.222,29.33,11.21,14.57,49.67,43.824,0.0,False,GPU -32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1261.0,+48/-47,0.101,29.92,18.94,16.397,1560.02,0.052,0.0,False,CPU -33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1257.0,+46/-40,0.124,30.14,16.66,16.239,13.5,0.311,0.0,False,GPU -34,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1256.0,+58/-56,0.138,30.25,18.4,15.841,559.38,4.08,0.0,False,CPU -35,🧠⚡,Foundation Model,[TabPFNv2 (default) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1256.0,+80/-80,0.275,30.26,11.7,17.22,2.71,0.276,33.33,True,GPU -36,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1253.0,+43/-43,0.094,30.44,20.02,15.156,9.33,1.116,0.0,False,GPU -37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1236.0,+48/-49,0.102,31.6,19.41,17.45,4.98,0.049,0.0,False,CPU -38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1221.0,+66/-56,0.096,32.61,22.15,16.712,1.24,0.11,0.0,False,CPU -39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1215.0,+43/-37,0.066,32.99,22.6,16.017,2241.65,0.136,0.0,False,CPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1186.0,+57/-60,0.052,34.94,22.15,17.87,559.38,0.276,0.0,False,CPU -41,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1183.0,+48/-45,0.045,35.09,25.49,18.346,125.38,0.674,0.0,False,CPU -42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1173.0,+46/-35,0.045,35.75,30.57,17.489,0.81,0.093,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1153.0,+50/-58,0.032,37.05,25.7,19.69,169.75,0.603,0.0,False,CPU -44,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1144.0,+50/-49,0.027,37.62,27.86,19.5,125.38,0.069,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1119.0,+42/-57,0.011,39.16,26.32,20.419,169.75,0.059,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1093.0,+53/-47,0.012,40.68,31.46,20.405,4.95,0.091,0.0,False,CPU -47,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1056.0,+61/-77,0.015,42.74,27.51,21.791,2.56,0.267,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1050.0,+83/-99,0.053,43.03,29.05,27.109,162.68,0.405,0.0,False,CPU -49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1041.0,+102/-102,0.076,43.53,26.39,25.867,0.85,0.129,0.0,False,GPU -50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1028.0,+61/-93,0.017,44.19,31.56,24.522,139.84,3.917,0.0,False,CPU -51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1022.0,+87/-118,0.034,44.49,31.08,27.802,162.68,0.054,0.0,False,CPU -52,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+62/-69,0.007,45.57,38.74,24.603,0.25,0.029,0.0,False,CPU -53,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,975.0,+72/-97,0.015,46.69,38.16,26.891,2.8,0.564,0.0,False,GPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,962.0,+77/-111,0.02,47.26,34.13,29.736,0.84,0.094,0.0,False,CPU -55,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,946.0,+72/-94,0.008,47.92,41.57,26.072,0.2,0.037,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,918.0,+76/-128,0.0,49.05,35.34,29.619,139.84,0.193,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,593.0,+109/-189,0.0,56.3,52.0,44.093,0.15,0.023,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1623.0,+89/-65,6.145,0.747,8.95,3.5,442.09,6.381,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1554.0,+100/-58,6.413,0.642,11.87,4.35,1923.88,9.103,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1529.0,+90/-51,7.496,0.639,13.04,6.02,4.91,0.694,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1521.0,+94/-54,7.523,0.594,13.46,5.24,1923.88,1.329,✔️,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1484.0,+71/-60,9.411,0.511,15.42,5.71,1152.95,1.784,✔️,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1472.0,+48/-54,10.519,0.455,16.12,10.09,2775.44,8.849,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabICL (default) [3.33% IMPUTED]](https://arxiv.org/abs/2502.05564),1448.0,+65/-54,9.774,0.468,17.52,6.71,8.06,1.743,✔️,3.33,True,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1448.0,+66/-50,11.17,0.407,17.54,8.17,2229.7,1.387,✔️,0.0,False,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1414.0,+51/-37,12.663,0.337,19.61,12.21,328.96,1.102,✔️,0.0,False,CPU +9,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1405.0,+66/-55,9.517,0.391,20.18,6.96,4982.66,358.753,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[LimiX (default) [17.04% IMPUTED]](https://arxiv.org/abs/2509.03505),1398.0,+82/-76,13.11,0.469,20.62,6.79,1.92,0.489,➖,17.04,True,GPU +11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1393.0,+72/-56,12.081,0.326,20.99,9.69,2229.7,0.133,✔️,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1386.0,+60/-51,12.442,0.309,21.41,11.06,1055.6,0.482,✔️,0.0,False,CPU +13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1382.0,+61/-52,12.536,0.317,21.68,10.08,1055.6,0.063,✔️,0.0,False,CPU +14,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1380.0,+54/-51,12.413,0.327,21.79,10.5,4.27,0.064,✔️,0.0,False,CPU +15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1379.0,+49/-54,12.311,0.271,21.9,11.37,2775.44,0.415,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1365.0,+38/-30,13.52,0.234,22.8,15.34,328.96,0.126,✔️,0.0,False,CPU +17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1364.0,+62/-51,13.566,0.28,22.87,12.42,472.99,0.621,✔️,0.0,False,CPU +18,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1362.0,+62/-55,14.431,0.323,22.98,11.51,250.36,1.708,➖,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1349.0,+46/-56,13.197,0.236,23.86,11.46,3383.91,0.51,✔️,0.0,False,GPU +20,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1349.0,+90/-92,13.238,0.348,23.88,9.54,3383.91,8.396,✔️,0.0,False,GPU +21,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1343.0,+59/-52,11.206,0.272,24.26,8.57,4982.66,42.199,✔️,0.0,False,GPU +22,🧠⚡,Foundation Model,[Mitra (default) [33.33% IMPUTED]](https://arxiv.org/abs/2510.21204),1336.0,+87/-81,14.039,0.395,24.76,8.22,103.86,2.616,✔️,33.33,True,GPU +23,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1332.0,+47/-40,14.081,0.206,24.99,15.8,472.99,0.13,✔️,0.0,False,CPU +24,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1330.0,+86/-76,14.004,0.346,25.18,8.69,3031.01,27.044,✔️,33.33,True,GPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1315.0,+56/-53,14.2,0.181,26.15,15.32,642.33,2.767,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1302.0,+49/-40,13.82,0.175,27.06,18.1,2241.65,2.415,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1298.0,+64/-63,14.773,0.226,27.36,14.59,8.72,0.126,✔️,0.0,False,GPU +28,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1293.0,+48/-43,15.686,0.152,27.68,18.0,1560.02,0.404,✔️,0.0,False,CPU +29,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1288.0,+46/-56,15.473,0.142,28.06,16.42,642.33,0.146,✔️,0.0,False,GPU +30,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1282.0,+80/-75,15.828,0.276,28.42,8.94,3031.01,0.514,✔️,33.33,True,GPU +31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1269.0,+62/-70,14.57,0.222,29.33,11.21,49.67,43.824,✔️,0.0,False,GPU +32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1261.0,+48/-47,16.397,0.101,29.92,18.94,1560.02,0.052,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1257.0,+46/-40,16.239,0.124,30.14,16.66,13.5,0.311,✔️,0.0,False,GPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1256.0,+58/-56,15.841,0.138,30.25,18.4,559.38,4.08,✔️,0.0,False,CPU +35,🧠⚡,Foundation Model,[TabPFNv2 (default) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1256.0,+80/-80,17.22,0.275,30.26,11.7,2.71,0.276,✔️,33.33,True,GPU +36,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1253.0,+43/-43,15.156,0.094,30.44,20.02,9.33,1.116,✔️,0.0,False,GPU +37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1236.0,+48/-49,17.45,0.102,31.6,19.41,4.98,0.049,✔️,0.0,False,CPU +38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1221.0,+66/-56,16.712,0.096,32.61,22.15,1.24,0.11,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1215.0,+43/-37,16.017,0.066,32.99,22.6,2241.65,0.136,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1186.0,+57/-60,17.87,0.052,34.94,22.15,559.38,0.276,✔️,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1183.0,+48/-45,18.346,0.045,35.09,25.49,125.38,0.674,✔️,0.0,False,CPU +42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1173.0,+46/-35,17.489,0.045,35.75,30.57,0.81,0.093,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1153.0,+50/-58,19.69,0.032,37.05,25.7,169.75,0.603,✔️,0.0,False,CPU +44,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1144.0,+50/-49,19.5,0.027,37.62,27.86,125.38,0.069,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1119.0,+42/-57,20.419,0.011,39.16,26.32,169.75,0.059,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1093.0,+53/-47,20.405,0.012,40.68,31.46,4.95,0.091,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1056.0,+61/-77,21.791,0.015,42.74,27.51,2.56,0.267,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1050.0,+83/-99,27.109,0.053,43.03,29.05,162.68,0.405,✔️,0.0,False,CPU +49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1041.0,+102/-102,25.867,0.076,43.53,26.39,0.85,0.129,➖,0.0,False,GPU +50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1028.0,+61/-93,24.522,0.017,44.19,31.56,139.84,3.917,✔️,0.0,False,CPU +51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1022.0,+87/-118,27.802,0.034,44.49,31.08,162.68,0.054,✔️,0.0,False,CPU +52,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+62/-69,24.603,0.007,45.57,38.74,0.25,0.029,✔️,0.0,False,CPU +53,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),975.0,+72/-97,26.891,0.015,46.69,38.16,2.8,0.564,✔️,0.0,False,GPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),962.0,+77/-111,29.736,0.02,47.26,34.13,0.84,0.094,✔️,0.0,False,CPU +55,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),946.0,+72/-94,26.072,0.008,47.92,41.57,0.2,0.037,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),918.0,+76/-128,29.619,0.0,49.05,35.34,139.84,0.193,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),593.0,+109/-189,44.093,0.0,56.3,52.0,0.15,0.023,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_all/winrate_matrix.png.zip index 5924697994483de3493806fe2b45026875c24d76..7b76b25a74a9c1f16d3326e8b8483f059be79525 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b449687c2f1243b1440732d53a6a346bdb503d7e4ba27a67622c42e676faee9b +oid sha256:c89e5ad4a6eee54e4760b89a82157cbe28b63f8e0263ee8da1d9b07a9cd34b29 size 2760741 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index dbadea366a0375a274c27ad08b1122827f183f00..2b28a717a097c6ba349ae48981e7d3e448bd2aab 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c6ea72dde0d62e5ce043218bb592eda595ea2782a92310ac853e50152402be14 +oid sha256:70f3377e354ad69aef50ae476d5d3f30ac41432724ac61f7970814a078181872 size 430512 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip index 7e4d53dc7adf17e93af26c3583c5b7df059f07b3..46890467250e8e4eaa06739f177d4287b0e2cdfc 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:de3c7513860b16aafc489a52272dea2dec3849db874fd96ada515da4b244302b -size 1001152 +oid sha256:40f924465c075a5bcdf59f115279d92fa0b195ecc6ed43b4e2de6f33004a2d00 +size 937488 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/tuning-impact-elo.png.zip index 3961b8a670fc5e5f11e4a150d951a83222a695ea..7055df9e05fc9288f8bde2f4b850b754ac91584b 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5f0153d85429faf4f0668bc0ab04de0abf1a1d06d62b0f7d739cdf10d994a072 +oid sha256:7c884aafd87b05486a559e8b56e19ec4ddd5898ff247f1fc2bebd8d261949043 size 134896 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/website_leaderboard.csv index de5864b86e9c7527b58e8fd133b851545edd1a45..8600c4c711836435bd5453accf4ca2d24c2526fe 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/website_leaderboard.csv @@ -1,55 +1,55 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1952.0,+156/-57,0.904,5.02,2.43,2.22,275.07,2.448,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1887.0,+175/-63,0.85,6.53,2.71,2.498,325.91,2.156,0.0,False,CPU -2,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1744.0,+147/-88,0.63,11.19,5.89,4.733,619.15,0.198,0.0,False,CPU -3,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1739.0,+142/-86,0.636,11.38,4.9,5.026,619.15,0.016,0.0,False,CPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1736.0,+118/-39,0.572,11.51,8.06,5.448,1633.8,2.053,0.0,False,GPU -5,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1722.0,+131/-79,0.63,12.09,5.58,4.771,2.21,0.025,0.0,False,CPU -6,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1705.0,+146/-89,0.544,12.8,7.1,7.516,110.13,0.386,0.0,False,CPU -7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1688.0,+146/-81,0.524,13.49,8.59,7.932,86.85,0.668,0.0,False,CPU -8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1668.0,+198/-135,0.512,14.38,5.61,7.278,804.65,0.71,0.0,False,GPU -9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1667.0,+146/-138,0.543,14.46,7.14,6.915,2139.17,6.41,0.0,False,GPU -10,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1650.0,+204/-120,0.489,15.23,5.4,8.389,842.24,15.119,0.0,False,GPU -11,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1638.0,+126/-65,0.372,15.81,8.84,7.036,1633.8,0.088,0.0,False,GPU -12,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1628.0,+117/-76,0.419,16.27,11.67,8.384,110.13,0.061,0.0,False,CPU -13,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1612.0,+196/-154,0.456,17.02,6.32,7.986,804.65,0.063,0.0,False,GPU -14,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1600.0,+193/-114,0.42,17.65,6.9,9.041,842.24,1.465,0.0,False,GPU -15,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1572.0,+156/-126,0.417,19.02,8.0,8.638,2.3,0.896,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1568.0,+158/-86,0.323,19.23,12.24,9.749,86.85,0.078,0.0,False,CPU -17,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1556.0,+165/-162,0.42,19.88,8.63,9.22,8.53,3.578,0.0,False,GPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1542.0,+120/-80,0.254,20.59,13.66,9.79,2139.17,0.349,0.0,False,GPU -19,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1527.0,+170/-152,0.346,21.37,7.59,9.027,4860.88,475.791,0.0,False,GPU -20,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1515.0,+96/-110,0.247,22.0,14.66,9.128,1156.69,1.558,0.0,False,CPU -21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1480.0,+146/-104,0.195,23.86,15.28,10.686,572.82,1.935,0.0,False,GPU -22,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1465.0,+127/-110,0.171,24.69,18.15,12.878,270.2,0.192,0.0,False,CPU -23,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1451.0,+121/-141,0.227,25.47,15.18,10.869,0.49,0.053,0.0,False,CPU -24,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1444.0,+132/-115,0.161,25.86,19.53,12.706,572.82,0.109,0.0,False,GPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1443.0,+149/-178,0.243,25.88,14.7,10.498,5.5,0.074,0.0,False,GPU -26,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1422.0,+105/-126,0.124,27.06,20.82,11.44,371.59,2.386,0.0,False,CPU -27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1420.0,+83/-65,0.079,27.19,24.01,10.931,6.41,0.247,0.0,False,GPU -28,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1407.0,+118/-135,0.136,27.88,20.37,13.968,96.61,0.754,0.0,False,GPU -29,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1393.0,+126/-107,0.093,28.67,24.4,14.005,270.2,0.015,0.0,False,CPU -30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1371.0,+97/-128,0.072,29.85,23.72,11.539,1156.69,0.054,0.0,False,CPU -31,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1365.0,+128/-130,0.085,30.14,19.07,12.77,4860.88,42.641,0.0,False,GPU -32,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1351.0,+127/-111,0.052,30.89,26.74,15.839,1.07,0.018,0.0,False,CPU -33,🧠🔁,Neural Network,[LimiX (default) [55.56% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1348.0,+200/-227,0.233,31.07,13.41,18.235,0.63,0.019,55.56,True,GPU -34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1337.0,+97/-131,0.067,31.63,28.44,12.469,0.29,0.033,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1313.0,+52/-59,0.002,32.88,31.05,14.156,42.95,0.226,0.0,False,CPU -36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1308.0,+86/-82,0.009,33.14,30.03,13.961,8.34,0.25,0.0,False,GPU -37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1275.0,+59/-86,0.0,34.77,32.44,13.465,40.36,0.318,0.0,False,CPU -38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1274.0,+76/-73,0.005,34.81,31.22,15.136,42.95,0.021,0.0,False,CPU -39,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1274.0,+171/-184,0.032,34.82,21.22,17.027,47.03,45.876,0.0,False,GPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1272.0,+81/-116,0.037,34.91,29.62,15.039,371.59,0.102,0.0,False,CPU -41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1220.0,+68/-96,0.0,37.36,35.66,15.181,1.86,0.046,0.0,False,CPU -42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1214.0,+64/-68,0.0,37.63,35.91,15.182,40.36,0.028,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1199.0,+131/-210,0.043,38.31,32.82,18.054,1.52,0.115,0.0,False,CPU -44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1157.0,+147/-229,0.008,40.12,35.81,24.041,43.64,0.085,0.0,False,CPU -45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1138.0,+96/-184,0.011,40.89,31.9,20.737,513.06,78.02,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1107.0,+165/-222,0.005,42.07,38.55,25.035,43.64,0.022,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+83/-194,0.0,45.68,44.94,22.283,0.12,0.014,0.0,False,CPU -48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,983.0,+131/-215,0.0,46.17,44.55,26.738,0.22,0.024,0.0,False,CPU -49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,969.0,+198/-439,0.025,46.56,40.45,28.52,2.62,0.186,0.0,False,GPU -50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,916.0,+95/-189,0.0,47.91,47.06,30.36,0.24,0.036,0.0,False,GPU -51,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,912.0,+93/-235,0.0,48.0,47.54,24.149,0.1,0.021,0.0,False,CPU -52,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,884.0,+82/-127,0.0,48.62,48.24,27.802,513.06,2.568,0.0,False,CPU -53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,508.0,+104/-387,0.0,53.27,53.24,45.598,0.11,0.087,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1952.0,+156/-57,2.22,0.904,5.02,2.43,275.07,2.448,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1887.0,+175/-63,2.498,0.85,6.53,2.71,325.91,2.156,✔️,0.0,False,CPU +2,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1744.0,+147/-88,4.733,0.63,11.19,5.89,619.15,0.198,✔️,0.0,False,CPU +3,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1739.0,+142/-86,5.026,0.636,11.38,4.9,619.15,0.016,✔️,0.0,False,CPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1736.0,+118/-39,5.448,0.572,11.51,8.06,1633.8,2.053,✔️,0.0,False,GPU +5,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1722.0,+131/-79,4.771,0.63,12.09,5.58,2.21,0.025,✔️,0.0,False,CPU +6,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1705.0,+146/-89,7.516,0.544,12.8,7.1,110.13,0.386,✔️,0.0,False,CPU +7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1688.0,+146/-81,7.932,0.524,13.49,8.59,86.85,0.668,✔️,0.0,False,CPU +8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1668.0,+198/-135,7.278,0.512,14.38,5.61,804.65,0.71,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1667.0,+146/-138,6.915,0.543,14.46,7.14,2139.17,6.41,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1650.0,+204/-120,8.389,0.489,15.23,5.4,842.24,15.119,✔️,0.0,False,GPU +11,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1638.0,+126/-65,7.036,0.372,15.81,8.84,1633.8,0.088,✔️,0.0,False,GPU +12,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1628.0,+117/-76,8.384,0.419,16.27,11.67,110.13,0.061,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1612.0,+196/-154,7.986,0.456,17.02,6.32,804.65,0.063,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1600.0,+193/-114,9.041,0.42,17.65,6.9,842.24,1.465,✔️,0.0,False,GPU +15,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1572.0,+156/-126,8.638,0.417,19.02,8.0,2.3,0.896,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1568.0,+158/-86,9.749,0.323,19.23,12.24,86.85,0.078,✔️,0.0,False,CPU +17,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1556.0,+165/-162,9.22,0.42,19.88,8.63,8.53,3.578,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1542.0,+120/-80,9.79,0.254,20.59,13.66,2139.17,0.349,✔️,0.0,False,GPU +19,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1527.0,+170/-152,9.027,0.346,21.37,7.59,4860.88,475.791,✔️,0.0,False,GPU +20,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1515.0,+96/-110,9.128,0.247,22.0,14.66,1156.69,1.558,✔️,0.0,False,CPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1480.0,+146/-104,10.686,0.195,23.86,15.28,572.82,1.935,✔️,0.0,False,GPU +22,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1465.0,+127/-110,12.878,0.171,24.69,18.15,270.2,0.192,✔️,0.0,False,CPU +23,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1451.0,+121/-141,10.869,0.227,25.47,15.18,0.49,0.053,✔️,0.0,False,CPU +24,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1444.0,+132/-115,12.706,0.161,25.86,19.53,572.82,0.109,✔️,0.0,False,GPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1443.0,+149/-178,10.498,0.243,25.88,14.7,5.5,0.074,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1422.0,+105/-126,11.44,0.124,27.06,20.82,371.59,2.386,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1420.0,+83/-65,10.931,0.079,27.19,24.01,6.41,0.247,✔️,0.0,False,GPU +28,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1407.0,+118/-135,13.968,0.136,27.88,20.37,96.61,0.754,➖,0.0,False,GPU +29,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1393.0,+126/-107,14.005,0.093,28.67,24.4,270.2,0.015,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1371.0,+97/-128,11.539,0.072,29.85,23.72,1156.69,0.054,✔️,0.0,False,CPU +31,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1365.0,+128/-130,12.77,0.085,30.14,19.07,4860.88,42.641,✔️,0.0,False,GPU +32,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1351.0,+127/-111,15.839,0.052,30.89,26.74,1.07,0.018,✔️,0.0,False,CPU +33,🧠⚡,Foundation Model,[LimiX (default) [55.56% IMPUTED]](https://arxiv.org/abs/2509.03505),1348.0,+200/-227,18.235,0.233,31.07,13.41,0.63,0.019,➖,55.56,True,GPU +34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1337.0,+97/-131,12.469,0.067,31.63,28.44,0.29,0.033,✔️,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1313.0,+52/-59,14.156,0.002,32.88,31.05,42.95,0.226,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1308.0,+86/-82,13.961,0.009,33.14,30.03,8.34,0.25,✔️,0.0,False,GPU +37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1275.0,+59/-86,13.465,0.0,34.77,32.44,40.36,0.318,✔️,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1274.0,+76/-73,15.136,0.005,34.81,31.22,42.95,0.021,✔️,0.0,False,CPU +39,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1274.0,+171/-184,17.027,0.032,34.82,21.22,47.03,45.876,✔️,0.0,False,GPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1272.0,+81/-116,15.039,0.037,34.91,29.62,371.59,0.102,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1220.0,+68/-96,15.181,0.0,37.36,35.66,1.86,0.046,✔️,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1214.0,+64/-68,15.182,0.0,37.63,35.91,40.36,0.028,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1199.0,+131/-210,18.054,0.043,38.31,32.82,1.52,0.115,✔️,0.0,False,CPU +44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1157.0,+147/-229,24.041,0.008,40.12,35.81,43.64,0.085,✔️,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1138.0,+96/-184,20.737,0.011,40.89,31.9,513.06,78.02,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1107.0,+165/-222,25.035,0.005,42.07,38.55,43.64,0.022,✔️,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+83/-194,22.283,0.0,45.68,44.94,0.12,0.014,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),983.0,+131/-215,26.738,0.0,46.17,44.55,0.22,0.024,✔️,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),969.0,+198/-439,28.52,0.025,46.56,40.45,2.62,0.186,✔️,0.0,False,GPU +50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),916.0,+95/-189,30.36,0.0,47.91,47.06,0.24,0.036,➖,0.0,False,GPU +51,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),912.0,+93/-235,24.149,0.0,48.0,47.54,0.1,0.021,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),884.0,+82/-127,27.802,0.0,48.62,48.24,513.06,2.568,✔️,0.0,False,CPU +53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),508.0,+104/-387,45.598,0.0,53.27,53.24,0.11,0.087,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/winrate_matrix.png.zip index 2323f41caf6eab6a8c6ffe2c219fedb06ae9a035..ff057e97e1002941df9cb2c12eabeaf6c999273b 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:48df8f16adb82f680a2411a7e25e4c514c07d25693c53ec233f93accb56e50ba +oid sha256:e1f3549cd30e501817685fc7855e5995bfaf4bd76a68595093515a9924971853 size 2264366 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index aabdf1ae3bc40987980c87c9dc36e13c78e70554..ce386c9526dbe7732ad2ebfffe93a5b6c4539aeb 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6677ec779d8216c9b5e173f90dcf1f917b1e61bb84d47d1bc411d0fb8e881c14 +oid sha256:ea778aab0e2d8b7d4f5d32deed403739b54a9b2e2d552726af97d1bddfae6b25 size 482352 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip index 574b4fc49dc4c952928559a96b32c23999b3326a..c99661ff7e4dd0b302acd30f15b22c73e11465d9 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:40402863f3ee036d58d994c79cfa7a80d5c4f21d8608d37571bda48046d5749c -size 1039896 +oid sha256:2897ea1ced63aaca1e16be267bd05e09763eef8947f6898124bb07315d3249fb +size 1038898 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_small/tuning-impact-elo.png.zip index e4dbc32c40fbabd1dd32e7649761d94b55f672ab..fa00e8ecc45f77952bb95a01ec33ff667fa57090 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4ee83e0e3ff5c7fd719bad2817bc4d7cb989b1b658ee43de6dc316a120c999ee +oid sha256:159170cbc24b8b4875788bf77a9b0a9165ee5a58d198217e75684db9128995a1 size 190540 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_binary/datasets_small/website_leaderboard.csv index e7bb7fbb02b28eba762c21f291465146c1316c1d..ce1c168e8451c3bf5ad5147a53098f47666dcd67 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_small/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_small/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1573.0,+135/-64,0.7,10.43,4.02,5.566,2052.28,8.252,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1573.0,+93/-51,0.72,10.43,5.44,7.007,5.96,0.554,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1568.0,+103/-72,0.675,10.63,4.33,7.828,593.57,8.084,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1545.0,+122/-58,0.658,11.64,4.75,6.873,2052.28,1.266,0.0,False,GPU -4,🧠🔁,Neural Network,[Mitra (default) [4.76% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1479.0,+89/-54,0.564,14.91,6.08,10.506,143.16,3.761,4.76,True,GPU -5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1466.0,+105/-56,0.494,15.6,6.44,10.456,3422.16,45.847,4.76,True,GPU -6,🧠🔁,Neural Network,[LimiX (default) [0.53% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1466.0,+94/-65,0.565,15.65,5.59,10.913,3.78,0.54,0.53,True,GPU -7,🧠⚡,Foundation Model,[TabICL (default) [4.76% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1450.0,+72/-70,0.477,16.52,6.13,10.012,7.78,1.706,4.76,True,GPU -8,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1423.0,+68/-46,0.391,18.1,11.32,12.692,2950.72,15.595,0.0,False,GPU -9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1410.0,+74/-55,0.353,18.88,10.15,12.838,2457.54,1.658,0.0,False,GPU -10,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1404.0,+80/-70,0.36,19.23,10.86,12.373,1921.63,1.718,0.0,False,CPU -11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1398.0,+79/-72,0.394,19.65,6.71,9.728,5351.68,268.579,0.0,False,GPU -12,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1388.0,+125/-63,0.394,20.24,6.64,13.062,3422.16,0.983,4.76,True,GPU -13,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1382.0,+76/-66,0.387,20.62,9.69,14.629,317.91,1.983,0.0,False,GPU -14,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1366.0,+84/-73,0.344,21.63,6.93,10.535,5351.68,37.621,0.0,False,GPU -15,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1357.0,+44/-36,0.244,22.23,14.91,14.691,512.57,1.18,0.0,False,CPU -16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1351.0,+73/-54,0.26,22.59,12.56,13.836,2457.54,0.178,0.0,False,GPU -17,🧠⚡,Foundation Model,[TabPFNv2 (default) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1349.0,+99/-81,0.393,22.76,8.85,15.05,4.05,0.421,4.76,True,GPU -18,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1325.0,+37/-34,0.178,24.31,17.21,15.136,512.57,0.243,0.0,False,CPU -19,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1322.0,+74/-59,0.209,24.48,12.95,14.571,2950.72,1.225,0.0,False,GPU -20,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1312.0,+80/-64,0.215,25.21,10.71,14.657,4806.04,0.512,0.0,False,GPU -21,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1303.0,+47/-43,0.159,25.78,17.73,15.746,1096.5,0.584,0.0,False,CPU -22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1301.0,+59/-44,0.185,25.92,16.88,15.688,4.81,0.08,0.0,False,CPU -23,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1299.0,+55/-42,0.168,26.08,18.41,15.755,1096.5,0.074,0.0,False,CPU -24,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1294.0,+86/-77,0.289,26.4,9.3,13.517,51.83,43.615,0.0,False,GPU -25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1284.0,+46/-66,0.169,27.09,15.33,15.706,711.84,2.983,0.0,False,GPU -26,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1282.0,+44/-47,0.155,27.18,18.29,16.16,693.49,1.398,0.0,False,CPU -27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1273.0,+67/-61,0.208,27.82,14.52,16.605,10.08,0.152,0.0,False,GPU -28,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1272.0,+102/-109,0.254,27.92,11.14,15.948,4806.04,8.461,0.0,False,GPU -29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1260.0,+46/-48,0.123,28.71,17.9,16.89,1634.54,0.422,0.0,False,CPU -30,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1260.0,+43/-37,0.1,28.73,18.62,16.522,693.49,0.194,0.0,False,CPU -31,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1259.0,+59/-64,0.166,28.77,13.98,17.215,14.37,0.334,0.0,False,GPU -32,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1258.0,+49/-61,0.127,28.81,15.35,16.659,711.84,0.162,0.0,False,GPU -33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1252.0,+77/-56,0.133,29.21,20.12,15.831,2665.36,2.619,0.0,False,CPU -34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1239.0,+47/-48,0.095,30.14,17.24,17.421,1634.54,0.054,0.0,False,CPU -35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1219.0,+60/-79,0.136,31.5,17.51,17.727,644.55,5.974,0.0,False,CPU -36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1218.0,+46/-53,0.11,31.57,17.33,18.14,7.32,0.054,0.0,False,CPU -37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1215.0,+52/-53,0.084,31.76,18.67,16.966,10.89,2.249,0.0,False,GPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1178.0,+69/-47,0.059,34.19,22.12,17.936,2665.36,0.143,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1170.0,+61/-84,0.055,34.69,19.95,19.083,644.55,0.398,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1163.0,+65/-80,0.064,35.18,23.33,20.437,203.15,0.968,0.0,False,CPU -41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1158.0,+56/-58,0.033,35.52,27.53,19.215,1.94,0.121,0.0,False,CPU -42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1127.0,+62/-79,0.038,37.43,25.36,21.351,203.15,0.091,0.0,False,CPU -43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1127.0,+40/-53,0.02,37.45,31.55,19.64,2.44,0.121,0.0,False,CPU -44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1106.0,+57/-92,0.042,38.77,23.91,22.062,484.22,1.025,0.0,False,CPU -45,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1073.0,+106/-126,0.109,40.71,22.04,23.941,1.3,0.15,0.0,False,GPU -46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1070.0,+52/-89,0.011,40.88,24.6,22.683,484.22,0.086,0.0,False,CPU -47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1052.0,+62/-82,0.017,41.89,29.86,22.644,9.39,0.13,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1020.0,+92/-135,0.067,43.63,26.7,28.424,278.97,0.873,0.0,False,CPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1011.0,+74/-90,0.0,44.09,25.59,23.393,2.97,0.388,0.0,False,CPU -50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+56/-113,0.01,44.63,36.01,25.597,0.42,0.041,0.0,False,CPU -51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,997.0,+93/-135,0.044,44.79,28.48,28.987,278.97,0.103,0.0,False,CPU -52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,988.0,+78/-117,0.019,45.2,31.26,26.144,90.71,1.623,0.0,False,CPU -53,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,984.0,+80/-114,0.01,45.39,36.59,26.193,3.71,0.919,0.0,False,GPU -54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,962.0,+82/-136,0.011,46.44,38.56,26.896,0.33,0.043,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,951.0,+89/-163,0.029,46.94,30.75,31.021,1.48,0.139,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,922.0,+83/-149,0.001,48.17,31.33,30.397,90.71,0.125,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,616.0,+111/-175,0.0,55.88,50.04,43.448,0.22,0.02,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1573.0,+135/-64,5.566,0.7,10.43,4.02,2052.28,8.252,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1573.0,+93/-51,7.007,0.72,10.43,5.44,5.96,0.554,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1568.0,+103/-72,7.828,0.675,10.63,4.33,593.57,8.084,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1545.0,+122/-58,6.873,0.658,11.64,4.75,2052.28,1.266,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[Mitra (default) [4.76% IMPUTED]](https://arxiv.org/abs/2510.21204),1479.0,+89/-54,10.506,0.564,14.91,6.08,143.16,3.761,✔️,4.76,True,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1466.0,+105/-56,10.456,0.494,15.6,6.44,3422.16,45.847,✔️,4.76,True,GPU +6,🧠⚡,Foundation Model,[LimiX (default) [0.53% IMPUTED]](https://arxiv.org/abs/2509.03505),1466.0,+94/-65,10.913,0.565,15.65,5.59,3.78,0.54,➖,0.53,True,GPU +7,🧠⚡,Foundation Model,[TabICL (default) [4.76% IMPUTED]](https://arxiv.org/abs/2502.05564),1450.0,+72/-70,10.012,0.477,16.52,6.13,7.78,1.706,✔️,4.76,True,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1423.0,+68/-46,12.692,0.391,18.1,11.32,2950.72,15.595,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1410.0,+74/-55,12.838,0.353,18.88,10.15,2457.54,1.658,✔️,0.0,False,GPU +10,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1404.0,+80/-70,12.373,0.36,19.23,10.86,1921.63,1.718,✔️,0.0,False,CPU +11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1398.0,+79/-72,9.728,0.394,19.65,6.71,5351.68,268.579,✔️,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1388.0,+125/-63,13.062,0.394,20.24,6.64,3422.16,0.983,✔️,4.76,True,GPU +13,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1382.0,+76/-66,14.629,0.387,20.62,9.69,317.91,1.983,➖,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1366.0,+84/-73,10.535,0.344,21.63,6.93,5351.68,37.621,✔️,0.0,False,GPU +15,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1357.0,+44/-36,14.691,0.244,22.23,14.91,512.57,1.18,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1351.0,+73/-54,13.836,0.26,22.59,12.56,2457.54,0.178,✔️,0.0,False,GPU +17,🧠⚡,Foundation Model,[TabPFNv2 (default) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1349.0,+99/-81,15.05,0.393,22.76,8.85,4.05,0.421,✔️,4.76,True,GPU +18,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1325.0,+37/-34,15.136,0.178,24.31,17.21,512.57,0.243,✔️,0.0,False,CPU +19,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1322.0,+74/-59,14.571,0.209,24.48,12.95,2950.72,1.225,✔️,0.0,False,GPU +20,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1312.0,+80/-64,14.657,0.215,25.21,10.71,4806.04,0.512,✔️,0.0,False,GPU +21,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1303.0,+47/-43,15.746,0.159,25.78,17.73,1096.5,0.584,✔️,0.0,False,CPU +22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1301.0,+59/-44,15.688,0.185,25.92,16.88,4.81,0.08,✔️,0.0,False,CPU +23,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1299.0,+55/-42,15.755,0.168,26.08,18.41,1096.5,0.074,✔️,0.0,False,CPU +24,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1294.0,+86/-77,13.517,0.289,26.4,9.3,51.83,43.615,✔️,0.0,False,GPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1284.0,+46/-66,15.706,0.169,27.09,15.33,711.84,2.983,✔️,0.0,False,GPU +26,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1282.0,+44/-47,16.16,0.155,27.18,18.29,693.49,1.398,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1273.0,+67/-61,16.605,0.208,27.82,14.52,10.08,0.152,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1272.0,+102/-109,15.948,0.254,27.92,11.14,4806.04,8.461,✔️,0.0,False,GPU +29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1260.0,+46/-48,16.89,0.123,28.71,17.9,1634.54,0.422,✔️,0.0,False,CPU +30,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1260.0,+43/-37,16.522,0.1,28.73,18.62,693.49,0.194,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1259.0,+59/-64,17.215,0.166,28.77,13.98,14.37,0.334,✔️,0.0,False,GPU +32,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1258.0,+49/-61,16.659,0.127,28.81,15.35,711.84,0.162,✔️,0.0,False,GPU +33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1252.0,+77/-56,15.831,0.133,29.21,20.12,2665.36,2.619,✔️,0.0,False,CPU +34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1239.0,+47/-48,17.421,0.095,30.14,17.24,1634.54,0.054,✔️,0.0,False,CPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1219.0,+60/-79,17.727,0.136,31.5,17.51,644.55,5.974,✔️,0.0,False,CPU +36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1218.0,+46/-53,18.14,0.11,31.57,17.33,7.32,0.054,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1215.0,+52/-53,16.966,0.084,31.76,18.67,10.89,2.249,✔️,0.0,False,GPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1178.0,+69/-47,17.936,0.059,34.19,22.12,2665.36,0.143,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1170.0,+61/-84,19.083,0.055,34.69,19.95,644.55,0.398,✔️,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1163.0,+65/-80,20.437,0.064,35.18,23.33,203.15,0.968,✔️,0.0,False,CPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1158.0,+56/-58,19.215,0.033,35.52,27.53,1.94,0.121,✔️,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1127.0,+62/-79,21.351,0.038,37.43,25.36,203.15,0.091,✔️,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1127.0,+40/-53,19.64,0.02,37.45,31.55,2.44,0.121,✔️,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1106.0,+57/-92,22.062,0.042,38.77,23.91,484.22,1.025,✔️,0.0,False,CPU +45,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1073.0,+106/-126,23.941,0.109,40.71,22.04,1.3,0.15,➖,0.0,False,GPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1070.0,+52/-89,22.683,0.011,40.88,24.6,484.22,0.086,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1052.0,+62/-82,22.644,0.017,41.89,29.86,9.39,0.13,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1020.0,+92/-135,28.424,0.067,43.63,26.7,278.97,0.873,✔️,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1011.0,+74/-90,23.393,0.0,44.09,25.59,2.97,0.388,✔️,0.0,False,CPU +50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+56/-113,25.597,0.01,44.63,36.01,0.42,0.041,✔️,0.0,False,CPU +51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),997.0,+93/-135,28.987,0.044,44.79,28.48,278.97,0.103,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),988.0,+78/-117,26.144,0.019,45.2,31.26,90.71,1.623,✔️,0.0,False,CPU +53,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),984.0,+80/-114,26.193,0.01,45.39,36.59,3.71,0.919,✔️,0.0,False,GPU +54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),962.0,+82/-136,26.896,0.011,46.44,38.56,0.33,0.043,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),951.0,+89/-163,31.021,0.029,46.94,30.75,1.48,0.139,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),922.0,+83/-149,30.397,0.001,48.17,31.33,90.71,0.125,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),616.0,+111/-175,43.448,0.0,55.88,50.04,0.22,0.02,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_small/winrate_matrix.png.zip index e2519a06c163102ae8d06a7d31344079f12a0e57..9c6aa5d338241b9108cb80ed3f8095eb0fa0365d 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5650f2ac484e1d96f2825a0291ed23243ac70fe438a913bf3c7fbe640c6083d9 +oid sha256:954c29c843e4042c9765e5e6b356f70c8d3944a0e732ba6b684df6b369b532c6 size 2725579 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 1458b1d6553597fc28fca8eb22339f9fa181af02..ce465ded0d2895af4a661e734b3907ad4601eec8 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:607e71c3612163bf5be8f74d67bc57a128bcad60c555c4733f81cbac7bc63712 +oid sha256:f64bdfe903509a16e33d5258f330a5a2c1b23c43062254218bd3d7941080cbf5 size 483576 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip index 5ef49fd18a5373d8e83ba08e66b60c4cfe17fc36..083bd614a024246ef718acd3b04043fbf8c06841 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:82071b0ea6a221a8aac580dae2637db883e8a0a4a2ec23edf47d19dfd9fb9b7f -size 1038997 +oid sha256:05ac7e35d5f3c73493f338740b208a7632e5ec2d446fd4b75de22099d1c55d63 +size 1037072 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip index af31c3e7a27565e4c00476c1b27d74c47fb912c2..8d9030cdf3201065790e098bb8d367c0f860f984 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4f25ee4c9c0f0ca2e61ec55a72d03d0b00973ada673f4110bdcc3702dc58e268 +oid sha256:658f1e0d88cc47bb0a0389366adf2dae5f864cae524e6392896a0fdcd344dd86 size 131179 diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/website_leaderboard.csv index f3f6c9e65b8ce5c20388b697463d899b2f1ab624..27e234f4851f3594c157236e6b1e39eca32dae81 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1600.0,+91/-54,0.729,10.36,5.33,7.184,5.52,0.529,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1588.0,+113/-69,0.685,10.87,4.46,5.826,2046.25,8.114,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1584.0,+88/-56,0.662,11.04,4.67,8.183,592.15,8.082,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1569.0,+104/-64,0.657,11.74,4.79,7.089,2046.25,1.254,0.0,False,GPU -4,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1510.0,+92/-60,0.585,14.67,5.88,10.733,144.27,3.772,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1497.0,+79/-54,0.512,15.33,6.23,10.68,3445.6,51.274,0.0,False,GPU -6,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1486.0,+98/-79,0.565,15.96,5.68,11.297,3.76,0.527,0.0,False,GPU -7,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1479.0,+79/-67,0.493,16.35,5.93,10.213,8.06,1.714,0.0,False,GPU -8,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1442.0,+65/-50,0.385,18.49,11.6,13.145,2879.46,14.626,0.0,False,GPU -9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1438.0,+75/-59,0.362,18.78,9.99,13.186,2379.18,1.595,0.0,False,GPU -10,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1430.0,+69/-59,0.406,19.25,9.32,14.423,316.0,1.931,0.0,False,GPU -11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1419.0,+76/-58,0.339,19.93,11.82,12.9,1828.68,1.648,0.0,False,CPU -12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1419.0,+84/-68,0.391,19.94,6.64,10.011,5319.2,249.314,0.0,False,GPU -13,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1415.0,+108/-75,0.407,20.2,6.42,13.416,3445.6,0.995,0.0,False,GPU -14,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1399.0,+104/-71,0.361,21.18,6.68,10.637,5319.2,29.488,0.0,False,GPU -15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1381.0,+86/-58,0.273,22.36,12.32,14.163,2379.18,0.176,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1376.0,+45/-44,0.233,22.67,15.32,15.222,449.53,1.149,0.0,False,CPU -17,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1372.0,+96/-85,0.405,22.91,8.63,15.503,3.73,0.488,0.0,False,GPU -18,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1352.0,+65/-52,0.22,24.25,12.67,14.896,2879.46,0.944,0.0,False,GPU -19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1347.0,+63/-58,0.225,24.59,10.35,14.824,4575.49,0.51,0.0,False,GPU -20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1347.0,+38/-37,0.17,24.6,17.27,15.649,449.53,0.215,0.0,False,CPU -21,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1326.0,+105/-79,0.303,25.96,8.98,13.65,51.44,43.709,0.0,False,GPU -22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1326.0,+61/-49,0.189,25.97,16.69,16.157,4.7,0.08,0.0,False,CPU -23,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1323.0,+56/-53,0.151,26.23,17.99,16.287,1081.93,0.556,0.0,False,CPU -24,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1316.0,+56/-45,0.159,26.64,19.09,16.312,1081.93,0.074,0.0,False,CPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1308.0,+67/-67,0.219,27.19,14.07,16.789,9.81,0.146,0.0,False,GPU -26,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1302.0,+53/-58,0.162,27.58,15.63,16.244,592.34,2.767,0.0,False,GPU -27,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1298.0,+52/-52,0.138,27.86,19.56,16.778,645.57,1.032,0.0,False,CPU -28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1295.0,+66/-71,0.174,28.07,13.53,17.352,14.36,0.311,0.0,False,GPU -29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1292.0,+50/-50,0.129,28.32,17.46,17.207,1618.46,0.415,0.0,False,CPU -30,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1290.0,+97/-103,0.254,28.39,11.16,16.484,4575.49,8.396,0.0,False,GPU -31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1286.0,+69/-58,0.14,28.68,19.64,15.992,2576.18,2.415,0.0,False,CPU -32,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1280.0,+54/-58,0.127,29.14,15.16,17.188,592.34,0.146,0.0,False,GPU -33,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1275.0,+42/-42,0.084,29.47,19.6,17.137,645.57,0.172,0.0,False,CPU -34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1271.0,+52/-55,0.1,29.74,16.79,17.71,1618.46,0.054,0.0,False,CPU -35,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1254.0,+54/-58,0.115,30.88,16.81,18.247,6.86,0.052,0.0,False,CPU -36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1253.0,+67/-78,0.143,30.93,17.0,17.887,618.9,5.468,0.0,False,CPU -37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1250.0,+55/-56,0.088,31.15,18.15,17.097,10.47,1.987,0.0,False,GPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1213.0,+63/-50,0.062,33.59,21.56,17.997,2576.18,0.136,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1206.0,+67/-74,0.058,34.09,19.4,19.212,618.9,0.352,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1183.0,+84/-72,0.067,35.56,23.19,21.088,199.83,0.865,0.0,False,CPU -41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1172.0,+64/-60,0.025,36.23,29.08,19.888,1.77,0.119,0.0,False,CPU -42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1150.0,+78/-78,0.04,37.6,25.09,21.933,199.83,0.086,0.0,False,CPU -43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1144.0,+40/-43,0.018,38.0,32.13,20.296,2.2,0.12,0.0,False,CPU -44,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1110.0,+95/-131,0.115,40.03,21.41,23.531,1.24,0.143,0.0,False,GPU -45,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1108.0,+65/-85,0.02,40.14,26.17,22.968,440.56,0.933,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1086.0,+75/-74,0.018,41.44,29.26,22.66,8.39,0.127,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1078.0,+52/-79,0.004,41.88,24.91,23.518,440.56,0.085,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1057.0,+98/-123,0.071,43.04,26.02,28.113,273.27,0.715,0.0,False,CPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1043.0,+78/-87,0.0,43.76,24.97,23.546,2.84,0.381,0.0,False,CPU -50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1034.0,+94/-126,0.047,44.19,27.79,28.584,273.27,0.098,0.0,False,CPU -51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1020.0,+83/-108,0.02,44.93,30.67,26.329,89.68,1.55,0.0,False,CPU -52,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1002.0,+98/-112,0.011,45.79,36.55,26.937,2.96,0.796,0.0,False,GPU -53,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+63/-105,0.003,45.87,37.86,26.578,0.42,0.04,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,989.0,+102/-139,0.03,46.39,30.04,30.404,1.44,0.138,0.0,False,CPU -55,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,976.0,+93/-114,0.012,46.96,38.72,27.729,0.31,0.042,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,953.0,+86/-138,0.001,47.95,30.7,30.595,89.68,0.12,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,639.0,+130/-215,0.0,55.9,49.79,43.912,0.21,0.019,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1600.0,+91/-54,7.184,0.729,10.36,5.33,5.52,0.529,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1588.0,+113/-69,5.826,0.685,10.87,4.46,2046.25,8.114,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1584.0,+88/-56,8.183,0.662,11.04,4.67,592.15,8.082,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1569.0,+104/-64,7.089,0.657,11.74,4.79,2046.25,1.254,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1510.0,+92/-60,10.733,0.585,14.67,5.88,144.27,3.772,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1497.0,+79/-54,10.68,0.512,15.33,6.23,3445.6,51.274,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1486.0,+98/-79,11.297,0.565,15.96,5.68,3.76,0.527,➖,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1479.0,+79/-67,10.213,0.493,16.35,5.93,8.06,1.714,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1442.0,+65/-50,13.145,0.385,18.49,11.6,2879.46,14.626,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1438.0,+75/-59,13.186,0.362,18.78,9.99,2379.18,1.595,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1430.0,+69/-59,14.423,0.406,19.25,9.32,316.0,1.931,➖,0.0,False,GPU +11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1419.0,+76/-58,12.9,0.339,19.93,11.82,1828.68,1.648,✔️,0.0,False,CPU +12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1419.0,+84/-68,10.011,0.391,19.94,6.64,5319.2,249.314,✔️,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1415.0,+108/-75,13.416,0.407,20.2,6.42,3445.6,0.995,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1399.0,+104/-71,10.637,0.361,21.18,6.68,5319.2,29.488,✔️,0.0,False,GPU +15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1381.0,+86/-58,14.163,0.273,22.36,12.32,2379.18,0.176,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1376.0,+45/-44,15.222,0.233,22.67,15.32,449.53,1.149,✔️,0.0,False,CPU +17,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1372.0,+96/-85,15.503,0.405,22.91,8.63,3.73,0.488,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1352.0,+65/-52,14.896,0.22,24.25,12.67,2879.46,0.944,✔️,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1347.0,+63/-58,14.824,0.225,24.59,10.35,4575.49,0.51,✔️,0.0,False,GPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1347.0,+38/-37,15.649,0.17,24.6,17.27,449.53,0.215,✔️,0.0,False,CPU +21,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1326.0,+105/-79,13.65,0.303,25.96,8.98,51.44,43.709,✔️,0.0,False,GPU +22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1326.0,+61/-49,16.157,0.189,25.97,16.69,4.7,0.08,✔️,0.0,False,CPU +23,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1323.0,+56/-53,16.287,0.151,26.23,17.99,1081.93,0.556,✔️,0.0,False,CPU +24,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1316.0,+56/-45,16.312,0.159,26.64,19.09,1081.93,0.074,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1308.0,+67/-67,16.789,0.219,27.19,14.07,9.81,0.146,✔️,0.0,False,GPU +26,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1302.0,+53/-58,16.244,0.162,27.58,15.63,592.34,2.767,✔️,0.0,False,GPU +27,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1298.0,+52/-52,16.778,0.138,27.86,19.56,645.57,1.032,✔️,0.0,False,CPU +28,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1295.0,+66/-71,17.352,0.174,28.07,13.53,14.36,0.311,✔️,0.0,False,GPU +29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1292.0,+50/-50,17.207,0.129,28.32,17.46,1618.46,0.415,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1290.0,+97/-103,16.484,0.254,28.39,11.16,4575.49,8.396,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1286.0,+69/-58,15.992,0.14,28.68,19.64,2576.18,2.415,✔️,0.0,False,CPU +32,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1280.0,+54/-58,17.188,0.127,29.14,15.16,592.34,0.146,✔️,0.0,False,GPU +33,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1275.0,+42/-42,17.137,0.084,29.47,19.6,645.57,0.172,✔️,0.0,False,CPU +34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1271.0,+52/-55,17.71,0.1,29.74,16.79,1618.46,0.054,✔️,0.0,False,CPU +35,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1254.0,+54/-58,18.247,0.115,30.88,16.81,6.86,0.052,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1253.0,+67/-78,17.887,0.143,30.93,17.0,618.9,5.468,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1250.0,+55/-56,17.097,0.088,31.15,18.15,10.47,1.987,✔️,0.0,False,GPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1213.0,+63/-50,17.997,0.062,33.59,21.56,2576.18,0.136,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1206.0,+67/-74,19.212,0.058,34.09,19.4,618.9,0.352,✔️,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1183.0,+84/-72,21.088,0.067,35.56,23.19,199.83,0.865,✔️,0.0,False,CPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1172.0,+64/-60,19.888,0.025,36.23,29.08,1.77,0.119,✔️,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1150.0,+78/-78,21.933,0.04,37.6,25.09,199.83,0.086,✔️,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1144.0,+40/-43,20.296,0.018,38.0,32.13,2.2,0.12,✔️,0.0,False,CPU +44,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1110.0,+95/-131,23.531,0.115,40.03,21.41,1.24,0.143,➖,0.0,False,GPU +45,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1108.0,+65/-85,22.968,0.02,40.14,26.17,440.56,0.933,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1086.0,+75/-74,22.66,0.018,41.44,29.26,8.39,0.127,✔️,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1078.0,+52/-79,23.518,0.004,41.88,24.91,440.56,0.085,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1057.0,+98/-123,28.113,0.071,43.04,26.02,273.27,0.715,✔️,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1043.0,+78/-87,23.546,0.0,43.76,24.97,2.84,0.381,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1034.0,+94/-126,28.584,0.047,44.19,27.79,273.27,0.098,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1020.0,+83/-108,26.329,0.02,44.93,30.67,89.68,1.55,✔️,0.0,False,CPU +52,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1002.0,+98/-112,26.937,0.011,45.79,36.55,2.96,0.796,✔️,0.0,False,GPU +53,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+63/-105,26.578,0.003,45.87,37.86,0.42,0.04,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),989.0,+102/-139,30.404,0.03,46.39,30.04,1.44,0.138,✔️,0.0,False,CPU +55,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),976.0,+93/-114,27.729,0.012,46.96,38.72,0.31,0.042,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),953.0,+86/-138,30.595,0.001,47.95,30.7,89.68,0.12,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),639.0,+130/-215,43.912,0.0,55.9,49.79,0.21,0.019,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip index 9a9597403be7612cf179c30fca0c70f875dd228b..4e908b6552c60b47211b113a80a3d20aaf093bae 100644 --- a/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:28826ea6797aca90c9e4190739a0fb902ff02c08ace5a831c651f8314a44b008 +oid sha256:d6aee50ebbd6d0dd5d4fd9e28423ac60c0db044a477c56d377dc5cfc80758256 size 2690375 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index f3f370097269fe11540d0e3e38c2f5946b25852d..2840613bbd8ce9d773d8bb5c2c3a2aa293705fba 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e13d5a2ed7c6edb04d4c6ac684c420d81c5ae70f8a274fb2287e9e20ef65e31f +oid sha256:0b0821532cd551daad0cc0c07140a2b7d022729611c88c5c1857eaa3909cbe3f size 481972 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip index 3191ab755af8386e22321129fe1159fecd554bed..4814ffbb620a4c2ea86e473937d54e0115b8953a 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:63e60df80df61a78e2dfac0636f42e297b9dc7f43e4e253d0f02a1edd8919805 -size 1075471 +oid sha256:38efb78b76aa096acde6bad709cca51e27be2a6c1e64a7886690446c24ada01e +size 1035298 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_all/tuning-impact-elo.png.zip index 5506aaab810afd4132e3678af06485fe5ade8817..f6d80f3af39a262ca0a78f3024251e3ff7f8eb23 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f353772dd7e83438ca05ec78338010e6013acb70a069b1e4f4a7119eaf0b4d5f +oid sha256:891d6d61fddc25cc518f270a2cf82e3ac0deca2ddd0eeedf07ee86e5714b3f36 size 183321 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_classification/datasets_all/website_leaderboard.csv index 8bfb01a9702580b56d035045b482fd76def637de..548b97b2fc4f61d4b27068e6623257e5bb661690 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_all/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_all/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1638.0,+69/-56,0.765,8.56,3.69,6.824,545.23,6.469,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1577.0,+79/-72,0.681,10.99,3.71,6.302,2046.25,8.976,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1550.0,+72/-63,0.643,12.21,4.72,7.349,2046.25,1.329,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1535.0,+66/-61,0.643,12.96,5.76,7.777,5.76,0.794,0.0,False,GPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1485.0,+66/-53,0.493,15.62,5.79,10.48,1684.49,1.97,0.0,False,CPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1472.0,+56/-46,0.462,16.32,9.49,11.041,2879.46,12.49,0.0,False,GPU -6,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1445.0,+61/-42,0.405,17.94,8.68,12.372,2466.21,1.502,0.0,False,GPU -7,🧠⚡,Foundation Model,[TabICL (default) [5.26% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1417.0,+62/-60,0.427,19.7,7.44,12.417,8.68,1.743,5.26,True,GPU -8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1416.0,+46/-33,0.333,19.76,12.62,13.471,382.05,1.488,0.0,False,CPU -9,🧠🔁,Neural Network,[LimiX (default) [16.08% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1415.0,+71/-72,0.47,19.82,6.63,12.898,3.45,0.504,16.08,True,GPU -10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1404.0,+76/-50,0.39,20.47,7.33,11.426,4940.61,307.751,0.0,False,GPU -11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1394.0,+57/-54,0.329,21.15,9.99,13.269,2466.21,0.176,0.0,False,GPU -12,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1393.0,+49/-43,0.305,21.19,11.07,12.692,2879.46,0.598,0.0,False,GPU -13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1392.0,+54/-41,0.308,21.23,11.95,12.971,1372.94,0.556,0.0,False,CPU -14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1384.0,+54/-44,0.303,21.76,11.08,13.175,1372.94,0.074,0.0,False,CPU -15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1375.0,+44/-31,0.246,22.4,15.39,14.193,382.05,0.254,0.0,False,CPU -16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1365.0,+48/-44,0.271,23.04,13.33,14.314,685.87,1.455,0.0,False,CPU -17,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1362.0,+46/-44,0.29,23.26,11.95,13.81,5.72,0.076,0.0,False,CPU -18,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1357.0,+79/-74,0.398,23.59,7.8,14.612,3008.22,20.849,31.58,True,GPU -19,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1356.0,+51/-48,0.312,23.63,12.55,15.62,282.72,1.886,0.0,False,GPU -20,🧠🔁,Neural Network,[Mitra (default) [31.58% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1355.0,+79/-79,0.41,23.72,8.41,15.045,121.91,2.776,31.58,True,GPU -21,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1353.0,+81/-75,0.319,23.8,10.42,13.855,4879.89,8.744,0.0,False,GPU -22,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1352.0,+55/-37,0.226,23.92,11.59,13.898,4879.89,0.525,0.0,False,GPU -23,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1345.0,+61/-52,0.284,24.39,8.86,13.143,4940.61,41.606,0.0,False,GPU -24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1341.0,+46/-40,0.214,24.64,16.3,14.692,685.87,0.205,0.0,False,CPU -25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1319.0,+43/-45,0.197,26.1,15.62,15.322,934.1,3.063,0.0,False,GPU -26,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1305.0,+73/-76,0.307,27.09,8.8,16.473,3008.22,0.514,31.58,True,GPU -27,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1300.0,+42/-36,0.16,27.4,18.06,16.365,2686.11,0.47,0.0,False,CPU -28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1299.0,+44/-41,0.162,27.48,18.83,15.242,2389.22,2.158,0.0,False,CPU -29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1299.0,+50/-48,0.229,27.49,15.1,16.043,10.21,0.138,0.0,False,GPU -30,🧠⚡,Foundation Model,[TabPFNv2 (default) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1282.0,+72/-71,0.292,28.68,11.75,17.271,3.37,0.315,31.58,True,GPU -31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1278.0,+53/-55,0.146,28.9,16.76,16.823,934.1,0.169,0.0,False,GPU -32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1266.0,+43/-36,0.111,29.75,19.24,17.087,2686.11,0.054,0.0,False,CPU -33,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1265.0,+62/-60,0.221,29.82,10.77,16.444,49.21,43.824,0.0,False,GPU -34,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1254.0,+60/-57,0.153,30.58,17.25,17.607,618.9,4.766,0.0,False,CPU -35,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1243.0,+41/-43,0.105,31.3,18.36,18.158,14.78,0.346,0.0,False,GPU -36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1240.0,+44/-44,0.112,31.51,20.18,18.322,6.86,0.057,0.0,False,CPU -37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1230.0,+40/-37,0.074,32.15,22.08,17.431,10.47,1.707,0.0,False,GPU -38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1221.0,+49/-53,0.09,32.76,23.28,17.518,1.77,0.117,0.0,False,CPU -39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1217.0,+48/-42,0.077,33.01,22.74,17.465,2389.22,0.152,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1206.0,+50/-47,0.086,33.72,21.84,19.045,189.76,0.743,0.0,False,CPU -41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1188.0,+52/-58,0.062,34.94,21.46,19.443,618.9,0.298,0.0,False,CPU -42,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1183.0,+59/-58,0.085,35.25,20.57,19.941,323.74,0.743,0.0,False,CPU -43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1179.0,+45/-38,0.064,35.49,29.61,18.579,1.79,0.12,0.0,False,CPU -44,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1166.0,+55/-51,0.062,36.33,22.54,20.27,189.76,0.079,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1146.0,+66/-57,0.056,37.59,19.77,20.807,323.74,0.076,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1084.0,+46/-55,0.01,41.26,32.08,22.16,6.83,0.147,0.0,False,CPU -47,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1037.0,+84/-85,0.041,43.74,31.33,29.833,252.58,0.528,0.0,False,CPU -48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1037.0,+68/-69,0.012,43.74,29.99,24.506,2.91,0.368,0.0,False,CPU -49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1023.0,+66/-93,0.023,44.46,29.66,27.652,139.84,3.615,0.0,False,CPU -50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1011.0,+85/-92,0.06,45.04,29.33,28.785,1.11,0.189,0.0,False,GPU -51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1008.0,+78/-89,0.027,45.18,33.3,30.589,252.58,0.089,0.0,False,CPU -52,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+56/-57,0.006,45.56,39.42,26.919,0.38,0.037,0.0,False,CPU -53,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,998.0,+74/-88,0.022,45.68,37.75,27.409,3.61,0.939,0.0,False,GPU -54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,938.0,+69/-71,0.006,48.2,42.58,29.159,0.25,0.041,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,929.0,+84/-111,0.016,48.58,36.92,33.418,1.29,0.131,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,904.0,+83/-104,0.009,49.5,36.12,33.842,139.84,0.193,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,555.0,+111/-114,0.0,56.61,53.11,49.665,0.18,0.029,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1638.0,+69/-56,6.824,0.765,8.56,3.69,545.23,6.469,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1577.0,+79/-72,6.302,0.681,10.99,3.71,2046.25,8.976,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1550.0,+72/-63,7.349,0.643,12.21,4.72,2046.25,1.329,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1535.0,+66/-61,7.777,0.643,12.96,5.76,5.76,0.794,✔️,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1485.0,+66/-53,10.48,0.493,15.62,5.79,1684.49,1.97,✔️,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1472.0,+56/-46,11.041,0.462,16.32,9.49,2879.46,12.49,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1445.0,+61/-42,12.372,0.405,17.94,8.68,2466.21,1.502,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabICL (default) [5.26% IMPUTED]](https://arxiv.org/abs/2502.05564),1417.0,+62/-60,12.417,0.427,19.7,7.44,8.68,1.743,✔️,5.26,True,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1416.0,+46/-33,13.471,0.333,19.76,12.62,382.05,1.488,✔️,0.0,False,CPU +9,🧠⚡,Foundation Model,[LimiX (default) [16.08% IMPUTED]](https://arxiv.org/abs/2509.03505),1415.0,+71/-72,12.898,0.47,19.82,6.63,3.45,0.504,➖,16.08,True,GPU +10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1404.0,+76/-50,11.426,0.39,20.47,7.33,4940.61,307.751,✔️,0.0,False,GPU +11,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1394.0,+57/-54,13.269,0.329,21.15,9.99,2466.21,0.176,✔️,0.0,False,GPU +12,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1393.0,+49/-43,12.692,0.305,21.19,11.07,2879.46,0.598,✔️,0.0,False,GPU +13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1392.0,+54/-41,12.971,0.308,21.23,11.95,1372.94,0.556,✔️,0.0,False,CPU +14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1384.0,+54/-44,13.175,0.303,21.76,11.08,1372.94,0.074,✔️,0.0,False,CPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1375.0,+44/-31,14.193,0.246,22.4,15.39,382.05,0.254,✔️,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1365.0,+48/-44,14.314,0.271,23.04,13.33,685.87,1.455,✔️,0.0,False,CPU +17,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1362.0,+46/-44,13.81,0.29,23.26,11.95,5.72,0.076,✔️,0.0,False,CPU +18,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1357.0,+79/-74,14.612,0.398,23.59,7.8,3008.22,20.849,✔️,31.58,True,GPU +19,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1356.0,+51/-48,15.62,0.312,23.63,12.55,282.72,1.886,➖,0.0,False,GPU +20,🧠⚡,Foundation Model,[Mitra (default) [31.58% IMPUTED]](https://arxiv.org/abs/2510.21204),1355.0,+79/-79,15.045,0.41,23.72,8.41,121.91,2.776,✔️,31.58,True,GPU +21,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1353.0,+81/-75,13.855,0.319,23.8,10.42,4879.89,8.744,✔️,0.0,False,GPU +22,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1352.0,+55/-37,13.898,0.226,23.92,11.59,4879.89,0.525,✔️,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1345.0,+61/-52,13.143,0.284,24.39,8.86,4940.61,41.606,✔️,0.0,False,GPU +24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1341.0,+46/-40,14.692,0.214,24.64,16.3,685.87,0.205,✔️,0.0,False,CPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1319.0,+43/-45,15.322,0.197,26.1,15.62,934.1,3.063,✔️,0.0,False,GPU +26,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1305.0,+73/-76,16.473,0.307,27.09,8.8,3008.22,0.514,✔️,31.58,True,GPU +27,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1300.0,+42/-36,16.365,0.16,27.4,18.06,2686.11,0.47,✔️,0.0,False,CPU +28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1299.0,+44/-41,15.242,0.162,27.48,18.83,2389.22,2.158,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1299.0,+50/-48,16.043,0.229,27.49,15.1,10.21,0.138,✔️,0.0,False,GPU +30,🧠⚡,Foundation Model,[TabPFNv2 (default) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1282.0,+72/-71,17.271,0.292,28.68,11.75,3.37,0.315,✔️,31.58,True,GPU +31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1278.0,+53/-55,16.823,0.146,28.9,16.76,934.1,0.169,✔️,0.0,False,GPU +32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1266.0,+43/-36,17.087,0.111,29.75,19.24,2686.11,0.054,✔️,0.0,False,CPU +33,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1265.0,+62/-60,16.444,0.221,29.82,10.77,49.21,43.824,✔️,0.0,False,GPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1254.0,+60/-57,17.607,0.153,30.58,17.25,618.9,4.766,✔️,0.0,False,CPU +35,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1243.0,+41/-43,18.158,0.105,31.3,18.36,14.78,0.346,✔️,0.0,False,GPU +36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1240.0,+44/-44,18.322,0.112,31.51,20.18,6.86,0.057,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1230.0,+40/-37,17.431,0.074,32.15,22.08,10.47,1.707,✔️,0.0,False,GPU +38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1221.0,+49/-53,17.518,0.09,32.76,23.28,1.77,0.117,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1217.0,+48/-42,17.465,0.077,33.01,22.74,2389.22,0.152,✔️,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1206.0,+50/-47,19.045,0.086,33.72,21.84,189.76,0.743,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1188.0,+52/-58,19.443,0.062,34.94,21.46,618.9,0.298,✔️,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1183.0,+59/-58,19.941,0.085,35.25,20.57,323.74,0.743,✔️,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1179.0,+45/-38,18.579,0.064,35.49,29.61,1.79,0.12,✔️,0.0,False,CPU +44,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1166.0,+55/-51,20.27,0.062,36.33,22.54,189.76,0.079,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1146.0,+66/-57,20.807,0.056,37.59,19.77,323.74,0.076,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1084.0,+46/-55,22.16,0.01,41.26,32.08,6.83,0.147,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1037.0,+84/-85,29.833,0.041,43.74,31.33,252.58,0.528,✔️,0.0,False,CPU +48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1037.0,+68/-69,24.506,0.012,43.74,29.99,2.91,0.368,✔️,0.0,False,CPU +49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1023.0,+66/-93,27.652,0.023,44.46,29.66,139.84,3.615,✔️,0.0,False,CPU +50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1011.0,+85/-92,28.785,0.06,45.04,29.33,1.11,0.189,➖,0.0,False,GPU +51,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1008.0,+78/-89,30.589,0.027,45.18,33.3,252.58,0.089,✔️,0.0,False,CPU +52,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+56/-57,26.919,0.006,45.56,39.42,0.38,0.037,✔️,0.0,False,CPU +53,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),998.0,+74/-88,27.409,0.022,45.68,37.75,3.61,0.939,✔️,0.0,False,GPU +54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),938.0,+69/-71,29.159,0.006,48.2,42.58,0.25,0.041,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),929.0,+84/-111,33.418,0.016,48.58,36.92,1.29,0.131,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),904.0,+83/-104,33.842,0.009,49.5,36.12,139.84,0.193,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),555.0,+111/-114,49.665,0.0,56.61,53.11,0.18,0.029,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_all/winrate_matrix.png.zip index fd4e6bea83453193a8a4b670ba988aa4f1e1292f..8d3246e083a35048500bce2ae86f44141998f569 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1b650769c5aa1d255c56cedf8be61aa4e83635bdce963675cda6452d927a0495 +oid sha256:c14a7249165162fda9e0dd900ccf4213d91bd623896d7428aef03cacdd7bb3f0 size 2678386 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index 732b0103ce6981d44d2dc74fb1c1950cc5aa6973..7a4318cd81da4b247f9711528d464ded18762c9b 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f9f7bcea3025b254ae711555bc2ce966dd60bd434503e88722b792ebe0ac027d +oid sha256:15cd9675844d68edb113b6f82a5262e07f7635d39d63eefa21a950fead1b055e size 437424 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip index 5b5daf5c1ed82b954abf236db37b76fbc501de8b..32239e698e2456c7da88dd78df0400e1f8857a40 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:79e44af46288d2300b48c6c8a3cf75e275958cdec8c1aaf3074ee15fab5279c1 -size 1016226 +oid sha256:4c7e93928ceada5ce77c33aab027034f339f9591c89c79fb527ea57439da632b +size 957078 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/tuning-impact-elo.png.zip index 4c07131492a6f59cd4cf5b023153a833efda5e85..148af277601af416712c5832032400705cb17f03 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f3dac5dd0c19d57c6b52c8c140dcb9e43b85236b6844067b0b6ee92d3ea6772b +oid sha256:80e128c092bdfbb917128af98332ef36cb7adda9ca14883791913b113fa3d777 size 134430 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/website_leaderboard.csv index 36a517387216c3cfcb38c11b5b9eac5fc4566d6c..ad80d991c6cb5300d9d1df4fb398e6a63a400a57 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/website_leaderboard.csv @@ -1,55 +1,55 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1917.0,+147/-60,0.906,4.82,2.46,2.185,275.97,2.818,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1858.0,+176/-74,0.858,6.13,2.65,2.418,314.17,2.249,0.0,False,CPU -2,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1695.0,+135/-72,0.618,11.37,6.22,5.413,697.8,0.199,0.0,False,CPU -3,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1694.0,+143/-70,0.624,11.43,5.19,5.658,697.8,0.023,0.0,False,CPU -4,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1665.0,+131/-67,0.609,12.62,5.98,5.6,2.44,0.027,0.0,False,CPU -5,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1662.0,+128/-94,0.542,12.72,7.4,7.881,207.65,0.398,0.0,False,CPU -6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1658.0,+143/-72,0.529,12.88,8.42,8.139,93.21,0.758,0.0,False,CPU -7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1639.0,+120/-89,0.515,13.72,8.72,7.26,1624.54,1.763,0.0,False,GPU -8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1637.0,+216/-124,0.54,13.82,3.83,7.552,783.29,13.79,0.0,False,GPU -9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1616.0,+142/-128,0.529,14.77,7.59,7.563,2338.19,14.248,0.0,False,GPU -10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1594.0,+182/-116,0.46,15.79,6.1,8.64,1398.9,0.645,0.0,False,GPU -11,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1589.0,+114/-75,0.427,16.03,11.85,8.696,207.65,0.056,0.0,False,CPU -12,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1585.0,+204/-117,0.468,16.22,6.11,8.38,783.29,1.924,0.0,False,GPU -13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1555.0,+126/-87,0.335,17.7,9.55,8.734,1624.54,0.083,0.0,False,GPU -14,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1544.0,+166/-78,0.346,18.22,11.76,9.818,93.21,0.087,0.0,False,CPU -15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1544.0,+180/-146,0.41,18.26,6.86,9.353,1398.9,0.062,0.0,False,GPU -16,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1533.0,+146/-118,0.415,18.82,8.42,9.128,2.3,0.82,0.0,False,GPU -17,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1512.0,+160/-142,0.413,19.9,9.15,9.75,7.51,3.146,0.0,False,GPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1500.0,+109/-79,0.264,20.5,14.09,10.252,2338.19,0.54,0.0,False,GPU -19,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1478.0,+154/-137,0.325,21.66,8.15,9.97,4805.66,461.797,0.0,False,GPU -20,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1467.0,+86/-76,0.234,22.26,15.27,10.088,1169.3,1.442,0.0,False,CPU -21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1455.0,+142/-105,0.214,22.91,15.36,12.545,893.93,0.205,0.0,False,CPU -22,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1430.0,+148/-126,0.255,24.28,14.98,10.933,0.47,0.052,0.0,False,CPU -23,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1413.0,+135/-129,0.175,25.19,16.24,12.445,709.85,2.929,0.0,False,GPU -24,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1388.0,+158/-107,0.141,26.62,20.06,13.595,893.93,0.016,0.0,False,CPU -25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1378.0,+132/-132,0.145,27.13,20.54,14.515,709.85,0.19,0.0,False,GPU -26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1376.0,+145/-149,0.218,27.26,15.69,12.68,5.67,0.081,0.0,False,GPU -27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1351.0,+84/-82,0.071,28.67,25.09,13.54,6.98,0.233,0.0,False,GPU -28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1349.0,+97/-138,0.112,28.74,21.98,14.46,389.62,2.114,0.0,False,CPU -29,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1341.0,+136/-132,0.122,29.18,21.44,15.921,84.36,0.919,0.0,False,GPU -30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1338.0,+83/-84,0.076,29.33,23.8,12.269,1169.3,0.053,0.0,False,CPU -31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1334.0,+125/-102,0.086,29.6,25.47,15.629,2.09,0.019,0.0,False,CPU -32,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1322.0,+111/-117,0.077,30.22,19.83,13.703,4805.66,43.211,0.0,False,GPU -33,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1321.0,+116/-99,0.062,30.27,22.3,13.699,52.45,0.219,0.0,False,CPU -34,🧠🔁,Neural Network,[LimiX (default) [60.00% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1303.0,+188/-187,0.21,31.21,14.25,18.746,0.44,0.016,60.0,True,GPU -35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1289.0,+90/-106,0.061,31.97,28.98,13.714,0.28,0.033,0.0,False,CPU -36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1284.0,+124/-111,0.063,32.2,24.11,14.614,52.45,0.021,0.0,False,CPU -37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1277.0,+79/-71,0.009,32.57,29.74,14.632,10.5,0.606,0.0,False,GPU -38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1259.0,+71/-86,0.026,33.5,30.99,13.754,38.42,0.32,0.0,False,CPU -39,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1241.0,+122/-150,0.029,34.41,21.89,17.572,46.9,45.623,0.0,False,GPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1214.0,+80/-141,0.033,35.73,30.58,17.64,389.62,0.112,0.0,False,CPU -41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1201.0,+75/-84,0.012,36.32,34.32,15.542,38.42,0.028,0.0,False,CPU -42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1176.0,+63/-70,0.0,37.5,35.95,16.74,1.87,0.046,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1139.0,+109/-210,0.038,39.1,33.8,21.377,1.61,0.112,0.0,False,CPU -44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1099.0,+142/-227,0.007,40.78,36.67,26.806,43.46,0.087,0.0,False,CPU -45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1068.0,+95/-197,0.009,41.96,33.16,24.396,331.68,44.947,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1048.0,+150/-236,0.004,42.71,39.35,28.029,43.46,0.021,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+99/-176,0.0,44.35,43.24,22.389,0.12,0.013,0.0,False,CPU -48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,928.0,+135/-232,0.0,46.49,44.98,29.601,0.2,0.023,0.0,False,CPU -49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,914.0,+201/-459,0.022,46.88,41.22,31.192,2.8,0.224,0.0,False,GPU -50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,880.0,+87/-199,0.0,47.71,47.28,26.755,0.09,0.019,0.0,False,CPU -51,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,859.0,+109/-176,0.0,48.2,47.41,32.973,0.2,0.033,0.0,False,GPU -52,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,818.0,+67/-150,0.0,49.06,48.67,32.101,331.68,1.626,0.0,False,CPU -53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,447.0,+126/-378,0.0,53.34,53.31,49.305,0.12,0.101,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1917.0,+147/-60,2.185,0.906,4.82,2.46,275.97,2.818,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1858.0,+176/-74,2.418,0.858,6.13,2.65,314.17,2.249,✔️,0.0,False,CPU +2,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1695.0,+135/-72,5.413,0.618,11.37,6.22,697.8,0.199,✔️,0.0,False,CPU +3,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1694.0,+143/-70,5.658,0.624,11.43,5.19,697.8,0.023,✔️,0.0,False,CPU +4,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1665.0,+131/-67,5.6,0.609,12.62,5.98,2.44,0.027,✔️,0.0,False,CPU +5,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1662.0,+128/-94,7.881,0.542,12.72,7.4,207.65,0.398,✔️,0.0,False,CPU +6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1658.0,+143/-72,8.139,0.529,12.88,8.42,93.21,0.758,✔️,0.0,False,CPU +7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1639.0,+120/-89,7.26,0.515,13.72,8.72,1624.54,1.763,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1637.0,+216/-124,7.552,0.54,13.82,3.83,783.29,13.79,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1616.0,+142/-128,7.563,0.529,14.77,7.59,2338.19,14.248,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1594.0,+182/-116,8.64,0.46,15.79,6.1,1398.9,0.645,✔️,0.0,False,GPU +11,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1589.0,+114/-75,8.696,0.427,16.03,11.85,207.65,0.056,✔️,0.0,False,CPU +12,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1585.0,+204/-117,8.38,0.468,16.22,6.11,783.29,1.924,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1555.0,+126/-87,8.734,0.335,17.7,9.55,1624.54,0.083,✔️,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1544.0,+166/-78,9.818,0.346,18.22,11.76,93.21,0.087,✔️,0.0,False,CPU +15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1544.0,+180/-146,9.353,0.41,18.26,6.86,1398.9,0.062,✔️,0.0,False,GPU +16,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1533.0,+146/-118,9.128,0.415,18.82,8.42,2.3,0.82,✔️,0.0,False,GPU +17,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1512.0,+160/-142,9.75,0.413,19.9,9.15,7.51,3.146,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1500.0,+109/-79,10.252,0.264,20.5,14.09,2338.19,0.54,✔️,0.0,False,GPU +19,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1478.0,+154/-137,9.97,0.325,21.66,8.15,4805.66,461.797,✔️,0.0,False,GPU +20,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1467.0,+86/-76,10.088,0.234,22.26,15.27,1169.3,1.442,✔️,0.0,False,CPU +21,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1455.0,+142/-105,12.545,0.214,22.91,15.36,893.93,0.205,✔️,0.0,False,CPU +22,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1430.0,+148/-126,10.933,0.255,24.28,14.98,0.47,0.052,✔️,0.0,False,CPU +23,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1413.0,+135/-129,12.445,0.175,25.19,16.24,709.85,2.929,✔️,0.0,False,GPU +24,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1388.0,+158/-107,13.595,0.141,26.62,20.06,893.93,0.016,✔️,0.0,False,CPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1378.0,+132/-132,14.515,0.145,27.13,20.54,709.85,0.19,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1376.0,+145/-149,12.68,0.218,27.26,15.69,5.67,0.081,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1351.0,+84/-82,13.54,0.071,28.67,25.09,6.98,0.233,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1349.0,+97/-138,14.46,0.112,28.74,21.98,389.62,2.114,✔️,0.0,False,CPU +29,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1341.0,+136/-132,15.921,0.122,29.18,21.44,84.36,0.919,➖,0.0,False,GPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1338.0,+83/-84,12.269,0.076,29.33,23.8,1169.3,0.053,✔️,0.0,False,CPU +31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1334.0,+125/-102,15.629,0.086,29.6,25.47,2.09,0.019,✔️,0.0,False,CPU +32,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1322.0,+111/-117,13.703,0.077,30.22,19.83,4805.66,43.211,✔️,0.0,False,GPU +33,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1321.0,+116/-99,13.699,0.062,30.27,22.3,52.45,0.219,✔️,0.0,False,CPU +34,🧠⚡,Foundation Model,[LimiX (default) [60.00% IMPUTED]](https://arxiv.org/abs/2509.03505),1303.0,+188/-187,18.746,0.21,31.21,14.25,0.44,0.016,➖,60.0,True,GPU +35,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1289.0,+90/-106,13.714,0.061,31.97,28.98,0.28,0.033,✔️,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1284.0,+124/-111,14.614,0.063,32.2,24.11,52.45,0.021,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1277.0,+79/-71,14.632,0.009,32.57,29.74,10.5,0.606,✔️,0.0,False,GPU +38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1259.0,+71/-86,13.754,0.026,33.5,30.99,38.42,0.32,✔️,0.0,False,CPU +39,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1241.0,+122/-150,17.572,0.029,34.41,21.89,46.9,45.623,✔️,0.0,False,GPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1214.0,+80/-141,17.64,0.033,35.73,30.58,389.62,0.112,✔️,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1201.0,+75/-84,15.542,0.012,36.32,34.32,38.42,0.028,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1176.0,+63/-70,16.74,0.0,37.5,35.95,1.87,0.046,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1139.0,+109/-210,21.377,0.038,39.1,33.8,1.61,0.112,✔️,0.0,False,CPU +44,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1099.0,+142/-227,26.806,0.007,40.78,36.67,43.46,0.087,✔️,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1068.0,+95/-197,24.396,0.009,41.96,33.16,331.68,44.947,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1048.0,+150/-236,28.029,0.004,42.71,39.35,43.46,0.021,✔️,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+99/-176,22.389,0.0,44.35,43.24,0.12,0.013,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),928.0,+135/-232,29.601,0.0,46.49,44.98,0.2,0.023,✔️,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),914.0,+201/-459,31.192,0.022,46.88,41.22,2.8,0.224,✔️,0.0,False,GPU +50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),880.0,+87/-199,26.755,0.0,47.71,47.28,0.09,0.019,✔️,0.0,False,CPU +51,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),859.0,+109/-176,32.973,0.0,48.2,47.41,0.2,0.033,➖,0.0,False,GPU +52,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),818.0,+67/-150,32.101,0.0,49.06,48.67,331.68,1.626,✔️,0.0,False,CPU +53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),447.0,+126/-378,49.305,0.0,53.34,53.31,0.12,0.101,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/winrate_matrix.png.zip index 435ab5b02da7eba063be02372f645a2c5e461ebc..2562a435e29faa379d30baa5908d592bc6822686 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8da259e29152fdee5ff682f90392202ada57ac3d15020a56cfe724911a4e438d +oid sha256:966440a682d3728d229ce7afad6e2b477f1602d0c52d078d8abd3c63ba14b249 size 2273190 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 31f68a600531270549a4e9315a31be48cacc06f2..06d1138c7a0aad011f2d81542d300b1763490154 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1a2cd7b6d3bc2bf4bc5fe0ce7f0a3b1f41d42d467efc29590222a0104a5244e7 +oid sha256:4d74b100ca6a413321c9dae6b8f21950982695910125c0986581e1a71bb4d896 size 471502 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip index a964b394c22b966a7aaeda8b5a94f84e1d773320..7b5ef8762e095faabeada40af45c311c551fb4ab 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:043be86e8746e0169982a8aaa9769291c0f8c46e5abf91ff3d06e8b8477fa6d6 -size 1059768 +oid sha256:aae23739517f2abc4c3e7df8ecf238bca412375257f9eadb70b119f3c6d56c03 +size 1027096 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_small/tuning-impact-elo.png.zip index 68a6d6c506517fbe86ee9ae9776cc9c9812b9d0b..4f9f9f634a114934f1653999e3d2739dd786a2ef 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d1426dbd5a5f0457aedbccd32c18bf18d86d4ed5e27158e59858c6509fbc9ce5 +oid sha256:0b5b5befc5edc9bfc71c0ebc812c59ec31733b3b120334cb71792f85f85334e0 size 190557 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_classification/datasets_small/website_leaderboard.csv index 783c8eca8668f7a8f76fb2770392eca7eee27a66..d3a4f497c8b258dfe78a765eefae142f48f71581 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_small/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_small/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1604.0,+78/-60,0.711,9.9,4.49,8.48,610.76,8.082,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1602.0,+88/-82,0.725,9.98,3.67,5.856,2289.05,8.114,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1583.0,+75/-67,0.697,10.77,4.36,6.98,2289.05,1.254,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1582.0,+78/-79,0.713,10.84,5.18,7.294,6.07,0.762,0.0,False,GPU -4,🧠🔁,Neural Network,[LimiX (default) [0.40% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1489.0,+85/-70,0.559,15.35,5.56,10.809,4.62,0.633,0.4,True,GPU -5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1488.0,+85/-75,0.54,15.43,6.02,11.831,3302.86,45.811,7.14,True,GPU -6,🧠🔁,Neural Network,[Mitra (default) [7.14% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1484.0,+92/-78,0.556,15.65,6.52,12.423,144.27,3.772,7.14,True,GPU -7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1457.0,+68/-52,0.433,17.16,9.79,12.392,3422.82,18.936,0.0,False,GPU -8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1431.0,+68/-56,0.375,18.68,10.21,13.705,3170.35,1.694,0.0,False,GPU -9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1426.0,+79/-52,0.359,19.01,10.04,13.359,3143.8,1.884,0.0,False,CPU -10,🧠⚡,Foundation Model,[TabICL (default) [7.14% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1416.0,+82/-60,0.421,19.62,6.98,13.369,8.89,1.714,7.14,True,GPU -11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1409.0,+67/-62,0.399,20.04,7.07,11.946,5271.34,231.986,0.0,False,GPU -12,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1407.0,+106/-83,0.416,20.18,6.83,14.357,3302.86,0.929,7.14,True,GPU -13,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1389.0,+67/-50,0.368,21.32,10.91,15.512,330.95,2.359,0.0,False,GPU -14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1376.0,+58/-60,0.291,22.09,11.93,14.668,3170.35,0.241,0.0,False,GPU -15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1375.0,+72/-61,0.352,22.17,7.4,12.943,5271.34,33.986,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1374.0,+46/-36,0.252,22.22,15.36,15.375,735.1,1.901,0.0,False,CPU -17,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1373.0,+58/-52,0.28,22.32,11.72,14.106,3422.82,1.534,0.0,False,GPU -18,🧠⚡,Foundation Model,[TabPFNv2 (default) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1372.0,+79/-73,0.396,22.38,9.29,15.443,4.15,0.42,7.14,True,GPU -19,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1349.0,+44/-29,0.196,23.88,17.3,15.756,735.1,0.311,0.0,False,CPU -20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1336.0,+41/-35,0.187,24.74,17.8,15.67,1714.62,0.768,0.0,False,CPU -21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1330.0,+51/-49,0.2,25.11,10.89,15.2,6047.72,0.525,0.0,False,GPU -22,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1325.0,+39/-32,0.178,25.44,18.66,15.86,1714.62,0.098,0.0,False,CPU -23,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1313.0,+55/-58,0.201,26.25,15.38,16.35,1053.84,3.063,0.0,False,GPU -24,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1306.0,+45/-47,0.163,26.73,18.68,16.611,797.9,2.228,0.0,False,CPU -25,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1302.0,+78/-77,0.234,27.03,12.02,16.103,6047.72,8.744,0.0,False,GPU -26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1301.0,+44/-38,0.164,27.05,18.55,16.742,7.19,0.118,0.0,False,CPU -27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1298.0,+59/-61,0.224,27.3,14.87,17.244,10.78,0.17,0.0,False,GPU -28,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1291.0,+42/-42,0.126,27.72,18.81,16.833,797.9,0.353,0.0,False,CPU -29,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1291.0,+77/-81,0.279,27.73,9.1,16.042,51.44,43.709,0.0,False,GPU -30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1275.0,+41/-37,0.125,28.82,19.23,17.729,3351.28,0.544,0.0,False,CPU -31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1268.0,+55/-60,0.141,29.25,15.68,17.648,1053.84,0.169,0.0,False,GPU -32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1267.0,+54/-50,0.126,29.33,20.53,17.082,2977.49,2.415,0.0,False,CPU -33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1248.0,+46/-42,0.092,30.63,18.92,18.335,3351.28,0.064,0.0,False,CPU -34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1246.0,+54/-57,0.13,30.78,16.14,19.416,15.54,0.346,0.0,False,GPU -35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1242.0,+70/-62,0.161,30.99,15.99,18.731,1423.89,9.208,0.0,False,CPU -36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1228.0,+57/-49,0.109,31.94,18.75,19.284,8.03,0.059,0.0,False,CPU -37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1209.0,+46/-41,0.063,33.18,21.12,18.821,14.89,4.433,0.0,False,GPU -38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1200.0,+81/-70,0.105,33.77,19.75,20.935,399.55,1.466,0.0,False,CPU -39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1194.0,+44/-56,0.072,34.22,22.35,19.321,2977.49,0.156,0.0,False,CPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1192.0,+58/-53,0.07,34.32,19.34,20.087,1423.89,0.749,0.0,False,CPU -41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1171.0,+41/-48,0.025,35.67,28.99,19.87,2.79,0.238,0.0,False,CPU -42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1163.0,+82/-76,0.078,36.19,20.05,21.958,399.55,0.174,0.0,False,CPU -43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1156.0,+50/-53,0.054,36.6,29.76,20.316,3.15,0.141,0.0,False,CPU -44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1150.0,+76/-89,0.091,36.98,19.99,22.17,571.65,1.39,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1110.0,+75/-74,0.051,39.41,18.55,23.019,571.65,0.129,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1059.0,+60/-66,0.013,42.3,30.74,24.096,11.29,0.204,0.0,False,CPU -47,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1044.0,+91/-103,0.082,43.07,25.61,27.29,2.59,0.533,0.0,False,GPU -48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1025.0,+75/-110,0.022,44.09,36.08,26.058,4.95,1.012,0.0,False,GPU -49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1023.0,+78/-96,0.051,44.16,29.56,30.914,483.02,0.874,0.0,False,CPU -50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1009.0,+67/-72,0.0,44.85,28.65,25.623,5.04,0.646,0.0,False,CPU -51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1008.0,+86/-111,0.027,44.91,28.44,28.815,121.17,1.63,0.0,False,CPU -52,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+67/-78,0.008,45.29,37.69,28.537,0.46,0.068,0.0,False,CPU -53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,998.0,+76/-98,0.033,45.36,31.3,31.504,483.02,0.113,0.0,False,CPU -54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,959.0,+78/-102,0.009,47.15,40.28,30.018,0.42,0.074,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,923.0,+98/-136,0.022,48.6,34.38,34.781,2.39,0.143,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,920.0,+105/-116,0.012,48.72,32.71,34.463,121.17,0.136,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,582.0,+114/-167,0.0,56.35,51.76,49.793,0.28,0.023,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1604.0,+78/-60,8.48,0.711,9.9,4.49,610.76,8.082,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1602.0,+88/-82,5.856,0.725,9.98,3.67,2289.05,8.114,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1583.0,+75/-67,6.98,0.697,10.77,4.36,2289.05,1.254,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1582.0,+78/-79,7.294,0.713,10.84,5.18,6.07,0.762,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[LimiX (default) [0.40% IMPUTED]](https://arxiv.org/abs/2509.03505),1489.0,+85/-70,10.809,0.559,15.35,5.56,4.62,0.633,➖,0.4,True,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1488.0,+85/-75,11.831,0.54,15.43,6.02,3302.86,45.811,✔️,7.14,True,GPU +6,🧠⚡,Foundation Model,[Mitra (default) [7.14% IMPUTED]](https://arxiv.org/abs/2510.21204),1484.0,+92/-78,12.423,0.556,15.65,6.52,144.27,3.772,✔️,7.14,True,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1457.0,+68/-52,12.392,0.433,17.16,9.79,3422.82,18.936,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1431.0,+68/-56,13.705,0.375,18.68,10.21,3170.35,1.694,✔️,0.0,False,GPU +9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1426.0,+79/-52,13.359,0.359,19.01,10.04,3143.8,1.884,✔️,0.0,False,CPU +10,🧠⚡,Foundation Model,[TabICL (default) [7.14% IMPUTED]](https://arxiv.org/abs/2502.05564),1416.0,+82/-60,13.369,0.421,19.62,6.98,8.89,1.714,✔️,7.14,True,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1409.0,+67/-62,11.946,0.399,20.04,7.07,5271.34,231.986,✔️,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1407.0,+106/-83,14.357,0.416,20.18,6.83,3302.86,0.929,✔️,7.14,True,GPU +13,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1389.0,+67/-50,15.512,0.368,21.32,10.91,330.95,2.359,➖,0.0,False,GPU +14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1376.0,+58/-60,14.668,0.291,22.09,11.93,3170.35,0.241,✔️,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1375.0,+72/-61,12.943,0.352,22.17,7.4,5271.34,33.986,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1374.0,+46/-36,15.375,0.252,22.22,15.36,735.1,1.901,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1373.0,+58/-52,14.106,0.28,22.32,11.72,3422.82,1.534,✔️,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabPFNv2 (default) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1372.0,+79/-73,15.443,0.396,22.38,9.29,4.15,0.42,✔️,7.14,True,GPU +19,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1349.0,+44/-29,15.756,0.196,23.88,17.3,735.1,0.311,✔️,0.0,False,CPU +20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1336.0,+41/-35,15.67,0.187,24.74,17.8,1714.62,0.768,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1330.0,+51/-49,15.2,0.2,25.11,10.89,6047.72,0.525,✔️,0.0,False,GPU +22,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1325.0,+39/-32,15.86,0.178,25.44,18.66,1714.62,0.098,✔️,0.0,False,CPU +23,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1313.0,+55/-58,16.35,0.201,26.25,15.38,1053.84,3.063,✔️,0.0,False,GPU +24,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1306.0,+45/-47,16.611,0.163,26.73,18.68,797.9,2.228,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1302.0,+78/-77,16.103,0.234,27.03,12.02,6047.72,8.744,✔️,0.0,False,GPU +26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1301.0,+44/-38,16.742,0.164,27.05,18.55,7.19,0.118,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1298.0,+59/-61,17.244,0.224,27.3,14.87,10.78,0.17,✔️,0.0,False,GPU +28,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1291.0,+42/-42,16.833,0.126,27.72,18.81,797.9,0.353,✔️,0.0,False,CPU +29,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1291.0,+77/-81,16.042,0.279,27.73,9.1,51.44,43.709,✔️,0.0,False,GPU +30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1275.0,+41/-37,17.729,0.125,28.82,19.23,3351.28,0.544,✔️,0.0,False,CPU +31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1268.0,+55/-60,17.648,0.141,29.25,15.68,1053.84,0.169,✔️,0.0,False,GPU +32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1267.0,+54/-50,17.082,0.126,29.33,20.53,2977.49,2.415,✔️,0.0,False,CPU +33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1248.0,+46/-42,18.335,0.092,30.63,18.92,3351.28,0.064,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1246.0,+54/-57,19.416,0.13,30.78,16.14,15.54,0.346,✔️,0.0,False,GPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1242.0,+70/-62,18.731,0.161,30.99,15.99,1423.89,9.208,✔️,0.0,False,CPU +36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1228.0,+57/-49,19.284,0.109,31.94,18.75,8.03,0.059,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1209.0,+46/-41,18.821,0.063,33.18,21.12,14.89,4.433,✔️,0.0,False,GPU +38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1200.0,+81/-70,20.935,0.105,33.77,19.75,399.55,1.466,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1194.0,+44/-56,19.321,0.072,34.22,22.35,2977.49,0.156,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1192.0,+58/-53,20.087,0.07,34.32,19.34,1423.89,0.749,✔️,0.0,False,CPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1171.0,+41/-48,19.87,0.025,35.67,28.99,2.79,0.238,✔️,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1163.0,+82/-76,21.958,0.078,36.19,20.05,399.55,0.174,✔️,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1156.0,+50/-53,20.316,0.054,36.6,29.76,3.15,0.141,✔️,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1150.0,+76/-89,22.17,0.091,36.98,19.99,571.65,1.39,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1110.0,+75/-74,23.019,0.051,39.41,18.55,571.65,0.129,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1059.0,+60/-66,24.096,0.013,42.3,30.74,11.29,0.204,✔️,0.0,False,CPU +47,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1044.0,+91/-103,27.29,0.082,43.07,25.61,2.59,0.533,➖,0.0,False,GPU +48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1025.0,+75/-110,26.058,0.022,44.09,36.08,4.95,1.012,✔️,0.0,False,GPU +49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1023.0,+78/-96,30.914,0.051,44.16,29.56,483.02,0.874,✔️,0.0,False,CPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1009.0,+67/-72,25.623,0.0,44.85,28.65,5.04,0.646,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1008.0,+86/-111,28.815,0.027,44.91,28.44,121.17,1.63,✔️,0.0,False,CPU +52,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+67/-78,28.537,0.008,45.29,37.69,0.46,0.068,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),998.0,+76/-98,31.504,0.033,45.36,31.3,483.02,0.113,✔️,0.0,False,CPU +54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),959.0,+78/-102,30.018,0.009,47.15,40.28,0.42,0.074,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),923.0,+98/-136,34.781,0.022,48.6,34.38,2.39,0.143,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),920.0,+105/-116,34.463,0.012,48.72,32.71,121.17,0.136,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),582.0,+114/-167,49.793,0.0,56.35,51.76,0.28,0.023,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_small/winrate_matrix.png.zip index 53a70e51f324c5bf76de97ed0ca359bdadf8e096..8fa40c958c736d9746d8bab938eed53d05843a32 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:200b8552c3bdee6f12c7e400ee9072a603a3c16e19bb2504a125a5470d1004ba +oid sha256:c6a7cc9e39c35919c1c2185fb2db054f6f1882a257f6ac4e2301f96bafaa7c11 size 2759671 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 14d00c02112a621975f53d4b7d91213220004358..b9279ecfb3085e261d89d00749f14ed2f29c0fa1 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:125ea11c0f227c4777b562ba3bacf5e58787fec2d5882b38a4827a94dd64658f +oid sha256:856bc4aeeccd3e684e795eff61eda1466cbd64477131ba8be24f024afddb5ec7 size 471650 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip index 07ecdf64d41515fc7b88505d16f663cd393225ce..bad1869aae7da54d4b36405eb45cf5d9111f49a3 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2d7aa972473c7332a2a51427ae7d348c7c8f3356ca7743f7d51666626d916e1c -size 1073311 +oid sha256:605635e0ca32888ba4fade93b2d53a3e50a4081fa32f16d340a54937efec630c +size 1051061 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip index d0d93a3aac096f6d2eaa7c23456727c62f3c92eb..b4e32d90960bb8de9907240843440b65ce427ea0 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0e95d8250a0f919b54feb88a15237118d5bc6ca6b828a78c0d58a23e2ec57851 +oid sha256:8b5c0420a632ab0b5df292d9576706ec365a5dc6fbb02c7621f8c6cee483107b size 120230 diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/website_leaderboard.csv index 3e24e20a24c4cda44ab4bc96fbb38217adfa91f4..9eb8e140ebb7cc10688c97ebfcf83baa97abecd1 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1634.0,+83/-79,0.728,9.78,3.8,6.234,2168.17,7.961,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1631.0,+70/-57,0.747,9.87,4.93,7.553,5.99,0.633,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1625.0,+85/-64,0.701,10.13,4.67,9.074,604.44,8.082,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1614.0,+81/-69,0.702,10.59,4.34,7.377,2168.17,1.23,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1544.0,+69/-56,0.576,13.78,5.67,11.21,3445.6,48.236,0.0,False,GPU -5,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1536.0,+80/-59,0.593,14.17,6.16,11.857,146.65,4.101,0.0,False,GPU -6,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1518.0,+87/-73,0.58,15.09,5.58,11.42,4.37,0.585,0.0,False,GPU -7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1486.0,+61/-44,0.447,16.86,9.64,13.101,3169.9,16.793,0.0,False,GPU -8,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1459.0,+71/-61,0.447,18.45,6.6,12.877,8.89,1.743,0.0,False,GPU -9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1454.0,+71/-46,0.373,18.73,10.01,14.494,2828.45,1.596,0.0,False,GPU -10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1452.0,+82/-72,0.442,18.89,6.45,13.931,3445.6,0.995,0.0,False,GPU -11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1444.0,+66/-55,0.356,19.32,10.43,14.093,2898.23,1.884,0.0,False,CPU -12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1425.0,+77/-70,0.388,20.47,6.97,12.671,5119.36,231.986,0.0,False,GPU -13,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1424.0,+60/-53,0.369,20.56,10.58,15.951,330.95,2.023,0.0,False,GPU -14,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1410.0,+84/-72,0.421,21.42,8.85,15.11,4.06,0.436,0.0,False,GPU -15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1400.0,+56/-54,0.297,22.04,11.42,14.802,3169.9,1.446,0.0,False,GPU -16,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1400.0,+82/-72,0.355,22.07,7.06,13.572,5119.36,26.967,0.0,False,GPU -17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1399.0,+72/-51,0.29,22.12,11.71,15.476,2828.45,0.216,0.0,False,GPU -18,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1388.0,+42/-35,0.23,22.83,15.77,16.363,647.56,1.72,0.0,False,CPU -19,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1362.0,+37/-30,0.175,24.46,17.6,16.741,647.56,0.284,0.0,False,CPU -20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1356.0,+50/-36,0.186,24.9,17.76,16.602,1465.86,0.692,0.0,False,CPU -21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1355.0,+56/-50,0.195,24.97,11.03,15.889,5944.88,0.516,0.0,False,GPU -22,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1342.0,+48/-35,0.173,25.77,18.98,16.823,1465.86,0.091,0.0,False,CPU -23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1326.0,+64/-56,0.23,26.84,14.3,18.007,10.42,0.155,0.0,False,GPU -24,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1325.0,+46/-41,0.157,26.9,19.33,17.617,766.06,1.917,0.0,False,CPU -25,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1323.0,+95/-83,0.242,27.06,11.78,17.015,5944.88,8.396,0.0,False,GPU -26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1320.0,+45/-44,0.159,27.23,18.32,17.726,5.72,0.11,0.0,False,CPU -27,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1320.0,+87/-95,0.3,27.25,8.62,16.755,50.32,43.824,0.0,False,GPU -28,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1320.0,+50/-53,0.172,27.25,16.63,17.396,934.1,2.767,0.0,False,GPU -29,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1309.0,+39/-36,0.119,27.97,19.24,17.841,766.06,0.278,0.0,False,CPU -30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1297.0,+46/-41,0.117,28.77,18.8,18.633,2686.11,0.47,0.0,False,CPU -31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1294.0,+56/-54,0.135,28.96,20.03,17.813,2862.05,2.158,0.0,False,CPU -32,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1283.0,+45/-58,0.14,29.68,15.39,19.49,14.8,0.337,0.0,False,GPU -33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1277.0,+50/-50,0.12,30.07,16.88,18.738,934.1,0.165,0.0,False,GPU -34,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1275.0,+77/-87,0.174,30.22,15.28,19.42,1358.63,8.067,0.0,False,CPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1269.0,+45/-48,0.083,30.63,18.46,19.24,2686.11,0.056,0.0,False,CPU -36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1244.0,+50/-47,0.089,32.25,19.07,20.124,7.4,0.057,0.0,False,CPU -37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1243.0,+44/-39,0.068,32.34,20.31,19.291,13.83,3.953,0.0,False,GPU -38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1219.0,+64/-78,0.076,33.88,19.08,20.796,1358.63,0.897,0.0,False,CPU -39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1213.0,+53/-43,0.061,34.25,22.19,20.109,2862.05,0.152,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1199.0,+60/-67,0.076,35.15,24.0,22.25,370.85,1.466,0.0,False,CPU -41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1184.0,+45/-43,0.019,36.07,30.03,21.0,2.4,0.218,0.0,False,CPU -42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1160.0,+62/-70,0.046,37.55,25.98,23.266,370.85,0.161,0.0,False,CPU -43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1153.0,+36/-44,0.027,37.95,32.6,21.599,2.9,0.133,0.0,False,CPU -44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1133.0,+70/-69,0.043,39.17,25.14,23.712,527.42,1.39,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1093.0,+61/-65,0.011,41.45,24.97,24.551,527.42,0.123,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1081.0,+56/-63,0.014,42.1,30.05,24.892,10.38,0.188,0.0,False,CPU -47,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1071.0,+83/-130,0.088,42.63,24.72,27.865,2.43,0.495,0.0,False,GPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1058.0,+100/-110,0.054,43.29,28.54,30.113,436.81,0.874,0.0,False,CPU -49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1034.0,+89/-103,0.024,44.5,35.91,27.438,4.4,0.974,0.0,False,GPU -50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1033.0,+101/-105,0.036,44.53,30.27,30.656,436.81,0.106,0.0,False,CPU -51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1032.0,+72/-81,0.0,44.59,27.79,26.107,4.73,0.623,0.0,False,CPU -52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1008.0,+77/-109,0.015,45.72,32.67,30.107,113.26,1.55,0.0,False,CPU -53,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+61/-83,0.003,46.1,38.93,29.212,0.45,0.066,0.0,False,CPU -54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,969.0,+75/-112,0.009,47.45,40.18,30.601,0.4,0.07,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,958.0,+91/-127,0.023,47.9,33.34,33.436,2.16,0.139,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,909.0,+87/-143,0.001,49.71,34.2,36.035,113.26,0.12,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,600.0,+121/-261,0.0,56.33,51.43,49.89,0.26,0.023,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1634.0,+83/-79,6.234,0.728,9.78,3.8,2168.17,7.961,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1631.0,+70/-57,7.553,0.747,9.87,4.93,5.99,0.633,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1625.0,+85/-64,9.074,0.701,10.13,4.67,604.44,8.082,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1614.0,+81/-69,7.377,0.702,10.59,4.34,2168.17,1.23,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1544.0,+69/-56,11.21,0.576,13.78,5.67,3445.6,48.236,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1536.0,+80/-59,11.857,0.593,14.17,6.16,146.65,4.101,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1518.0,+87/-73,11.42,0.58,15.09,5.58,4.37,0.585,➖,0.0,False,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1486.0,+61/-44,13.101,0.447,16.86,9.64,3169.9,16.793,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1459.0,+71/-61,12.877,0.447,18.45,6.6,8.89,1.743,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1454.0,+71/-46,14.494,0.373,18.73,10.01,2828.45,1.596,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1452.0,+82/-72,13.931,0.442,18.89,6.45,3445.6,0.995,✔️,0.0,False,GPU +11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1444.0,+66/-55,14.093,0.356,19.32,10.43,2898.23,1.884,✔️,0.0,False,CPU +12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1425.0,+77/-70,12.671,0.388,20.47,6.97,5119.36,231.986,✔️,0.0,False,GPU +13,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1424.0,+60/-53,15.951,0.369,20.56,10.58,330.95,2.023,➖,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1410.0,+84/-72,15.11,0.421,21.42,8.85,4.06,0.436,✔️,0.0,False,GPU +15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1400.0,+56/-54,14.802,0.297,22.04,11.42,3169.9,1.446,✔️,0.0,False,GPU +16,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1400.0,+82/-72,13.572,0.355,22.07,7.06,5119.36,26.967,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1399.0,+72/-51,15.476,0.29,22.12,11.71,2828.45,0.216,✔️,0.0,False,GPU +18,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1388.0,+42/-35,16.363,0.23,22.83,15.77,647.56,1.72,✔️,0.0,False,CPU +19,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1362.0,+37/-30,16.741,0.175,24.46,17.6,647.56,0.284,✔️,0.0,False,CPU +20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1356.0,+50/-36,16.602,0.186,24.9,17.76,1465.86,0.692,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1355.0,+56/-50,15.889,0.195,24.97,11.03,5944.88,0.516,✔️,0.0,False,GPU +22,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1342.0,+48/-35,16.823,0.173,25.77,18.98,1465.86,0.091,✔️,0.0,False,CPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1326.0,+64/-56,18.007,0.23,26.84,14.3,10.42,0.155,✔️,0.0,False,GPU +24,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1325.0,+46/-41,17.617,0.157,26.9,19.33,766.06,1.917,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1323.0,+95/-83,17.015,0.242,27.06,11.78,5944.88,8.396,✔️,0.0,False,GPU +26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1320.0,+45/-44,17.726,0.159,27.23,18.32,5.72,0.11,✔️,0.0,False,CPU +27,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1320.0,+87/-95,16.755,0.3,27.25,8.62,50.32,43.824,✔️,0.0,False,GPU +28,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1320.0,+50/-53,17.396,0.172,27.25,16.63,934.1,2.767,✔️,0.0,False,GPU +29,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1309.0,+39/-36,17.841,0.119,27.97,19.24,766.06,0.278,✔️,0.0,False,CPU +30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1297.0,+46/-41,18.633,0.117,28.77,18.8,2686.11,0.47,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1294.0,+56/-54,17.813,0.135,28.96,20.03,2862.05,2.158,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1283.0,+45/-58,19.49,0.14,29.68,15.39,14.8,0.337,✔️,0.0,False,GPU +33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1277.0,+50/-50,18.738,0.12,30.07,16.88,934.1,0.165,✔️,0.0,False,GPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1275.0,+77/-87,19.42,0.174,30.22,15.28,1358.63,8.067,✔️,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1269.0,+45/-48,19.24,0.083,30.63,18.46,2686.11,0.056,✔️,0.0,False,CPU +36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1244.0,+50/-47,20.124,0.089,32.25,19.07,7.4,0.057,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1243.0,+44/-39,19.291,0.068,32.34,20.31,13.83,3.953,✔️,0.0,False,GPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1219.0,+64/-78,20.796,0.076,33.88,19.08,1358.63,0.897,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1213.0,+53/-43,20.109,0.061,34.25,22.19,2862.05,0.152,✔️,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1199.0,+60/-67,22.25,0.076,35.15,24.0,370.85,1.466,✔️,0.0,False,CPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1184.0,+45/-43,21.0,0.019,36.07,30.03,2.4,0.218,✔️,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1160.0,+62/-70,23.266,0.046,37.55,25.98,370.85,0.161,✔️,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1153.0,+36/-44,21.599,0.027,37.95,32.6,2.9,0.133,✔️,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1133.0,+70/-69,23.712,0.043,39.17,25.14,527.42,1.39,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1093.0,+61/-65,24.551,0.011,41.45,24.97,527.42,0.123,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1081.0,+56/-63,24.892,0.014,42.1,30.05,10.38,0.188,✔️,0.0,False,CPU +47,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1071.0,+83/-130,27.865,0.088,42.63,24.72,2.43,0.495,➖,0.0,False,GPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1058.0,+100/-110,30.113,0.054,43.29,28.54,436.81,0.874,✔️,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1034.0,+89/-103,27.438,0.024,44.5,35.91,4.4,0.974,✔️,0.0,False,GPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1033.0,+101/-105,30.656,0.036,44.53,30.27,436.81,0.106,✔️,0.0,False,CPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1032.0,+72/-81,26.107,0.0,44.59,27.79,4.73,0.623,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1008.0,+77/-109,30.107,0.015,45.72,32.67,113.26,1.55,✔️,0.0,False,CPU +53,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+61/-83,29.212,0.003,46.1,38.93,0.45,0.066,✔️,0.0,False,CPU +54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),969.0,+75/-112,30.601,0.009,47.45,40.18,0.4,0.07,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),958.0,+91/-127,33.436,0.023,47.9,33.34,2.16,0.139,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),909.0,+87/-143,36.035,0.001,49.71,34.2,113.26,0.12,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),600.0,+121/-261,49.89,0.0,56.33,51.43,0.26,0.023,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip index d95123a58ea83701b518cce39a14bf179497db75..c7309b3a8849e785b5086e846aa08d1860522c78 100644 --- a/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fc0f448c09c36a25e04e35f41f96d68f04df436f0f0fd1ee65a44df5d6af2519 +oid sha256:6540112dedaffdf05bc9503bfd629e0202ba29e3036de071a9d79292807a443b size 2801926 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index 026ccc3d2e50968a4ecb229dc6a83aecc4cf4502..d4802503480f1965655786e083e3dac556db20c5 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ad5e312dd06fafd0cf2724488f1f948d7da696f42e1ff4cc102477662390c2b0 +oid sha256:5f1b9f78b7791b06b684b5096d1fc9b28f6d7aa11c1fce9c77c7fa92c980273e size 472876 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip index f060ce78db3612e8a709a424fdb17a68f2cc947f..6acb5521d7adef875b5a8fdb03c881dfab42dfba 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6210edaa62ab3520e79558f5995e74429e8594969089932a89f062690443f3b2 -size 1019606 +oid sha256:e5cbdc87d5255748326ffc8ae81b7d27245925787568802cd830dd1bb1dc34f4 +size 994907 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip index 686463bd674296dbca0f3c1057b342174f3ae3ef..c084561a4283df1a086828f10b11249b881a2130 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e3d957aebfa4fd233504ba7215a147cecf3c26084090132fba117661707e7f33 +oid sha256:bd738ebbfd20708f2c4cacf7dd853255d12d3b3a88bca7b82b9c598afb65cf58 size 172927 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/website_leaderboard.csv index fdd8297dc3173f3f41147939737e658c0b626dd2..b8bb6c4898eae2ea332fd830c592c63af9fc9814 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1724.0,+151/-70,0.833,7.12,4.6,9.367,1045.67,7.086,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1710.0,+222/-98,0.825,7.55,3.44,6.694,2710.27,1.347,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1706.0,+284/-137,0.825,7.67,2.39,5.887,2710.27,8.099,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1578.0,+229/-151,0.66,12.67,4.96,8.831,10.23,0.844,0.0,False,GPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1504.0,+192/-99,0.428,16.41,6.12,14.488,6219.24,3.78,0.0,False,CPU -5,🧠🔁,Neural Network,[LimiX (default) [12.50% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1497.0,+174/-104,0.471,16.84,6.1,12.101,5.85,0.91,12.5,True,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1493.0,+163/-107,0.49,17.05,7.75,12.999,4150.25,19.131,0.0,False,GPU -7,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1482.0,+262/-213,0.593,17.66,5.63,16.891,2589.89,10.803,25.0,True,GPU -8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1467.0,+128/-74,0.432,18.53,10.07,14.123,4150.25,3.423,0.0,False,GPU -9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1452.0,+121/-82,0.395,19.44,11.32,16.878,5885.87,1.632,0.0,False,GPU -10,🧠🔁,Neural Network,[Mitra (default) [25.00% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1446.0,+200/-163,0.467,19.8,9.18,18.817,161.61,3.705,25.0,True,GPU -11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1437.0,+120/-84,0.318,20.35,14.42,16.499,1377.29,5.202,0.0,False,CPU -12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1434.0,+96/-58,0.303,20.56,17.07,14.953,3104.39,1.095,0.0,False,CPU -13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1428.0,+103/-77,0.291,20.92,15.57,16.719,1377.29,1.259,0.0,False,CPU -14,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1417.0,+164/-123,0.387,21.57,9.17,18.584,4635.14,212.285,0.0,False,GPU -15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1415.0,+113/-80,0.338,21.74,11.29,17.724,5885.87,0.299,0.0,False,GPU -16,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1409.0,+95/-59,0.248,22.06,17.73,15.572,3104.39,0.212,0.0,False,CPU -17,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1409.0,+198/-168,0.421,22.11,8.31,18.891,2589.89,0.54,25.0,True,GPU -18,🧠⚡,Foundation Model,[TabPFNv2 (default) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1398.0,+188/-143,0.357,22.75,11.93,17.461,6.47,0.363,25.0,True,GPU -19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1389.0,+92/-81,0.244,23.32,18.49,16.984,2210.02,0.974,0.0,False,CPU -20,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1386.0,+106/-80,0.209,23.52,15.97,16.17,7127.77,9.214,0.0,False,GPU -21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1383.0,+100/-84,0.235,23.69,18.39,17.116,2210.02,3.985,0.0,False,CPU -22,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1376.0,+72/-52,0.187,24.16,12.11,16.529,7127.77,0.776,0.0,False,GPU -23,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1364.0,+176/-118,0.33,24.87,10.15,20.408,4635.14,34.77,0.0,False,GPU -24,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1348.0,+147/-123,0.258,25.92,16.87,19.531,1980.6,3.605,0.0,False,GPU -25,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1346.0,+132/-110,0.27,26.08,18.97,20.079,507.73,4.428,0.0,False,GPU -26,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1341.0,+109/-97,0.191,26.36,18.26,18.908,36484.23,0.827,0.0,False,CPU -27,🧠⚡,Foundation Model,[TabICL (default) [12.50% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1319.0,+169/-167,0.269,27.84,12.57,22.325,10.98,1.898,12.5,True,GPU -28,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1317.0,+124/-118,0.238,27.97,17.42,20.804,17.83,0.197,0.0,False,GPU -29,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1309.0,+180/-174,0.287,28.5,11.75,20.881,737.91,1.864,0.0,False,CPU -30,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1307.0,+169/-141,0.238,28.59,14.2,21.669,717.36,1.612,0.0,False,CPU -31,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1304.0,+86/-71,0.147,28.79,24.75,19.049,50.54,0.227,0.0,False,CPU -32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1300.0,+136/-109,0.113,29.04,22.15,20.573,3901.28,2.028,0.0,False,CPU -33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1299.0,+107/-113,0.147,29.14,20.44,19.677,36484.23,0.127,0.0,False,CPU -34,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1267.0,+126/-136,0.148,31.18,23.71,21.594,19.31,0.108,0.0,False,CPU -35,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1262.0,+163/-157,0.196,31.48,13.13,23.154,717.36,0.326,0.0,False,CPU -36,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1259.0,+202/-138,0.219,31.68,9.38,23.472,43.39,42.575,0.0,False,GPU -37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1259.0,+199/-203,0.226,31.69,10.23,22.263,737.91,0.316,0.0,False,CPU -38,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1257.0,+176/-164,0.207,31.79,13.98,24.229,2644.99,12.294,0.0,False,CPU -39,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1253.0,+134/-120,0.161,32.06,18.15,21.886,1980.6,0.61,0.0,False,GPU -40,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1236.0,+119/-144,0.119,33.09,23.26,22.898,3901.28,0.223,0.0,False,CPU -41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1233.0,+118/-80,0.067,33.31,28.8,20.542,4.72,0.542,0.0,False,CPU -42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1212.0,+147/-122,0.134,34.53,26.5,22.667,5.04,0.429,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1206.0,+131/-146,0.1,34.95,19.21,25.342,2644.99,1.148,0.0,False,CPU -44,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1194.0,+79/-84,0.031,35.67,29.76,25.353,17.94,0.691,0.0,False,GPU -45,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1145.0,+58/-74,0.0,38.53,35.95,25.964,19.2,7.624,0.0,False,GPU -46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1081.0,+118/-200,0.05,41.88,36.27,29.352,9.42,1.675,0.0,False,GPU -47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1048.0,+72/-119,0.0,43.44,34.63,28.741,13.69,0.314,0.0,False,CPU -48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1001.0,+214/-366,0.043,45.5,24.18,39.389,151.69,2.346,0.0,False,CPU -49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+107/-182,0.0,45.54,42.17,35.605,0.74,0.137,0.0,False,CPU -50,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,979.0,+94/-231,0.0,46.4,44.39,40.046,892.95,0.818,0.0,False,CPU -51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,950.0,+122/-199,0.0,47.5,45.33,34.685,8.51,0.734,0.0,False,CPU -52,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,943.0,+84/-237,0.0,47.74,45.41,41.043,892.95,0.146,0.0,False,CPU -53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,899.0,+132/-177,0.0,49.25,46.82,40.735,0.69,0.137,0.0,False,CPU -54,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,850.0,+110/-155,0.0,50.73,50.4,39.73,5.05,1.013,0.0,False,GPU -55,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,833.0,+268/-604,0.04,51.18,39.36,49.678,151.69,0.258,0.0,False,CPU -56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,731.0,+89/-254,0.0,53.53,53.21,47.224,4.53,0.164,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,188.0,+131/-663,0.0,57.77,57.76,70.56,0.52,0.076,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1724.0,+151/-70,9.367,0.833,7.12,4.6,1045.67,7.086,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1710.0,+222/-98,6.694,0.825,7.55,3.44,2710.27,1.347,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1706.0,+284/-137,5.887,0.825,7.67,2.39,2710.27,8.099,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1578.0,+229/-151,8.831,0.66,12.67,4.96,10.23,0.844,✔️,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1504.0,+192/-99,14.488,0.428,16.41,6.12,6219.24,3.78,✔️,0.0,False,CPU +5,🧠⚡,Foundation Model,[LimiX (default) [12.50% IMPUTED]](https://arxiv.org/abs/2509.03505),1497.0,+174/-104,12.101,0.471,16.84,6.1,5.85,0.91,➖,12.5,True,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1493.0,+163/-107,12.999,0.49,17.05,7.75,4150.25,19.131,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1482.0,+262/-213,16.891,0.593,17.66,5.63,2589.89,10.803,✔️,25.0,True,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1467.0,+128/-74,14.123,0.432,18.53,10.07,4150.25,3.423,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1452.0,+121/-82,16.878,0.395,19.44,11.32,5885.87,1.632,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[Mitra (default) [25.00% IMPUTED]](https://arxiv.org/abs/2510.21204),1446.0,+200/-163,18.817,0.467,19.8,9.18,161.61,3.705,✔️,25.0,True,GPU +11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1437.0,+120/-84,16.499,0.318,20.35,14.42,1377.29,5.202,✔️,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1434.0,+96/-58,14.953,0.303,20.56,17.07,3104.39,1.095,✔️,0.0,False,CPU +13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1428.0,+103/-77,16.719,0.291,20.92,15.57,1377.29,1.259,✔️,0.0,False,CPU +14,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1417.0,+164/-123,18.584,0.387,21.57,9.17,4635.14,212.285,✔️,0.0,False,GPU +15,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1415.0,+113/-80,17.724,0.338,21.74,11.29,5885.87,0.299,✔️,0.0,False,GPU +16,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1409.0,+95/-59,15.572,0.248,22.06,17.73,3104.39,0.212,✔️,0.0,False,CPU +17,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1409.0,+198/-168,18.891,0.421,22.11,8.31,2589.89,0.54,✔️,25.0,True,GPU +18,🧠⚡,Foundation Model,[TabPFNv2 (default) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1398.0,+188/-143,17.461,0.357,22.75,11.93,6.47,0.363,✔️,25.0,True,GPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1389.0,+92/-81,16.984,0.244,23.32,18.49,2210.02,0.974,✔️,0.0,False,CPU +20,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1386.0,+106/-80,16.17,0.209,23.52,15.97,7127.77,9.214,✔️,0.0,False,GPU +21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1383.0,+100/-84,17.116,0.235,23.69,18.39,2210.02,3.985,✔️,0.0,False,CPU +22,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1376.0,+72/-52,16.529,0.187,24.16,12.11,7127.77,0.776,✔️,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1364.0,+176/-118,20.408,0.33,24.87,10.15,4635.14,34.77,✔️,0.0,False,GPU +24,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1348.0,+147/-123,19.531,0.258,25.92,16.87,1980.6,3.605,✔️,0.0,False,GPU +25,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1346.0,+132/-110,20.079,0.27,26.08,18.97,507.73,4.428,➖,0.0,False,GPU +26,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1341.0,+109/-97,18.908,0.191,26.36,18.26,36484.23,0.827,✔️,0.0,False,CPU +27,🧠⚡,Foundation Model,[TabICL (default) [12.50% IMPUTED]](https://arxiv.org/abs/2502.05564),1319.0,+169/-167,22.325,0.269,27.84,12.57,10.98,1.898,✔️,12.5,True,GPU +28,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1317.0,+124/-118,20.804,0.238,27.97,17.42,17.83,0.197,✔️,0.0,False,GPU +29,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1309.0,+180/-174,20.881,0.287,28.5,11.75,737.91,1.864,✔️,0.0,False,CPU +30,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1307.0,+169/-141,21.669,0.238,28.59,14.2,717.36,1.612,✔️,0.0,False,CPU +31,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1304.0,+86/-71,19.049,0.147,28.79,24.75,50.54,0.227,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1300.0,+136/-109,20.573,0.113,29.04,22.15,3901.28,2.028,✔️,0.0,False,CPU +33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1299.0,+107/-113,19.677,0.147,29.14,20.44,36484.23,0.127,✔️,0.0,False,CPU +34,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1267.0,+126/-136,21.594,0.148,31.18,23.71,19.31,0.108,✔️,0.0,False,CPU +35,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1262.0,+163/-157,23.154,0.196,31.48,13.13,717.36,0.326,✔️,0.0,False,CPU +36,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1259.0,+202/-138,23.472,0.219,31.68,9.38,43.39,42.575,✔️,0.0,False,GPU +37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1259.0,+199/-203,22.263,0.226,31.69,10.23,737.91,0.316,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1257.0,+176/-164,24.229,0.207,31.79,13.98,2644.99,12.294,✔️,0.0,False,CPU +39,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1253.0,+134/-120,21.886,0.161,32.06,18.15,1980.6,0.61,✔️,0.0,False,GPU +40,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1236.0,+119/-144,22.898,0.119,33.09,23.26,3901.28,0.223,✔️,0.0,False,CPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1233.0,+118/-80,20.542,0.067,33.31,28.8,4.72,0.542,✔️,0.0,False,CPU +42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1212.0,+147/-122,22.667,0.134,34.53,26.5,5.04,0.429,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1206.0,+131/-146,25.342,0.1,34.95,19.21,2644.99,1.148,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1194.0,+79/-84,25.353,0.031,35.67,29.76,17.94,0.691,✔️,0.0,False,GPU +45,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1145.0,+58/-74,25.964,0.0,38.53,35.95,19.2,7.624,✔️,0.0,False,GPU +46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1081.0,+118/-200,29.352,0.05,41.88,36.27,9.42,1.675,✔️,0.0,False,GPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1048.0,+72/-119,28.741,0.0,43.44,34.63,13.69,0.314,✔️,0.0,False,CPU +48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1001.0,+214/-366,39.389,0.043,45.5,24.18,151.69,2.346,✔️,0.0,False,CPU +49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+107/-182,35.605,0.0,45.54,42.17,0.74,0.137,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),979.0,+94/-231,40.046,0.0,46.4,44.39,892.95,0.818,✔️,0.0,False,CPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),950.0,+122/-199,34.685,0.0,47.5,45.33,8.51,0.734,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),943.0,+84/-237,41.043,0.0,47.74,45.41,892.95,0.146,✔️,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),899.0,+132/-177,40.735,0.0,49.25,46.82,0.69,0.137,✔️,0.0,False,CPU +54,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),850.0,+110/-155,39.73,0.0,50.73,50.4,5.05,1.013,➖,0.0,False,GPU +55,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),833.0,+268/-604,49.678,0.04,51.18,39.36,151.69,0.258,✔️,0.0,False,CPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),731.0,+89/-254,47.224,0.0,53.53,53.21,4.53,0.164,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),188.0,+131/-663,70.56,0.0,57.77,57.76,0.52,0.076,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/winrate_matrix.png.zip index 9e3a4380d6f5fb5059290fb0a1f036a3028e0b85..28d0e6d9e2aa5417ce16ccd9a6eaadef594129f6 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f0d6eadd414018d1f579c8c97dcf83ee1d0246d73eb94cdc815cfe99123fcfea +oid sha256:c481a2e8dbf92568e12b01670f04f10913e3e14d39b84dd50b7d19d512c3fa60 size 2634680 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index ea1a0801d6eb9497ee4cfb5098840b2741363401..7275e144f4c0fdaf48b873a4340a9a6b478cb2db 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c47205bdca7aa36ca18a681f3208a8301fb66cac6e50cb2fe2db8dc8b94d5fb3 +oid sha256:7c7d7746e8be4d7d51f869a2d131235909cd5216e31cba92ee2fad01460c798b size 444887 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip index 3d874df419b12bff529f12338f841427910b5e3b..7edce77b3e40d6cb88d090d7dd9879ddfb47aaed 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bfbe22e2d0b0e5e61981f999f52c0d0122cd3042e043406c10a193936f6e0d2a -size 977457 +oid sha256:4e0dd60a73c61b50382fc50fba7a66b269ce0d2ea60baae93afdbdf5d70c14d5 +size 951729 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip index 6a0351baa6eee60f21fe1c713c366298d4ccc924..5c753136d7f344589c615e02a51e65d7e91c6d85 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b731b62ff22925891023afe53607d0a7970d248ab5dc8eb1f3b2be2282b0979a +oid sha256:627adb339e35ae5905ec75da15ae52e93ff026a215063a1f917a984f7297d8c6 size 106892 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/website_leaderboard.csv index c4f7f7abf85b38dcd22115ae20c737de87eb39da..a419ca2395ccc34bf4d1abe9827fdbf5cbfafd8c 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/website_leaderboard.csv @@ -1,54 +1,54 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,9270.0,+0/-0,1.0,1.11,1.06,0.019,133.98,8.253,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,8803.0,+0/-0,0.934,2.56,2.2,1.7,276.87,4.662,0.0,False,CPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,8700.0,+0/-0,0.927,3.0,2.84,1.878,276.86,3.44,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,8621.0,+0/-0,0.905,3.33,3.0,2.424,133.98,2.816,0.0,False,GPU -4,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,6195.0,+0/-0,0.594,6.78,6.31,9.581,61.95,0.17,0.0,False,CPU -5,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,6182.0,+0/-0,0.595,6.89,6.45,9.545,7606.4,0.826,0.0,False,CPU -6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,6134.0,+0/-0,0.573,7.33,7.13,10.011,157.72,3.649,0.0,False,CPU -7,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,6038.0,+0/-0,0.578,8.22,7.72,9.898,7606.4,0.048,0.0,False,CPU -8,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,5997.0,+0/-0,0.578,8.67,7.91,9.921,61.95,0.016,0.0,False,CPU -9,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,5959.0,+0/-0,0.552,9.11,8.66,10.441,157.72,0.374,0.0,False,CPU -10,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,5682.0,+0/-0,0.508,11.89,10.98,11.347,2688.02,0.24,0.0,False,CPU -11,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,5671.0,+0/-0,0.518,12.0,11.9,11.168,466.47,1.036,0.0,False,CPU -12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,5573.0,+0/-0,0.499,13.0,12.51,11.536,2688.02,0.726,0.0,False,CPU -13,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,5519.0,+0/-0,0.501,13.56,13.39,11.505,0.39,0.051,0.0,False,CPU -14,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,5486.0,+0/-0,0.501,13.89,13.77,11.505,466.47,0.051,0.0,False,CPU -15,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,5145.0,+0/-0,0.399,17.0,16.0,13.544,1.48,0.53,0.0,False,GPU -16,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,5108.0,+0/-0,0.423,17.33,17.24,13.063,7.54,0.047,0.0,False,CPU -17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,5082.0,+0/-0,0.407,17.56,17.52,13.396,4423.24,29.2,0.0,False,GPU -18,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,5032.0,+0/-0,0.389,18.0,17.84,13.739,17.7,0.086,0.0,False,CPU -19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,4812.0,+0/-0,0.354,19.67,19.62,14.412,4423.24,2.237,0.0,False,GPU -20,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,4739.0,+0/-0,0.348,20.11,20.02,14.523,2.82,1.073,0.0,False,GPU -21,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,3202.0,+0/-0,0.25,22.11,22.11,16.353,20.98,0.329,0.0,False,CPU -22,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,2532.0,+0/-0,0.132,24.22,24.17,18.457,4638.91,343.215,0.0,False,GPU -23,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,2475.0,+0/-0,0.115,24.56,24.47,18.732,2389.31,0.267,0.0,False,CPU -24,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,2475.0,+0/-0,0.113,24.56,24.51,18.777,20.98,0.027,0.0,False,CPU -25,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,2456.0,+0/-0,0.109,24.67,24.61,18.841,2389.31,0.023,0.0,False,CPU -26,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1738.0,+0/-0,0.0,27.44,27.43,20.674,14.87,1.855,0.0,False,GPU -27,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1554.0,+0/-0,0.0,28.33,28.3,20.901,1993.14,0.452,0.0,False,GPU -28,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1404.0,+0/-0,0.0,29.22,29.16,21.65,1993.14,0.059,0.0,False,GPU -29,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1218.0,+0/-0,0.0,30.44,30.4,22.097,4638.91,43.781,0.0,False,GPU -30,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1218.0,+0/-0,0.0,30.44,30.35,22.475,40.54,38.741,0.0,False,GPU -31,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+0/-0,0.0,31.89,31.77,23.341,0.18,0.008,0.0,False,CPU -32,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,821.0,+0/-0,0.0,33.0,32.98,23.568,1294.93,0.417,0.0,False,GPU -33,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,665.0,+0/-0,0.0,33.89,33.86,24.01,1294.93,0.06,0.0,False,GPU -34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,599.0,+0/-0,0.0,34.22,34.18,24.919,0.18,0.04,0.0,False,CPU -35,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,-215.0,+0/-0,0.0,36.11,36.11,28.274,1863.58,6.302,0.0,False,GPU -36,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,-734.0,+0/-0,0.0,37.56,37.54,30.79,1863.58,1.131,0.0,False,GPU -37,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,-788.0,+0/-0,0.0,37.78,37.75,30.768,3.96,0.025,0.0,False,CPU -38,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,-1010.0,+0/-0,0.0,38.67,38.66,32.321,6.85,0.088,0.0,False,GPU -39,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,-1462.0,+0/-0,0.0,39.89,39.89,33.496,66.06,1.792,0.0,False,GPU -40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,-2063.0,+0/-0,0.0,41.0,41.0,37.017,7.55,0.149,0.0,False,GPU -41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,-2631.0,+0/-0,0.0,42.11,42.11,41.044,569.0,0.17,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,-2955.0,+0/-0,0.0,42.89,42.89,41.644,569.0,1.658,0.0,False,CPU -43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,-3626.0,+0/-0,0.0,44.11,44.11,50.207,0.07,0.015,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,-4076.0,+0/-0,0.0,45.22,45.21,51.289,2.86,0.085,0.0,False,CPU -45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-4255.0,+0/-0,0.0,45.78,45.77,51.687,30.41,0.089,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-4823.0,+0/-0,0.0,47.44,47.44,54.976,30.41,0.015,0.0,False,CPU -47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-5018.0,+0/-0,0.0,48.33,48.32,55.365,0.13,0.014,0.0,False,CPU -48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,-5106.0,+0/-0,0.0,48.78,48.72,55.242,3.09,0.997,0.0,False,GPU -49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,-5308.0,+0/-0,0.0,49.78,49.77,56.49,0.11,0.023,0.0,False,GPU -50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-5500.0,+0/-0,0.0,50.56,50.54,57.332,76.27,3.054,0.0,False,CPU -51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-6498.0,+0/-0,0.0,52.0,52.0,70.796,76.27,0.37,0.0,False,CPU -52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-7418.0,+0/-0,0.0,53.0,53.0,82.671,0.14,0.136,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),9270.0,+0/-0,0.019,1.0,1.11,1.06,133.98,8.253,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",8803.0,+0/-0,1.7,0.934,2.56,2.2,276.87,4.662,✔️,0.0,False,CPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",8700.0,+0/-0,1.878,0.927,3.0,2.84,276.86,3.44,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),8621.0,+0/-0,2.424,0.905,3.33,3.0,133.98,2.816,✔️,0.0,False,GPU +4,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),6195.0,+0/-0,9.581,0.594,6.78,6.31,61.95,0.17,✔️,0.0,False,CPU +5,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),6182.0,+0/-0,9.545,0.595,6.89,6.45,7606.4,0.826,✔️,0.0,False,CPU +6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),6134.0,+0/-0,10.011,0.573,7.33,7.13,157.72,3.649,✔️,0.0,False,CPU +7,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),6038.0,+0/-0,9.898,0.578,8.22,7.72,7606.4,0.048,✔️,0.0,False,CPU +8,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),5997.0,+0/-0,9.921,0.578,8.67,7.91,61.95,0.016,✔️,0.0,False,CPU +9,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),5959.0,+0/-0,10.441,0.552,9.11,8.66,157.72,0.374,✔️,0.0,False,CPU +10,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),5682.0,+0/-0,11.347,0.508,11.89,10.98,2688.02,0.24,✔️,0.0,False,CPU +11,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),5671.0,+0/-0,11.168,0.518,12.0,11.9,466.47,1.036,✔️,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),5573.0,+0/-0,11.536,0.499,13.0,12.51,2688.02,0.726,✔️,0.0,False,CPU +13,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),5519.0,+0/-0,11.505,0.501,13.56,13.39,0.39,0.051,✔️,0.0,False,CPU +14,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),5486.0,+0/-0,11.505,0.501,13.89,13.77,466.47,0.051,✔️,0.0,False,CPU +15,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),5145.0,+0/-0,13.544,0.399,17.0,16.0,1.48,0.53,✔️,0.0,False,GPU +16,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),5108.0,+0/-0,13.063,0.423,17.33,17.24,7.54,0.047,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),5082.0,+0/-0,13.396,0.407,17.56,17.52,4423.24,29.2,✔️,0.0,False,GPU +18,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),5032.0,+0/-0,13.739,0.389,18.0,17.84,17.7,0.086,✔️,0.0,False,CPU +19,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),4812.0,+0/-0,14.412,0.354,19.67,19.62,4423.24,2.237,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),4739.0,+0/-0,14.523,0.348,20.11,20.02,2.82,1.073,✔️,0.0,False,GPU +21,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),3202.0,+0/-0,16.353,0.25,22.11,22.11,20.98,0.329,✔️,0.0,False,CPU +22,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),2532.0,+0/-0,18.457,0.132,24.22,24.17,4638.91,343.215,✔️,0.0,False,GPU +23,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),2475.0,+0/-0,18.732,0.115,24.56,24.47,2389.31,0.267,✔️,0.0,False,CPU +24,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),2475.0,+0/-0,18.777,0.113,24.56,24.51,20.98,0.027,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),2456.0,+0/-0,18.841,0.109,24.67,24.61,2389.31,0.023,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1738.0,+0/-0,20.674,0.0,27.44,27.43,14.87,1.855,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1554.0,+0/-0,20.901,0.0,28.33,28.3,1993.14,0.452,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1404.0,+0/-0,21.65,0.0,29.22,29.16,1993.14,0.059,✔️,0.0,False,GPU +29,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1218.0,+0/-0,22.097,0.0,30.44,30.4,4638.91,43.781,✔️,0.0,False,GPU +30,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1218.0,+0/-0,22.475,0.0,30.44,30.35,40.54,38.741,✔️,0.0,False,GPU +31,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+0/-0,23.341,0.0,31.89,31.77,0.18,0.008,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),821.0,+0/-0,23.568,0.0,33.0,32.98,1294.93,0.417,✔️,0.0,False,GPU +33,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),665.0,+0/-0,24.01,0.0,33.89,33.86,1294.93,0.06,✔️,0.0,False,GPU +34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),599.0,+0/-0,24.919,0.0,34.22,34.18,0.18,0.04,✔️,0.0,False,CPU +35,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),-215.0,+0/-0,28.274,0.0,36.11,36.11,1863.58,6.302,✔️,0.0,False,GPU +36,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),-734.0,+0/-0,30.79,0.0,37.56,37.54,1863.58,1.131,✔️,0.0,False,GPU +37,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),-788.0,+0/-0,30.768,0.0,37.78,37.75,3.96,0.025,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),-1010.0,+0/-0,32.321,0.0,38.67,38.66,6.85,0.088,✔️,0.0,False,GPU +39,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),-1462.0,+0/-0,33.496,0.0,39.89,39.89,66.06,1.792,➖,0.0,False,GPU +40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),-2063.0,+0/-0,37.017,0.0,41.0,41.0,7.55,0.149,✔️,0.0,False,GPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),-2631.0,+0/-0,41.044,0.0,42.11,42.11,569.0,0.17,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),-2955.0,+0/-0,41.644,0.0,42.89,42.89,569.0,1.658,✔️,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),-3626.0,+0/-0,50.207,0.0,44.11,44.11,0.07,0.015,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),-4076.0,+0/-0,51.289,0.0,45.22,45.21,2.86,0.085,✔️,0.0,False,CPU +45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),-4255.0,+0/-0,51.687,0.0,45.78,45.77,30.41,0.089,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),-4823.0,+0/-0,54.976,0.0,47.44,47.44,30.41,0.015,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),-5018.0,+0/-0,55.365,0.0,48.33,48.32,0.13,0.014,✔️,0.0,False,CPU +48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),-5106.0,+0/-0,55.242,0.0,48.78,48.72,3.09,0.997,✔️,0.0,False,GPU +49,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),-5308.0,+0/-0,56.49,0.0,49.78,49.77,0.11,0.023,➖,0.0,False,GPU +50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),-5500.0,+0/-0,57.332,0.0,50.56,50.54,76.27,3.054,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),-6498.0,+0/-0,70.796,0.0,52.0,52.0,76.27,0.37,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-7418.0,+0/-0,82.671,0.0,53.0,53.0,0.14,0.136,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/winrate_matrix.png.zip index eae64c32497ce508043396d6517707de0418a1e0..0a738ee205cf1ea6e56f22298c3c5ec5a92c453c 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ca98ad1f4ef11a968abbb5e1e69ddb5c96de4a27e8d9b49b8ba51e74cefba221 +oid sha256:35921b6a257591d7f0e463089aecf81634d72a2b7d1ed995ff0ed5610cf59a47 size 1124587 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 7c001f0846bb12168fdb94774164df6fd9862553..1a2b7fb7f82d3fce972f5d06cc78b82b1425372e 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cb49dfae0005dcef9e69ea2ff9f7cd13b7b3a87337e6eab3eb521d6d7fdeae28 +oid sha256:2b4ddaa7ccd24dbe565113585d909e64694185e275c47c6a27e1369b15581373 size 466173 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip index 47d97bc1e2ccb504c48f8369485d78a9401240ee..7ea570ae99276b3100d482ba9e3444f1a1918169 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:23a01081f441da74c00b988b40d2dfe5b5533834b364c0122491d0b288ddd830 -size 1027072 +oid sha256:e259b664f7856914c363fdffa3e3b4a194971125ca4551c1bf241c094ac7a4e4 +size 977440 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip index a3a07ebc6bcad222592cdc256fed8d65b7c166e8..863e95395cf116c482726849783a9211543db174 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e473feba4e9bc3958b806f6fb5cea95d4b421b7f9197f5d915b1229cc18428b6 +oid sha256:7e21e39ca976633ff81babc99c5524189acb391cd2186298be234b759c6500ce size 167986 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/website_leaderboard.csv index 39db0b91a56bd83a7bc3fc7381c7ed4cdcd22ab2..330b59da3ca4a96ed49105b0bb88eae2b8928c87 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1751.0,+173/-64,0.819,7.7,5.05,10.436,1154.67,7.879,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1737.0,+229/-89,0.813,8.15,3.51,7.304,2742.37,1.218,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1724.0,+264/-127,0.8,8.61,2.91,6.725,2742.37,7.945,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1637.0,+311/-165,0.691,12.05,4.51,8.157,11.21,0.845,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1589.0,+185/-115,0.56,14.33,6.97,11.489,4471.59,21.354,0.0,False,GPU -5,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1586.0,+164/-99,0.539,14.47,5.46,10.495,6.22,1.092,0.0,False,GPU -6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1578.0,+333/-218,0.678,14.92,5.03,15.957,2986.29,15.005,14.29,True,GPU -7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1560.0,+120/-62,0.494,15.82,9.12,12.71,4471.59,3.76,0.0,False,GPU -8,🧠🔁,Neural Network,[Mitra (default) [14.29% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1523.0,+229/-174,0.533,17.86,8.32,18.171,200.23,4.42,14.29,True,GPU -9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1519.0,+153/-74,0.443,18.09,10.41,16.304,5929.46,1.73,0.0,False,GPU -10,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1514.0,+213/-86,0.355,18.38,8.2,16.315,6548.58,2.898,0.0,False,CPU -11,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1487.0,+230/-196,0.482,20.01,7.48,18.242,2986.29,0.63,14.29,True,GPU -12,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1477.0,+144/-80,0.384,20.59,10.38,17.164,5929.46,0.337,0.0,False,GPU -13,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1467.0,+185/-157,0.415,21.19,8.42,18.602,4631.38,190.647,0.0,False,GPU -14,🧠⚡,Foundation Model,[TabPFNv2 (default) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1466.0,+203/-166,0.408,21.23,10.93,16.621,7.26,0.403,14.29,True,GPU -15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1460.0,+92/-51,0.27,21.64,18.01,15.441,3272.91,1.185,0.0,False,CPU -16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1451.0,+161/-83,0.278,22.2,16.88,17.426,1502.45,6.33,0.0,False,CPU -17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1444.0,+136/-72,0.249,22.61,17.58,17.616,1502.45,1.367,0.0,False,CPU -18,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1431.0,+130/-108,0.309,23.4,17.52,18.163,684.78,4.816,0.0,False,GPU -19,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1429.0,+82/-46,0.206,23.52,19.44,16.176,3272.91,0.21,0.0,False,CPU -20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1426.0,+107/-136,0.295,23.75,15.56,18.282,2097.62,3.142,0.0,False,GPU -21,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1425.0,+196/-162,0.377,23.79,9.25,20.167,4631.38,30.351,0.0,False,GPU -22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1416.0,+127/-95,0.174,24.37,15.77,16.566,7359.32,9.026,0.0,False,GPU -23,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1411.0,+108/-73,0.202,24.67,19.44,17.767,2551.65,1.021,0.0,False,CPU -24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1409.0,+78/-58,0.157,24.81,11.48,16.831,7359.32,0.596,0.0,False,GPU -25,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1400.0,+124/-73,0.189,25.36,19.94,17.966,2551.65,4.527,0.0,False,CPU -26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1394.0,+158/-112,0.272,25.73,16.04,19.159,19.65,0.236,0.0,False,GPU -27,🧠⚡,Foundation Model,[TabICL (default) [14.29% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1344.0,+200/-161,0.252,28.94,11.94,23.44,11.18,2.132,14.29,True,GPU -28,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1341.0,+80/-84,0.129,29.14,24.71,20.246,63121.14,0.827,0.0,False,CPU -29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1336.0,+217/-166,0.237,29.49,12.7,21.741,2775.63,12.592,0.0,False,CPU -30,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1336.0,+190/-189,0.229,29.51,13.51,22.429,717.53,1.628,0.0,False,CPU -31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1333.0,+164/-116,0.103,29.68,21.86,20.836,4142.25,2.105,0.0,False,CPU -32,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1321.0,+93/-70,0.102,30.43,26.4,19.905,64.18,0.255,0.0,False,CPU -33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1319.0,+102/-118,0.184,30.56,16.78,20.614,2097.62,0.515,0.0,False,GPU -34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1302.0,+185/-212,0.239,31.6,13.41,22.496,755.05,2.155,0.0,False,CPU -35,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1301.0,+229/-201,0.25,31.69,8.53,23.615,46.25,45.74,0.0,False,GPU -36,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1294.0,+86/-94,0.081,32.12,26.74,21.074,63121.14,0.173,0.0,False,CPU -37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1288.0,+190/-222,0.199,32.47,12.32,23.78,717.53,0.339,0.0,False,CPU -38,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1279.0,+146/-130,0.108,33.06,24.87,22.716,20.91,0.126,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1276.0,+199/-153,0.115,33.22,17.71,23.099,2775.63,1.431,0.0,False,CPU -40,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1263.0,+171/-174,0.153,34.05,25.43,22.345,5.32,0.431,0.0,False,CPU -41,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1259.0,+129/-141,0.112,34.29,23.07,23.478,4142.25,0.287,0.0,False,CPU -42,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1247.0,+184/-230,0.171,34.98,10.68,24.026,755.05,0.317,0.0,False,CPU -43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1228.0,+38/-42,0.0,36.13,34.47,21.833,5.48,0.599,0.0,False,CPU -44,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1216.0,+98/-86,0.025,36.84,30.12,26.021,20.96,0.482,0.0,False,GPU -45,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1205.0,+52/-73,0.0,37.46,34.86,24.386,19.63,7.686,0.0,False,GPU -46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1156.0,+98/-168,0.057,40.18,34.66,25.653,10.84,2.189,0.0,False,GPU -47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1088.0,+73/-139,0.0,43.54,33.72,28.451,14.75,0.337,0.0,False,CPU -48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1077.0,+206/-394,0.049,44.06,22.38,36.825,173.37,1.638,0.0,False,CPU -49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1038.0,+99/-212,0.0,45.77,43.6,38.383,905.14,0.875,0.0,False,CPU -50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1005.0,+97/-238,0.0,47.07,44.56,39.053,905.14,0.175,0.0,False,CPU -51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1004.0,+153/-222,0.0,47.11,44.7,32.313,8.97,0.743,0.0,False,CPU -52,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+121/-211,0.0,47.28,43.84,37.357,0.85,0.148,0.0,False,CPU -53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,943.0,+142/-240,0.0,49.27,46.51,39.382,0.78,0.147,0.0,False,CPU -54,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,915.0,+88/-170,0.0,50.15,49.83,37.336,5.83,1.167,0.0,False,GPU -55,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,908.0,+270/-615,0.046,50.35,37.69,46.661,173.37,0.147,0.0,False,CPU -56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,776.0,+96/-280,0.0,53.56,53.19,46.061,4.72,0.188,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,261.0,+123/-665,0.0,57.74,57.72,68.83,0.57,0.067,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1751.0,+173/-64,10.436,0.819,7.7,5.05,1154.67,7.879,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1737.0,+229/-89,7.304,0.813,8.15,3.51,2742.37,1.218,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1724.0,+264/-127,6.725,0.8,8.61,2.91,2742.37,7.945,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1637.0,+311/-165,8.157,0.691,12.05,4.51,11.21,0.845,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1589.0,+185/-115,11.489,0.56,14.33,6.97,4471.59,21.354,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1586.0,+164/-99,10.495,0.539,14.47,5.46,6.22,1.092,➖,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1578.0,+333/-218,15.957,0.678,14.92,5.03,2986.29,15.005,✔️,14.29,True,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1560.0,+120/-62,12.71,0.494,15.82,9.12,4471.59,3.76,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[Mitra (default) [14.29% IMPUTED]](https://arxiv.org/abs/2510.21204),1523.0,+229/-174,18.171,0.533,17.86,8.32,200.23,4.42,✔️,14.29,True,GPU +9,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1519.0,+153/-74,16.304,0.443,18.09,10.41,5929.46,1.73,✔️,0.0,False,GPU +10,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1514.0,+213/-86,16.315,0.355,18.38,8.2,6548.58,2.898,✔️,0.0,False,CPU +11,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1487.0,+230/-196,18.242,0.482,20.01,7.48,2986.29,0.63,✔️,14.29,True,GPU +12,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1477.0,+144/-80,17.164,0.384,20.59,10.38,5929.46,0.337,✔️,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1467.0,+185/-157,18.602,0.415,21.19,8.42,4631.38,190.647,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabPFNv2 (default) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1466.0,+203/-166,16.621,0.408,21.23,10.93,7.26,0.403,✔️,14.29,True,GPU +15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1460.0,+92/-51,15.441,0.27,21.64,18.01,3272.91,1.185,✔️,0.0,False,CPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1451.0,+161/-83,17.426,0.278,22.2,16.88,1502.45,6.33,✔️,0.0,False,CPU +17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1444.0,+136/-72,17.616,0.249,22.61,17.58,1502.45,1.367,✔️,0.0,False,CPU +18,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1431.0,+130/-108,18.163,0.309,23.4,17.52,684.78,4.816,➖,0.0,False,GPU +19,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1429.0,+82/-46,16.176,0.206,23.52,19.44,3272.91,0.21,✔️,0.0,False,CPU +20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1426.0,+107/-136,18.282,0.295,23.75,15.56,2097.62,3.142,✔️,0.0,False,GPU +21,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1425.0,+196/-162,20.167,0.377,23.79,9.25,4631.38,30.351,✔️,0.0,False,GPU +22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1416.0,+127/-95,16.566,0.174,24.37,15.77,7359.32,9.026,✔️,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1411.0,+108/-73,17.767,0.202,24.67,19.44,2551.65,1.021,✔️,0.0,False,CPU +24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1409.0,+78/-58,16.831,0.157,24.81,11.48,7359.32,0.596,✔️,0.0,False,GPU +25,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1400.0,+124/-73,17.966,0.189,25.36,19.94,2551.65,4.527,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1394.0,+158/-112,19.159,0.272,25.73,16.04,19.65,0.236,✔️,0.0,False,GPU +27,🧠⚡,Foundation Model,[TabICL (default) [14.29% IMPUTED]](https://arxiv.org/abs/2502.05564),1344.0,+200/-161,23.44,0.252,28.94,11.94,11.18,2.132,✔️,14.29,True,GPU +28,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1341.0,+80/-84,20.246,0.129,29.14,24.71,63121.14,0.827,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1336.0,+217/-166,21.741,0.237,29.49,12.7,2775.63,12.592,✔️,0.0,False,CPU +30,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1336.0,+190/-189,22.429,0.229,29.51,13.51,717.53,1.628,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1333.0,+164/-116,20.836,0.103,29.68,21.86,4142.25,2.105,✔️,0.0,False,CPU +32,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1321.0,+93/-70,19.905,0.102,30.43,26.4,64.18,0.255,✔️,0.0,False,CPU +33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1319.0,+102/-118,20.614,0.184,30.56,16.78,2097.62,0.515,✔️,0.0,False,GPU +34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1302.0,+185/-212,22.496,0.239,31.6,13.41,755.05,2.155,✔️,0.0,False,CPU +35,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1301.0,+229/-201,23.615,0.25,31.69,8.53,46.25,45.74,✔️,0.0,False,GPU +36,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1294.0,+86/-94,21.074,0.081,32.12,26.74,63121.14,0.173,✔️,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1288.0,+190/-222,23.78,0.199,32.47,12.32,717.53,0.339,✔️,0.0,False,CPU +38,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1279.0,+146/-130,22.716,0.108,33.06,24.87,20.91,0.126,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1276.0,+199/-153,23.099,0.115,33.22,17.71,2775.63,1.431,✔️,0.0,False,CPU +40,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1263.0,+171/-174,22.345,0.153,34.05,25.43,5.32,0.431,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1259.0,+129/-141,23.478,0.112,34.29,23.07,4142.25,0.287,✔️,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1247.0,+184/-230,24.026,0.171,34.98,10.68,755.05,0.317,✔️,0.0,False,CPU +43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1228.0,+38/-42,21.833,0.0,36.13,34.47,5.48,0.599,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1216.0,+98/-86,26.021,0.025,36.84,30.12,20.96,0.482,✔️,0.0,False,GPU +45,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1205.0,+52/-73,24.386,0.0,37.46,34.86,19.63,7.686,✔️,0.0,False,GPU +46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1156.0,+98/-168,25.653,0.057,40.18,34.66,10.84,2.189,✔️,0.0,False,GPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1088.0,+73/-139,28.451,0.0,43.54,33.72,14.75,0.337,✔️,0.0,False,CPU +48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1077.0,+206/-394,36.825,0.049,44.06,22.38,173.37,1.638,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1038.0,+99/-212,38.383,0.0,45.77,43.6,905.14,0.875,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1005.0,+97/-238,39.053,0.0,47.07,44.56,905.14,0.175,✔️,0.0,False,CPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1004.0,+153/-222,32.313,0.0,47.11,44.7,8.97,0.743,✔️,0.0,False,CPU +52,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+121/-211,37.357,0.0,47.28,43.84,0.85,0.148,✔️,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),943.0,+142/-240,39.382,0.0,49.27,46.51,0.78,0.147,✔️,0.0,False,CPU +54,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),915.0,+88/-170,37.336,0.0,50.15,49.83,5.83,1.167,➖,0.0,False,GPU +55,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),908.0,+270/-615,46.661,0.046,50.35,37.69,173.37,0.147,✔️,0.0,False,CPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),776.0,+96/-280,46.061,0.0,53.56,53.19,4.72,0.188,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),261.0,+123/-665,68.83,0.0,57.74,57.72,0.57,0.067,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/winrate_matrix.png.zip index 5049ab9e852d249566e6cfb5340742ab6bd3b82b..ff288e173c5328d8029a3f594f7867cff5038bc9 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:29cab2deb2da72071dea2ced83b3f2c8292d28cf3738ed27dcdfc3ecf29e1db3 +oid sha256:2f55ba77437cdf32cc6097928bc3b4ec3998b63fb850b15f511fd7aadc6cf4a6 size 2647495 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 490d61ee1c997046948b35ce8d13badf37cd4503..99ae29aafd20ef6e3a6efce576f564bfb8c2bdaf 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:93e27a9c51454bcf66318850fa618e22f7d7a9f539b1615f62efee62a3095252 +oid sha256:01058b28cdf66bea9ac44de109845a5d68623282edbae027f301b710860fbbf4 size 470825 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip index 063c551d022b9085fd6da874144145278f97a5e9..e93bbf13b6edcace10d5541e91c86bc4e9eac54b 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:afcc5018dab96739735269a5e491334464445c02b8bb54b2289b1e0fa0f68d1d -size 1035459 +oid sha256:52d3f25db480d8807d0731b89399343a0d0a2b337fd843fbfa6f16140e6a5a95 +size 993086 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip index fe734021b8a95d9e3ec02c1fb22d6535e7e6a41c..55f9afbe3f2734b4bbd475e2d971f90c32fa27c7 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7861e72ce185c1c220153766d6afcdf9f7cd2c8a9c2f2a0462381d00ee6bcf00 +oid sha256:d4793da334dd47f8dabcc8c3d71845b4bf61d19513b32a3bfb5b5502c9d40f1f size 122677 diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv index fe131ba2fc1d272b12013893e0a5463f60893637..76f7eaabc270279279db328ea3355f3fc07d6416 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1876.0,+270/-100,0.87,6.14,2.55,7.591,2710.27,7.889,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1852.0,+240/-92,0.851,6.75,3.29,8.338,2710.27,1.076,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1838.0,+207/-77,0.833,7.12,4.7,12.046,1374.26,12.843,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1799.0,+306/-137,0.807,8.24,3.94,8.785,10.23,0.844,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1788.0,+296/-112,0.791,8.58,4.37,12.979,3585.96,32.815,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1708.0,+168/-44,0.653,11.4,6.17,12.954,4150.25,19.131,0.0,False,GPU -6,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1688.0,+166/-61,0.628,12.2,5.26,11.831,5.85,0.91,0.0,False,GPU -7,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1680.0,+197/-107,0.622,12.51,7.31,15.607,205.23,4.488,0.0,False,GPU -8,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1633.0,+220/-107,0.562,14.52,6.55,15.646,3585.96,1.167,0.0,False,GPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1630.0,+135/-43,0.553,14.68,8.59,14.49,4150.25,3.423,0.0,False,GPU -10,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1592.0,+203/-102,0.476,16.45,9.67,13.798,6.47,0.427,0.0,False,GPU -11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1575.0,+284/-135,0.414,17.28,7.5,18.07,7307.18,4.205,0.0,False,CPU -12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1549.0,+239/-110,0.412,18.59,10.08,18.854,5885.87,1.632,0.0,False,GPU -13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1513.0,+108/-46,0.303,20.47,17.01,17.651,3104.39,1.095,0.0,False,CPU -14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1497.0,+197/-114,0.346,21.34,10.08,19.852,5885.87,0.299,0.0,False,GPU -15,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1481.0,+240/-202,0.377,22.22,8.34,21.539,4513.63,201.505,0.0,False,GPU -16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1474.0,+122/-64,0.203,22.61,14.46,18.788,7367.31,7.816,0.0,False,GPU -17,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1470.0,+95/-49,0.218,22.86,18.63,18.528,3104.39,0.18,0.0,False,CPU -18,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1468.0,+142/-63,0.236,22.95,18.11,20.186,2210.02,1.236,0.0,False,CPU -19,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1460.0,+224/-87,0.219,23.37,17.51,20.164,1377.29,6.652,0.0,False,CPU -20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1455.0,+147/-63,0.221,23.69,18.61,20.414,2210.02,7.723,0.0,False,CPU -21,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1450.0,+196/-70,0.188,23.97,18.81,20.379,1377.29,1.395,0.0,False,CPU -22,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1433.0,+202/-128,0.245,24.92,19.16,21.045,696.38,4.428,0.0,False,GPU -23,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1431.0,+237/-220,0.338,25.04,8.66,23.355,4513.63,26.967,0.0,False,GPU -24,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1424.0,+226/-151,0.294,25.44,10.59,21.754,12.57,2.243,0.0,False,GPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1420.0,+212/-145,0.267,25.68,15.1,22.067,17.83,0.197,0.0,False,GPU -26,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1412.0,+144/-153,0.205,26.15,21.17,21.235,1880.81,2.744,0.0,False,GPU -27,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1410.0,+86/-72,0.092,26.26,14.18,19.439,7367.31,0.587,0.0,False,GPU -28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1381.0,+315/-274,0.276,27.89,11.42,24.53,2644.99,12.716,0.0,False,CPU -29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1347.0,+200/-164,0.121,29.89,21.48,23.884,4654.35,2.028,0.0,False,CPU -30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1340.0,+132/-106,0.076,30.27,25.3,23.387,36484.23,0.712,0.0,False,CPU -31,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1320.0,+134/-78,0.06,31.45,27.13,22.957,264.83,0.227,0.0,False,CPU -32,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1318.0,+331/-279,0.291,31.55,7.6,27.104,40.51,46.546,0.0,False,GPU -33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1289.0,+146/-150,0.099,33.19,27.21,23.905,1880.81,0.353,0.0,False,GPU -34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1289.0,+262/-248,0.134,33.2,18.09,26.079,2644.99,1.47,0.0,False,CPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1282.0,+107/-99,0.027,33.59,27.61,24.339,36484.23,0.127,0.0,False,CPU -36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1278.0,+176/-198,0.106,33.8,27.13,26.121,717.36,1.838,0.0,False,CPU -37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1255.0,+115/-83,0.029,35.05,28.39,26.62,17.94,0.418,0.0,False,GPU -38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1247.0,+64/-58,0.0,35.52,33.72,24.705,4.72,0.714,0.0,False,CPU -39,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1239.0,+180/-243,0.119,35.94,22.24,26.195,737.91,2.289,0.0,False,CPU -40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1232.0,+70/-89,0.0,36.29,33.66,26.605,20.73,8.179,0.0,False,GPU -41,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1230.0,+181/-140,0.06,36.45,24.6,27.149,4654.35,0.223,0.0,False,CPU -42,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1223.0,+101/-89,0.0,36.81,34.51,26.38,14.75,0.108,0.0,False,CPU -43,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1212.0,+159/-217,0.065,37.39,29.45,27.709,717.36,0.326,0.0,False,CPU -44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1204.0,+196/-142,0.055,37.78,34.29,25.943,5.04,0.449,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1160.0,+160/-231,0.033,40.02,25.17,27.997,737.91,0.316,0.0,False,CPU -46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1156.0,+183/-247,0.067,40.22,33.93,29.106,9.81,2.345,0.0,False,GPU -47,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1069.0,+131/-286,0.0,44.15,42.1,36.78,892.95,0.895,0.0,False,CPU -48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1066.0,+132/-195,0.0,44.29,33.03,32.333,13.69,0.314,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1031.0,+119/-328,0.0,45.67,43.14,37.562,892.95,0.146,0.0,False,CPU -50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+170/-313,0.0,46.83,42.97,37.99,0.74,0.137,0.0,False,CPU -51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,985.0,+174/-259,0.0,47.35,44.55,34.643,8.51,0.828,0.0,False,CPU -52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,954.0,+212/-444,0.0,48.38,41.73,42.698,151.69,1.252,0.0,False,CPU -53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,932.0,+195/-360,0.0,49.06,45.92,40.173,0.69,0.137,0.0,False,CPU -54,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,851.0,+29/-142,0.0,51.27,51.09,42.312,5.05,1.013,0.0,False,GPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,775.0,+103/-312,0.0,52.93,52.56,43.542,4.53,0.164,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,580.0,+156/-543,0.0,55.58,55.16,54.165,151.69,0.099,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,178.0,+157/-1027,0.0,57.75,57.73,69.817,0.52,0.056,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1876.0,+270/-100,7.591,0.87,6.14,2.55,2710.27,7.889,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1852.0,+240/-92,8.338,0.851,6.75,3.29,2710.27,1.076,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1838.0,+207/-77,12.046,0.833,7.12,4.7,1374.26,12.843,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1799.0,+306/-137,8.785,0.807,8.24,3.94,10.23,0.844,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1788.0,+296/-112,12.979,0.791,8.58,4.37,3585.96,32.815,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1708.0,+168/-44,12.954,0.653,11.4,6.17,4150.25,19.131,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1688.0,+166/-61,11.831,0.628,12.2,5.26,5.85,0.91,➖,0.0,False,GPU +7,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1680.0,+197/-107,15.607,0.622,12.51,7.31,205.23,4.488,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1633.0,+220/-107,15.646,0.562,14.52,6.55,3585.96,1.167,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1630.0,+135/-43,14.49,0.553,14.68,8.59,4150.25,3.423,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1592.0,+203/-102,13.798,0.476,16.45,9.67,6.47,0.427,✔️,0.0,False,GPU +11,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1575.0,+284/-135,18.07,0.414,17.28,7.5,7307.18,4.205,✔️,0.0,False,CPU +12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1549.0,+239/-110,18.854,0.412,18.59,10.08,5885.87,1.632,✔️,0.0,False,GPU +13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1513.0,+108/-46,17.651,0.303,20.47,17.01,3104.39,1.095,✔️,0.0,False,CPU +14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1497.0,+197/-114,19.852,0.346,21.34,10.08,5885.87,0.299,✔️,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1481.0,+240/-202,21.539,0.377,22.22,8.34,4513.63,201.505,✔️,0.0,False,GPU +16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1474.0,+122/-64,18.788,0.203,22.61,14.46,7367.31,7.816,✔️,0.0,False,GPU +17,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1470.0,+95/-49,18.528,0.218,22.86,18.63,3104.39,0.18,✔️,0.0,False,CPU +18,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1468.0,+142/-63,20.186,0.236,22.95,18.11,2210.02,1.236,✔️,0.0,False,CPU +19,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1460.0,+224/-87,20.164,0.219,23.37,17.51,1377.29,6.652,✔️,0.0,False,CPU +20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1455.0,+147/-63,20.414,0.221,23.69,18.61,2210.02,7.723,✔️,0.0,False,CPU +21,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1450.0,+196/-70,20.379,0.188,23.97,18.81,1377.29,1.395,✔️,0.0,False,CPU +22,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1433.0,+202/-128,21.045,0.245,24.92,19.16,696.38,4.428,➖,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1431.0,+237/-220,23.355,0.338,25.04,8.66,4513.63,26.967,✔️,0.0,False,GPU +24,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1424.0,+226/-151,21.754,0.294,25.44,10.59,12.57,2.243,✔️,0.0,False,GPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1420.0,+212/-145,22.067,0.267,25.68,15.1,17.83,0.197,✔️,0.0,False,GPU +26,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1412.0,+144/-153,21.235,0.205,26.15,21.17,1880.81,2.744,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1410.0,+86/-72,19.439,0.092,26.26,14.18,7367.31,0.587,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1381.0,+315/-274,24.53,0.276,27.89,11.42,2644.99,12.716,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1347.0,+200/-164,23.884,0.121,29.89,21.48,4654.35,2.028,✔️,0.0,False,CPU +30,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1340.0,+132/-106,23.387,0.076,30.27,25.3,36484.23,0.712,✔️,0.0,False,CPU +31,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1320.0,+134/-78,22.957,0.06,31.45,27.13,264.83,0.227,✔️,0.0,False,CPU +32,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1318.0,+331/-279,27.104,0.291,31.55,7.6,40.51,46.546,✔️,0.0,False,GPU +33,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1289.0,+146/-150,23.905,0.099,33.19,27.21,1880.81,0.353,✔️,0.0,False,GPU +34,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1289.0,+262/-248,26.079,0.134,33.2,18.09,2644.99,1.47,✔️,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1282.0,+107/-99,24.339,0.027,33.59,27.61,36484.23,0.127,✔️,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1278.0,+176/-198,26.121,0.106,33.8,27.13,717.36,1.838,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1255.0,+115/-83,26.62,0.029,35.05,28.39,17.94,0.418,✔️,0.0,False,GPU +38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1247.0,+64/-58,24.705,0.0,35.52,33.72,4.72,0.714,✔️,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1239.0,+180/-243,26.195,0.119,35.94,22.24,737.91,2.289,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1232.0,+70/-89,26.605,0.0,36.29,33.66,20.73,8.179,✔️,0.0,False,GPU +41,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1230.0,+181/-140,27.149,0.06,36.45,24.6,4654.35,0.223,✔️,0.0,False,CPU +42,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1223.0,+101/-89,26.38,0.0,36.81,34.51,14.75,0.108,✔️,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1212.0,+159/-217,27.709,0.065,37.39,29.45,717.36,0.326,✔️,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1204.0,+196/-142,25.943,0.055,37.78,34.29,5.04,0.449,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1160.0,+160/-231,27.997,0.033,40.02,25.17,737.91,0.316,✔️,0.0,False,CPU +46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1156.0,+183/-247,29.106,0.067,40.22,33.93,9.81,2.345,✔️,0.0,False,GPU +47,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1069.0,+131/-286,36.78,0.0,44.15,42.1,892.95,0.895,✔️,0.0,False,CPU +48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1066.0,+132/-195,32.333,0.0,44.29,33.03,13.69,0.314,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1031.0,+119/-328,37.562,0.0,45.67,43.14,892.95,0.146,✔️,0.0,False,CPU +50,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+170/-313,37.99,0.0,46.83,42.97,0.74,0.137,✔️,0.0,False,CPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),985.0,+174/-259,34.643,0.0,47.35,44.55,8.51,0.828,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),954.0,+212/-444,42.698,0.0,48.38,41.73,151.69,1.252,✔️,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),932.0,+195/-360,40.173,0.0,49.06,45.92,0.69,0.137,✔️,0.0,False,CPU +54,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),851.0,+29/-142,42.312,0.0,51.27,51.09,5.05,1.013,➖,0.0,False,GPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),775.0,+103/-312,43.542,0.0,52.93,52.56,4.53,0.164,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),580.0,+156/-543,54.165,0.0,55.58,55.16,151.69,0.099,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),178.0,+157/-1027,69.817,0.0,57.75,57.73,0.52,0.056,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip index d6630dd27a82e3488e31f060d871b54649d2c4eb..0283a9a362ffb8fe9c812fb95f595f1e0065650b 100644 --- a/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a69f25312265ff9537b977a8308ec208056de898285291041eb7d4cbfd46b4a6 +oid sha256:4b515780fe16db2c2a6a719375aab2e3ff732c2395831088070bcc6b5d0cc3fd size 2656649 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index 9eaf7f1f340953ecf906da410d77e8a5b6ecfaff..18eb87b58a1276c0e199a118e22cb0d72b0d8cbc 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f0d7e8c50c6ce63284d5f95c16332b90947a79ba9940a24791aabbc42abdb297 +oid sha256:d0b9fd084e27e289410b1d17c6cb797cecffb8f820d2b875f34753ef58e0b65f size 450891 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip index 367f39271545a7b159c2e2db12d262837b6531aa..4ea67acb76c7c3342da5ec53791f072e93a1928e 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cbd624640db18c22b903e51841e2de7dab9a4a1dbadbcc60f2e58093ac5387ec -size 1035802 +oid sha256:f15cf0c6dce6d0688816075ec45b3651093bebe496aeb725ca68c9c6182cdd31 +size 1012565 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_all/tuning-impact-elo.png.zip index da4963fdc167d37fc4428017d1ef5ac1bd10add1..50b4fd66a30f280d47bd4d6e46d6e83fce0c2dc2 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0e3ae13c1cc306c69bd718739145f17cfd8a101c7dc3b9b0bb5fb8fb76aec1a5 +oid sha256:a8b54bcefa8b112e1bbf33cb098a29205bff08d9c739ef4537157ca4ed46b8b8 size 146665 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_regression/datasets_all/website_leaderboard.csv index a6d1ab79311d902a0de1df3bb9b31d8bcc26f029..a13830b2ec3c234295495c10706f3db5345ddd0a 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_all/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_all/website_leaderboard.csv @@ -1,56 +1,56 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1779.0,+172/-123,0.805,6.65,2.66,1.541,1709.05,8.122,0.0,False,GPU -1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1741.0,+134/-74,0.789,7.69,2.75,2.451,4786.55,239.537,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1723.0,+109/-80,0.689,8.24,4.37,2.962,649.34,5.869,0.0,False,GPU -3,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1691.0,+101/-62,0.676,9.28,5.76,3.249,3995.01,10.051,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1690.0,+109/-63,0.727,9.3,4.54,2.902,4786.55,38.501,0.0,False,GPU -5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1654.0,+164/-114,0.691,10.59,4.46,2.299,1709.05,0.812,0.0,False,GPU -6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1632.0,+109/-89,0.553,11.42,6.63,4.693,1866.07,6.07,0.0,False,CPU -7,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1598.0,+123/-63,0.603,12.77,4.99,3.844,46.62,39.212,0.0,False,GPU -8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1572.0,+107/-73,0.506,13.85,8.41,4.229,3995.01,0.844,0.0,False,GPU -9,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1567.0,+141/-106,0.56,14.06,7.91,3.812,7.04,0.508,0.0,False,GPU -10,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1558.0,+132/-115,0.499,14.44,5.51,5.58,3779.52,7.69,0.0,False,GPU -11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1506.0,+102/-62,0.361,16.85,10.35,6.274,3552.96,0.966,0.0,False,CPU -12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1498.0,+92/-84,0.356,17.22,13.25,6.736,700.15,9.322,0.0,False,CPU -13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1488.0,+107/-82,0.376,17.72,10.78,5.764,714.5,1.376,0.0,False,GPU -14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1483.0,+105/-68,0.335,17.95,10.1,6.394,3552.96,0.105,0.0,False,CPU -15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1469.0,+125/-91,0.348,18.65,13.69,5.166,4158.29,1.41,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1434.0,+89/-74,0.274,20.42,15.98,7.328,700.15,0.968,0.0,False,CPU -17,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1426.0,+183/-165,0.426,20.84,5.7,13.181,5.2,0.755,0.0,False,GPU -18,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1419.0,+60/-46,0.248,21.16,18.57,7.27,834.93,2.614,0.0,False,CPU -19,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1404.0,+94/-77,0.265,21.9,17.61,6.439,714.5,0.099,0.0,False,GPU -20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1398.0,+63/-51,0.218,22.21,19.08,7.418,834.93,0.388,0.0,False,CPU -21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1393.0,+108/-87,0.277,22.51,13.43,7.578,3779.52,0.396,0.0,False,GPU -22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1387.0,+120/-99,0.247,22.8,15.86,6.122,4158.29,0.167,0.0,False,GPU -23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1383.0,+91/-93,0.254,22.99,16.39,7.975,10.89,0.092,0.0,False,CPU -24,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1335.0,+170/-110,0.323,25.52,8.92,6.948,4223.87,27.543,46.15,True,GPU -25,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1292.0,+87/-74,0.138,27.84,21.63,9.117,15.5,0.299,0.0,False,GPU -26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1292.0,+127/-111,0.157,27.84,22.13,7.84,13.32,0.131,0.0,False,GPU -27,🧠🔁,Neural Network,[Mitra (default) [46.15% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1281.0,+136/-104,0.194,28.41,10.03,8.622,71.01,1.848,46.15,True,GPU -28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1260.0,+117/-101,0.111,29.53,20.46,9.33,4608.59,1.233,0.0,False,CPU -29,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1253.0,+144/-139,0.21,29.91,15.14,8.01,4223.87,0.452,46.15,True,GPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1250.0,+117/-90,0.087,30.07,24.85,8.854,8.9,1.645,0.0,False,GPU -31,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1228.0,+106/-100,0.101,31.22,23.16,11.668,158.22,0.844,0.0,False,CPU -32,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1222.0,+44/-43,0.006,31.49,30.02,9.759,2.11,0.275,0.0,False,CPU -33,🧠⚡,Foundation Model,[TabPFNv2 (default) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1205.0,+137/-137,0.153,32.37,19.91,9.409,2.8,0.313,46.15,True,GPU -34,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1200.0,+82/-82,0.071,32.61,28.64,10.418,2.24,0.242,0.0,False,CPU -35,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1199.0,+95/-95,0.076,32.65,22.52,11.88,158.22,0.151,0.0,False,CPU -36,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1198.0,+111/-102,0.076,32.69,24.55,10.122,4608.59,0.097,0.0,False,CPU -37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1172.0,+64/-71,0.033,33.99,31.5,12.437,515.73,0.771,0.0,False,CPU -38,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1165.0,+117/-116,0.038,34.34,26.93,12.184,2.45,0.743,0.0,False,GPU -39,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1159.0,+132/-162,0.108,34.62,25.51,12.916,2929.85,0.294,0.0,False,CPU -40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1128.0,+68/-83,0.023,36.1,33.54,12.925,515.73,0.124,0.0,False,CPU -41,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1112.0,+129/-163,0.09,36.89,27.31,13.453,2929.85,0.03,0.0,False,CPU -42,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1078.0,+103/-121,0.004,38.39,34.16,13.818,0.47,0.055,0.0,False,CPU -43,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1048.0,+120/-182,0.046,39.7,33.61,14.389,8.47,0.035,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1042.0,+100/-116,0.022,39.93,36.96,13.796,540.06,2.672,0.0,False,CPU -45,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1040.0,+89/-111,0.001,40.01,37.65,13.552,20.48,0.08,0.0,False,CPU -46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+49/-76,0.0,41.58,40.25,14.44,0.53,0.062,0.0,False,CPU -47,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,986.0,+100/-110,0.012,42.11,39.99,14.283,540.06,0.325,0.0,False,CPU -48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,896.0,+144/-178,0.0,45.08,39.28,19.61,92.55,0.897,0.0,False,CPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,860.0,+113/-170,0.0,46.1,44.94,18.544,2.6,0.392,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,792.0,+148/-192,0.0,47.8,43.61,22.032,92.55,0.052,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,664.0,+192/-310,0.0,50.22,42.38,29.211,0.19,0.04,0.0,False,CPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,490.0,+135/-304,0.0,52.38,52.21,36.423,193.95,0.173,0.0,False,CPU -53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,442.0,+158/-371,0.0,52.82,52.63,36.597,193.95,0.074,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,223.0,+141/-372,0.0,54.31,54.29,39.031,0.95,0.097,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1779.0,+172/-123,1.541,0.805,6.65,2.66,1709.05,8.122,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1741.0,+134/-74,2.451,0.789,7.69,2.75,4786.55,239.537,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1723.0,+109/-80,2.962,0.689,8.24,4.37,649.34,5.869,✔️,0.0,False,GPU +3,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1691.0,+101/-62,3.249,0.676,9.28,5.76,3995.01,10.051,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1690.0,+109/-63,2.902,0.727,9.3,4.54,4786.55,38.501,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1654.0,+164/-114,2.299,0.691,10.59,4.46,1709.05,0.812,✔️,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1632.0,+109/-89,4.693,0.553,11.42,6.63,1866.07,6.07,✔️,0.0,False,CPU +7,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1598.0,+123/-63,3.844,0.603,12.77,4.99,46.62,39.212,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1572.0,+107/-73,4.229,0.506,13.85,8.41,3995.01,0.844,✔️,0.0,False,GPU +9,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1567.0,+141/-106,3.812,0.56,14.06,7.91,7.04,0.508,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1558.0,+132/-115,5.58,0.499,14.44,5.51,3779.52,7.69,✔️,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1506.0,+102/-62,6.274,0.361,16.85,10.35,3552.96,0.966,✔️,0.0,False,CPU +12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1498.0,+92/-84,6.736,0.356,17.22,13.25,700.15,9.322,✔️,0.0,False,CPU +13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1488.0,+107/-82,5.764,0.376,17.72,10.78,714.5,1.376,✔️,0.0,False,GPU +14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1483.0,+105/-68,6.394,0.335,17.95,10.1,3552.96,0.105,✔️,0.0,False,CPU +15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1469.0,+125/-91,5.166,0.348,18.65,13.69,4158.29,1.41,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1434.0,+89/-74,7.328,0.274,20.42,15.98,700.15,0.968,✔️,0.0,False,CPU +17,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1426.0,+183/-165,13.181,0.426,20.84,5.7,5.2,0.755,➖,0.0,False,GPU +18,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1419.0,+60/-46,7.27,0.248,21.16,18.57,834.93,2.614,✔️,0.0,False,CPU +19,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1404.0,+94/-77,6.439,0.265,21.9,17.61,714.5,0.099,✔️,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1398.0,+63/-51,7.418,0.218,22.21,19.08,834.93,0.388,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1393.0,+108/-87,7.578,0.277,22.51,13.43,3779.52,0.396,✔️,0.0,False,GPU +22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1387.0,+120/-99,6.122,0.247,22.8,15.86,4158.29,0.167,✔️,0.0,False,GPU +23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1383.0,+91/-93,7.975,0.254,22.99,16.39,10.89,0.092,✔️,0.0,False,CPU +24,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1335.0,+170/-110,6.948,0.323,25.52,8.92,4223.87,27.543,✔️,46.15,True,GPU +25,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1292.0,+87/-74,9.117,0.138,27.84,21.63,15.5,0.299,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1292.0,+127/-111,7.84,0.157,27.84,22.13,13.32,0.131,✔️,0.0,False,GPU +27,🧠⚡,Foundation Model,[Mitra (default) [46.15% IMPUTED]](https://arxiv.org/abs/2510.21204),1281.0,+136/-104,8.622,0.194,28.41,10.03,71.01,1.848,✔️,46.15,True,GPU +28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1260.0,+117/-101,9.33,0.111,29.53,20.46,4608.59,1.233,✔️,0.0,False,CPU +29,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1253.0,+144/-139,8.01,0.21,29.91,15.14,4223.87,0.452,✔️,46.15,True,GPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1250.0,+117/-90,8.854,0.087,30.07,24.85,8.9,1.645,✔️,0.0,False,GPU +31,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1228.0,+106/-100,11.668,0.101,31.22,23.16,158.22,0.844,✔️,0.0,False,CPU +32,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1222.0,+44/-43,9.759,0.006,31.49,30.02,2.11,0.275,✔️,0.0,False,CPU +33,🧠⚡,Foundation Model,[TabPFNv2 (default) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1205.0,+137/-137,9.409,0.153,32.37,19.91,2.8,0.313,✔️,46.15,True,GPU +34,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1200.0,+82/-82,10.418,0.071,32.61,28.64,2.24,0.242,✔️,0.0,False,CPU +35,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1199.0,+95/-95,11.88,0.076,32.65,22.52,158.22,0.151,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1198.0,+111/-102,10.122,0.076,32.69,24.55,4608.59,0.097,✔️,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1172.0,+64/-71,12.437,0.033,33.99,31.5,515.73,0.771,✔️,0.0,False,CPU +38,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1165.0,+117/-116,12.184,0.038,34.34,26.93,2.45,0.743,✔️,0.0,False,GPU +39,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1159.0,+132/-162,12.916,0.108,34.62,25.51,2929.85,0.294,✔️,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1128.0,+68/-83,12.925,0.023,36.1,33.54,515.73,0.124,✔️,0.0,False,CPU +41,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1112.0,+129/-163,13.453,0.09,36.89,27.31,2929.85,0.03,✔️,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1078.0,+103/-121,13.818,0.004,38.39,34.16,0.47,0.055,✔️,0.0,False,CPU +43,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1048.0,+120/-182,14.389,0.046,39.7,33.61,8.47,0.035,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1042.0,+100/-116,13.796,0.022,39.93,36.96,540.06,2.672,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1040.0,+89/-111,13.552,0.001,40.01,37.65,20.48,0.08,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+49/-76,14.44,0.0,41.58,40.25,0.53,0.062,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),986.0,+100/-110,14.283,0.012,42.11,39.99,540.06,0.325,✔️,0.0,False,CPU +48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),896.0,+144/-178,19.61,0.0,45.08,39.28,92.55,0.897,✔️,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),860.0,+113/-170,18.544,0.0,46.1,44.94,2.6,0.392,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),792.0,+148/-192,22.032,0.0,47.8,43.61,92.55,0.052,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),664.0,+192/-310,29.211,0.0,50.22,42.38,0.19,0.04,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),490.0,+135/-304,36.423,0.0,52.38,52.21,193.95,0.173,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),442.0,+158/-371,36.597,0.0,52.82,52.63,193.95,0.074,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),223.0,+141/-372,39.031,0.0,54.31,54.29,0.95,0.097,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_all/winrate_matrix.png.zip index 841b3874e16f65754a5be18d6ac201d1ffe91c2b..0f6f06aab2f8ed4768a0c9370357148a111b5636 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:85ce88243265981856e7c5b5cf865891c948ed319ad43bdc8bf5145f2a0a2a19 +oid sha256:606159a521a7d039fd66432384bdd9d435724fcd34d513dc4993e88436d88a97 size 2131657 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index e2ec593d4ba6b3c234d468ed2d1d81ca06503bf4..dd56a8e951d78af2dc92084490677654d022ba72 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1602d02e7601316d5c1127a5af5c804cbf555616fe1ffe6fe6c9a2782101b00a +oid sha256:10a1fb80b2917bb99671af7473075c26a3c071ae4c30bc6f8e5297cb05fd1690 size 429005 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip index 22d8a2edb056361ffc2ce7ba43fd8899301f6e3a..c3c6ffc7f6c80ca425880c7deb9eac51cf340bca 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4f0151062424c02a1d161ee84eca4a76cb65b171253e10262409d438f75826b1 -size 984608 +oid sha256:422dfdc35bcd212f01a7e528ff65f7d427c5e32c1d35f7c2f4ee59a2827a7399 +size 928150 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/tuning-impact-elo.png.zip index 932ac8a4653da6f1fd49440732888c9e4383dbff..34716789f8d53fbc5a2e1e68aebe15438166d14b 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:809c87da8e16e4d96a7d5026be38632861635f0d0437620d175451a89da53662 +oid sha256:f3b29f0820d7bfc06bfea5a97fbff3701d1842dbb104382a4608c4b9ef057abc size 119720 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/website_leaderboard.csv index d41a2046e31a460a63a19e7ed98b54b547edec91..e3cd8978a55c78ce453557b9d24fccd2ef76e30c 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/website_leaderboard.csv @@ -1,52 +1,52 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,2069.0,+1045/-120,0.862,4.4,2.67,1.209,735.58,9.557,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1971.0,+783/-126,0.802,6.31,4.21,1.696,735.58,0.714,0.0,False,GPU -2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1922.0,+681/-41,0.713,7.47,5.68,2.304,2526.28,3.717,0.0,False,GPU -3,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1917.0,+1210/-123,0.782,7.6,2.16,1.293,4786.55,361.217,0.0,False,GPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1909.0,+515/-11,0.661,7.8,4.92,3.331,473.01,8.73,0.0,False,CPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1900.0,+797/-114,0.688,8.04,5.61,2.7,2028.29,1.675,0.0,False,GPU -6,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1884.0,+532/-21,0.644,8.47,4.35,3.488,430.4,7.174,0.0,False,GPU -7,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1850.0,+1175/-128,0.717,9.42,4.22,1.707,4786.55,41.599,0.0,False,GPU -8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1802.0,+856/-218,0.564,10.87,7.04,2.636,1.6,0.508,0.0,False,GPU -9,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1753.0,+1086/-143,0.591,12.42,4.95,2.355,45.77,39.212,0.0,False,GPU -10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1721.0,+794/-134,0.535,13.47,8.53,3.741,2028.29,0.081,0.0,False,GPU -11,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1684.0,+503/-95,0.474,14.71,9.45,4.288,425.61,0.615,0.0,False,GPU -12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1671.0,+376/-133,0.388,15.18,11.21,5.936,222.05,11.236,0.0,False,CPU -13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1650.0,+347/-105,0.356,15.93,8.83,6.099,1658.41,0.742,0.0,False,CPU -14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1623.0,+616/-142,0.443,16.89,13.4,4.108,2526.28,0.166,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1619.0,+361/-178,0.35,17.04,7.26,6.197,1658.41,0.059,0.0,False,CPU -16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1618.0,+331/-121,0.326,17.09,13.63,6.24,222.05,0.968,0.0,False,CPU -17,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1602.0,+522/-214,0.341,17.64,14.24,5.117,3440.87,0.619,0.0,False,GPU -18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1555.0,+327/-104,0.296,19.4,17.23,5.243,425.61,0.04,0.0,False,GPU -19,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1545.0,+667/-210,0.305,19.76,5.04,4.564,2.06,0.755,0.0,False,GPU -20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1533.0,+425/-345,0.326,20.22,14.49,6.882,5.49,0.037,0.0,False,CPU -21,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1526.0,+421/-182,0.24,20.44,18.38,5.96,3440.87,0.052,0.0,False,GPU -22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1509.0,+268/-136,0.217,21.09,19.43,6.83,282.13,1.74,0.0,False,CPU -23,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1506.0,+647/-156,0.285,21.18,18.43,5.272,6.88,0.094,0.0,False,GPU -24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1497.0,+267/-142,0.197,21.51,19.3,6.953,282.13,0.388,0.0,False,CPU -25,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1360.0,+317/-308,0.159,26.42,21.95,8.067,0.6,0.08,0.0,False,CPU -26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1330.0,+330/-211,0.091,27.49,25.06,7.827,5.03,0.025,0.0,False,GPU -27,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1305.0,+240/-126,0.011,28.33,27.32,8.79,2.28,0.13,0.0,False,GPU -28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1301.0,+209/-114,0.061,28.49,26.91,8.242,4348.61,0.528,0.0,False,CPU -29,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1292.0,+283/-160,0.013,28.78,27.91,8.309,0.55,0.233,0.0,False,CPU -30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1219.0,+204/-297,0.038,31.24,29.53,9.231,4348.61,0.034,0.0,False,CPU -31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1199.0,+231/-316,0.0,31.89,30.73,9.7,6.75,0.235,0.0,False,GPU -32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1176.0,+359/-419,0.022,32.62,30.66,10.264,35.67,0.529,0.0,False,CPU -33,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1156.0,+87/-156,0.0,33.27,33.0,9.993,79.08,0.546,0.0,False,CPU -34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1144.0,+260/-394,0.006,33.64,32.41,10.361,35.67,0.05,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1127.0,+112/-248,0.0,34.16,33.79,10.297,79.08,0.057,0.0,False,CPU -36,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1102.0,+387/-661,0.143,34.91,29.95,13.832,2044.56,0.128,0.0,False,CPU -37,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1037.0,+193/-452,0.001,36.8,35.28,13.252,15.08,0.028,0.0,False,CPU -38,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1029.0,+364/-709,0.114,37.02,33.51,14.402,2044.56,0.01,0.0,False,CPU -39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1026.0,+358/-676,0.006,37.11,35.07,12.06,0.17,0.049,0.0,False,CPU -40,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+37/-248,0.0,37.82,37.67,11.446,0.43,0.053,0.0,False,CPU -41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,976.0,+370/-780,0.102,38.47,35.09,15.083,3.66,0.011,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,848.0,+60/-540,0.0,41.4,41.16,14.323,318.98,0.827,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,794.0,+54/-529,0.0,42.44,42.26,14.781,318.98,0.114,0.0,False,CPU -44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,789.0,+112/-640,0.0,42.53,42.33,18.087,142.58,1.184,0.0,False,CPU -45,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,664.0,+83/-704,0.0,44.53,44.43,20.643,142.58,0.064,0.0,False,CPU -46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,607.0,+116/-707,0.0,45.27,45.07,18.384,1.36,0.111,0.0,False,CPU -47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,349.0,+100/-1185,0.0,47.51,47.41,28.1,0.06,0.034,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,81.0,+131/-1638,0.0,48.92,48.92,39.828,44.3,0.048,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-114.0,+182/-2167,0.0,49.77,49.76,40.21,44.3,0.011,0.0,False,CPU -50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-414.0,+224/-2121,0.0,50.8,50.8,43.378,0.13,0.018,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),2069.0,+1045/-120,1.209,0.862,4.4,2.67,735.58,9.557,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1971.0,+783/-126,1.696,0.802,6.31,4.21,735.58,0.714,✔️,0.0,False,GPU +2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1922.0,+681/-41,2.304,0.713,7.47,5.68,2526.28,3.717,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1917.0,+1210/-123,1.293,0.782,7.6,2.16,4786.55,361.217,✔️,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1909.0,+515/-11,3.331,0.661,7.8,4.92,473.01,8.73,✔️,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1900.0,+797/-114,2.7,0.688,8.04,5.61,2028.29,1.675,✔️,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1884.0,+532/-21,3.488,0.644,8.47,4.35,430.4,7.174,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1850.0,+1175/-128,1.707,0.717,9.42,4.22,4786.55,41.599,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1802.0,+856/-218,2.636,0.564,10.87,7.04,1.6,0.508,✔️,0.0,False,GPU +9,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1753.0,+1086/-143,2.355,0.591,12.42,4.95,45.77,39.212,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1721.0,+794/-134,3.741,0.535,13.47,8.53,2028.29,0.081,✔️,0.0,False,GPU +11,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1684.0,+503/-95,4.288,0.474,14.71,9.45,425.61,0.615,✔️,0.0,False,GPU +12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1671.0,+376/-133,5.936,0.388,15.18,11.21,222.05,11.236,✔️,0.0,False,CPU +13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1650.0,+347/-105,6.099,0.356,15.93,8.83,1658.41,0.742,✔️,0.0,False,CPU +14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1623.0,+616/-142,4.108,0.443,16.89,13.4,2526.28,0.166,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1619.0,+361/-178,6.197,0.35,17.04,7.26,1658.41,0.059,✔️,0.0,False,CPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1618.0,+331/-121,6.24,0.326,17.09,13.63,222.05,0.968,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1602.0,+522/-214,5.117,0.341,17.64,14.24,3440.87,0.619,✔️,0.0,False,GPU +18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1555.0,+327/-104,5.243,0.296,19.4,17.23,425.61,0.04,✔️,0.0,False,GPU +19,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1545.0,+667/-210,4.564,0.305,19.76,5.04,2.06,0.755,➖,0.0,False,GPU +20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1533.0,+425/-345,6.882,0.326,20.22,14.49,5.49,0.037,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1526.0,+421/-182,5.96,0.24,20.44,18.38,3440.87,0.052,✔️,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1509.0,+268/-136,6.83,0.217,21.09,19.43,282.13,1.74,✔️,0.0,False,CPU +23,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1506.0,+647/-156,5.272,0.285,21.18,18.43,6.88,0.094,✔️,0.0,False,GPU +24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1497.0,+267/-142,6.953,0.197,21.51,19.3,282.13,0.388,✔️,0.0,False,CPU +25,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1360.0,+317/-308,8.067,0.159,26.42,21.95,0.6,0.08,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1330.0,+330/-211,7.827,0.091,27.49,25.06,5.03,0.025,✔️,0.0,False,GPU +27,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1305.0,+240/-126,8.79,0.011,28.33,27.32,2.28,0.13,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1301.0,+209/-114,8.242,0.061,28.49,26.91,4348.61,0.528,✔️,0.0,False,CPU +29,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1292.0,+283/-160,8.309,0.013,28.78,27.91,0.55,0.233,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1219.0,+204/-297,9.231,0.038,31.24,29.53,4348.61,0.034,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1199.0,+231/-316,9.7,0.0,31.89,30.73,6.75,0.235,✔️,0.0,False,GPU +32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1176.0,+359/-419,10.264,0.022,32.62,30.66,35.67,0.529,✔️,0.0,False,CPU +33,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1156.0,+87/-156,9.993,0.0,33.27,33.0,79.08,0.546,✔️,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1144.0,+260/-394,10.361,0.006,33.64,32.41,35.67,0.05,✔️,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1127.0,+112/-248,10.297,0.0,34.16,33.79,79.08,0.057,✔️,0.0,False,CPU +36,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1102.0,+387/-661,13.832,0.143,34.91,29.95,2044.56,0.128,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1037.0,+193/-452,13.252,0.001,36.8,35.28,15.08,0.028,✔️,0.0,False,CPU +38,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1029.0,+364/-709,14.402,0.114,37.02,33.51,2044.56,0.01,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1026.0,+358/-676,12.06,0.006,37.11,35.07,0.17,0.049,✔️,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+37/-248,11.446,0.0,37.82,37.67,0.43,0.053,✔️,0.0,False,CPU +41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),976.0,+370/-780,15.083,0.102,38.47,35.09,3.66,0.011,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),848.0,+60/-540,14.323,0.0,41.4,41.16,318.98,0.827,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),794.0,+54/-529,14.781,0.0,42.44,42.26,318.98,0.114,✔️,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),789.0,+112/-640,18.087,0.0,42.53,42.33,142.58,1.184,✔️,0.0,False,CPU +45,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),664.0,+83/-704,20.643,0.0,44.53,44.43,142.58,0.064,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),607.0,+116/-707,18.384,0.0,45.27,45.07,1.36,0.111,✔️,0.0,False,CPU +47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),349.0,+100/-1185,28.1,0.0,47.51,47.41,0.06,0.034,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),81.0,+131/-1638,39.828,0.0,48.92,48.92,44.3,0.048,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),-114.0,+182/-2167,40.21,0.0,49.77,49.76,44.3,0.011,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),-414.0,+224/-2121,43.378,0.0,50.8,50.8,0.13,0.018,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/winrate_matrix.png.zip index ef957bbcc4409333f1f292772059a25c42dda644..d89d739d2a6bcb5a1ec4e70aade6b02883e9e1c6 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ff312f1b97b4a62e9b723c827e650657371d7f7845f952179911e893c04ff57c +oid sha256:9d3c74b837da47402b288b5d1b16560ff38851ecc10edfdc0ac1295c66466286 size 1489576 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 9637fe9797a805bc3880393fc2e964e90bba4b13..0145f7036b9213b0f283b3f10143bc8ad85aa92f 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d2598c44942741229ee3625a943a4e2d38bb1cf4218857161f7523d616305b56 +oid sha256:dca244a7249bc27fd9ccef2a0ec6aec3e9bff0b929743767fa7891cc015f4b97 size 449740 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip index 5cdf55f92355a664329c88d0a16e3f24ccb877d1..1af65caa1c21f07a8448f1029291ab99a1fa3f86 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:789580861ff5a749789c72048d60709ab6891d47ebb6fcd6392ef75936e60687 -size 1066800 +oid sha256:6d0d1fdb578bc3575be12ae1c031831a0e2e345fd640074e1954dc588c5621f5 +size 1055036 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_small/tuning-impact-elo.png.zip index e54a71e37a8be63038b9a34e797390de4195a62d..3d895dae4fe3589d606cf2e24a9cda961c6e7321 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:985f58682cb3eb29d070ea553d53182ecc1640815b0bc7db69a270d7e7440e7d +oid sha256:add6494fc27962748116fba959a0895e4776331e2ba44037e77963f479a874ec size 157524 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_regression/datasets_small/website_leaderboard.csv index d9ad1d77f80774b121f60757ced973abed73872b..a69c6b779ace8aa97c32b11679e3cb3369f4afd1 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_small/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_small/website_leaderboard.csv @@ -1,56 +1,56 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1732.0,+139/-53,0.779,7.74,3.31,3.175,4822.0,140.81,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1722.0,+271/-134,0.765,8.05,2.65,1.748,2306.87,7.231,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1721.0,+208/-83,0.707,8.09,4.39,2.633,709.58,5.864,0.0,False,GPU -3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1687.0,+107/-42,0.716,9.22,4.77,3.649,4822.0,22.324,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1664.0,+125/-50,0.658,10.05,5.86,3.591,4558.61,33.853,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1591.0,+114/-48,0.584,12.98,5.02,4.775,48.13,42.001,0.0,False,GPU -6,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1585.0,+224/-129,0.615,13.26,4.62,2.677,2306.87,0.902,0.0,False,GPU -7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1576.0,+131/-73,0.476,13.68,8.49,5.544,5805.42,4.31,0.0,False,CPU -8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1566.0,+119/-72,0.476,14.1,8.34,4.535,4558.61,2.234,0.0,False,GPU -9,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1525.0,+209/-141,0.537,16.05,8.57,4.547,7.67,0.547,0.0,False,GPU -10,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1514.0,+258/-178,0.525,16.57,6.01,4.137,6559.81,56.314,12.5,True,GPU -11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1498.0,+128/-64,0.346,17.42,11.6,6.384,4430.73,1.103,0.0,False,CPU -12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1476.0,+120/-73,0.319,18.5,14.95,7.235,1203.24,7.106,0.0,False,CPU -13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1476.0,+125/-64,0.31,18.52,13.37,6.518,4430.73,0.179,0.0,False,CPU -14,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1470.0,+177/-92,0.358,18.8,5.41,7.628,8977.36,8.017,0.0,False,GPU -15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1461.0,+187/-96,0.342,19.27,13.36,5.197,4438.37,1.981,0.0,False,GPU -16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1455.0,+118/-86,0.294,19.59,11.82,6.686,2039.04,2.144,0.0,False,GPU -17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1425.0,+95/-43,0.241,21.2,18.07,7.544,1265.82,3.15,0.0,False,CPU -18,🧠🔁,Neural Network,[Mitra (default) [12.50% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1423.0,+172/-124,0.315,21.3,6.84,6.857,93.39,2.396,12.5,True,GPU -19,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1419.0,+279/-232,0.478,21.52,6.2,18.567,6.44,0.709,0.0,False,GPU -20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1401.0,+106/-79,0.221,22.5,17.92,8.008,1203.24,0.956,0.0,False,CPU -21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1399.0,+88/-44,0.204,22.65,18.94,7.708,1265.82,0.476,0.0,False,CPU -22,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1384.0,+125/-75,0.215,23.47,17.87,7.186,2039.04,0.109,0.0,False,GPU -23,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1380.0,+255/-186,0.342,23.69,10.92,5.862,6559.81,0.485,12.5,True,GPU -24,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1369.0,+174/-114,0.237,24.27,14.61,6.224,4438.37,0.231,0.0,False,GPU -25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1361.0,+141/-86,0.192,24.72,17.85,8.658,12.31,0.204,0.0,False,CPU -26,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1339.0,+167/-97,0.155,25.96,13.44,9.748,8977.36,0.421,0.0,False,GPU -27,🧠⚡,Foundation Model,[TabPFNv2 (default) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1308.0,+225/-226,0.248,27.73,15.18,8.136,8.74,0.773,12.5,True,GPU -28,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1304.0,+171/-124,0.183,27.94,20.59,7.848,22.14,0.179,0.0,False,GPU -29,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1298.0,+135/-104,0.133,28.32,22.03,8.326,21.51,8.356,0.0,False,GPU -30,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1271.0,+153/-168,0.141,29.79,17.71,10.01,5150.31,1.592,0.0,False,CPU -31,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1270.0,+110/-149,0.149,29.84,19.99,12.545,664.2,1.363,0.0,False,CPU -32,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1240.0,+109/-141,0.118,31.48,18.81,12.829,664.2,0.274,0.0,False,CPU -33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1233.0,+75/-77,0.018,31.88,24.2,11.52,26.36,0.302,0.0,False,GPU -34,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1214.0,+152/-154,0.099,32.86,21.99,10.679,5150.31,0.136,0.0,False,CPU -35,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1213.0,+175/-164,0.079,32.94,22.94,12.343,4333.58,0.422,0.0,False,CPU -36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1208.0,+59/-44,0.0,33.18,31.5,10.665,4.3,0.318,0.0,False,CPU -37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1185.0,+95/-138,0.053,34.38,30.6,13.964,675.17,1.541,0.0,False,CPU -38,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1170.0,+178/-174,0.065,35.15,23.99,12.86,4333.58,0.037,0.0,False,CPU -39,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1143.0,+50/-59,0.0,36.48,35.38,11.888,4.06,0.299,0.0,False,CPU -40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1141.0,+88/-140,0.037,36.59,32.81,14.568,675.17,0.206,0.0,False,CPU -41,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1136.0,+107/-134,0.036,36.85,33.44,13.467,2214.9,7.795,0.0,False,CPU -42,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1112.0,+152/-226,0.048,37.98,26.64,14.305,10.2,2.718,0.0,False,GPU -43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1110.0,+96/-178,0.001,38.08,33.03,14.916,0.5,0.116,0.0,False,CPU -44,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1094.0,+148/-172,0.001,38.8,31.92,13.954,16.01,0.041,0.0,False,CPU -45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1077.0,+103/-140,0.02,39.57,37.02,13.971,2214.9,0.838,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1049.0,+116/-145,0.0,40.74,38.28,13.739,22.39,0.134,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+87/-150,0.0,42.69,40.63,16.312,0.63,0.12,0.0,False,CPU -48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,959.0,+106/-170,0.0,44.17,42.75,18.644,6.97,0.818,0.0,False,CPU -49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,954.0,+157/-376,0.0,44.34,35.99,20.561,88.43,0.512,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,853.0,+158/-384,0.0,47.34,41.05,22.9,88.43,0.051,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,762.0,+178/-559,0.0,49.41,38.18,29.906,0.42,0.047,0.0,False,CPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,594.0,+118/-414,0.0,52.04,51.78,34.296,777.57,0.312,0.0,False,CPU -53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,579.0,+128/-434,0.0,52.23,51.95,34.339,777.57,0.108,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,396.0,+82/-395,0.0,54.0,53.97,36.314,3.91,0.126,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1732.0,+139/-53,3.175,0.779,7.74,3.31,4822.0,140.81,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1722.0,+271/-134,1.748,0.765,8.05,2.65,2306.87,7.231,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1721.0,+208/-83,2.633,0.707,8.09,4.39,709.58,5.864,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1687.0,+107/-42,3.649,0.716,9.22,4.77,4822.0,22.324,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1664.0,+125/-50,3.591,0.658,10.05,5.86,4558.61,33.853,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1591.0,+114/-48,4.775,0.584,12.98,5.02,48.13,42.001,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1585.0,+224/-129,2.677,0.615,13.26,4.62,2306.87,0.902,✔️,0.0,False,GPU +7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1576.0,+131/-73,5.544,0.476,13.68,8.49,5805.42,4.31,✔️,0.0,False,CPU +8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1566.0,+119/-72,4.535,0.476,14.1,8.34,4558.61,2.234,✔️,0.0,False,GPU +9,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1525.0,+209/-141,4.547,0.537,16.05,8.57,7.67,0.547,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1514.0,+258/-178,4.137,0.525,16.57,6.01,6559.81,56.314,✔️,12.5,True,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1498.0,+128/-64,6.384,0.346,17.42,11.6,4430.73,1.103,✔️,0.0,False,CPU +12,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1476.0,+120/-73,7.235,0.319,18.5,14.95,1203.24,7.106,✔️,0.0,False,CPU +13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1476.0,+125/-64,6.518,0.31,18.52,13.37,4430.73,0.179,✔️,0.0,False,CPU +14,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1470.0,+177/-92,7.628,0.358,18.8,5.41,8977.36,8.017,✔️,0.0,False,GPU +15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1461.0,+187/-96,5.197,0.342,19.27,13.36,4438.37,1.981,✔️,0.0,False,GPU +16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1455.0,+118/-86,6.686,0.294,19.59,11.82,2039.04,2.144,✔️,0.0,False,GPU +17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1425.0,+95/-43,7.544,0.241,21.2,18.07,1265.82,3.15,✔️,0.0,False,CPU +18,🧠⚡,Foundation Model,[Mitra (default) [12.50% IMPUTED]](https://arxiv.org/abs/2510.21204),1423.0,+172/-124,6.857,0.315,21.3,6.84,93.39,2.396,✔️,12.5,True,GPU +19,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1419.0,+279/-232,18.567,0.478,21.52,6.2,6.44,0.709,➖,0.0,False,GPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1401.0,+106/-79,8.008,0.221,22.5,17.92,1203.24,0.956,✔️,0.0,False,CPU +21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1399.0,+88/-44,7.708,0.204,22.65,18.94,1265.82,0.476,✔️,0.0,False,CPU +22,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1384.0,+125/-75,7.186,0.215,23.47,17.87,2039.04,0.109,✔️,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1380.0,+255/-186,5.862,0.342,23.69,10.92,6559.81,0.485,✔️,12.5,True,GPU +24,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1369.0,+174/-114,6.224,0.237,24.27,14.61,4438.37,0.231,✔️,0.0,False,GPU +25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1361.0,+141/-86,8.658,0.192,24.72,17.85,12.31,0.204,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1339.0,+167/-97,9.748,0.155,25.96,13.44,8977.36,0.421,✔️,0.0,False,GPU +27,🧠⚡,Foundation Model,[TabPFNv2 (default) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1308.0,+225/-226,8.136,0.248,27.73,15.18,8.74,0.773,✔️,12.5,True,GPU +28,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1304.0,+171/-124,7.848,0.183,27.94,20.59,22.14,0.179,✔️,0.0,False,GPU +29,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1298.0,+135/-104,8.326,0.133,28.32,22.03,21.51,8.356,✔️,0.0,False,GPU +30,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1271.0,+153/-168,10.01,0.141,29.79,17.71,5150.31,1.592,✔️,0.0,False,CPU +31,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1270.0,+110/-149,12.545,0.149,29.84,19.99,664.2,1.363,✔️,0.0,False,CPU +32,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1240.0,+109/-141,12.829,0.118,31.48,18.81,664.2,0.274,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1233.0,+75/-77,11.52,0.018,31.88,24.2,26.36,0.302,✔️,0.0,False,GPU +34,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1214.0,+152/-154,10.679,0.099,32.86,21.99,5150.31,0.136,✔️,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1213.0,+175/-164,12.343,0.079,32.94,22.94,4333.58,0.422,✔️,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1208.0,+59/-44,10.665,0.0,33.18,31.5,4.3,0.318,✔️,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1185.0,+95/-138,13.964,0.053,34.38,30.6,675.17,1.541,✔️,0.0,False,CPU +38,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1170.0,+178/-174,12.86,0.065,35.15,23.99,4333.58,0.037,✔️,0.0,False,CPU +39,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1143.0,+50/-59,11.888,0.0,36.48,35.38,4.06,0.299,✔️,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1141.0,+88/-140,14.568,0.037,36.59,32.81,675.17,0.206,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1136.0,+107/-134,13.467,0.036,36.85,33.44,2214.9,7.795,✔️,0.0,False,CPU +42,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1112.0,+152/-226,14.305,0.048,37.98,26.64,10.2,2.718,✔️,0.0,False,GPU +43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1110.0,+96/-178,14.916,0.001,38.08,33.03,0.5,0.116,✔️,0.0,False,CPU +44,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1094.0,+148/-172,13.954,0.001,38.8,31.92,16.01,0.041,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1077.0,+103/-140,13.971,0.02,39.57,37.02,2214.9,0.838,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1049.0,+116/-145,13.739,0.0,40.74,38.28,22.39,0.134,✔️,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+87/-150,16.312,0.0,42.69,40.63,0.63,0.12,✔️,0.0,False,CPU +48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),959.0,+106/-170,18.644,0.0,44.17,42.75,6.97,0.818,✔️,0.0,False,CPU +49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),954.0,+157/-376,20.561,0.0,44.34,35.99,88.43,0.512,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),853.0,+158/-384,22.9,0.0,47.34,41.05,88.43,0.051,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),762.0,+178/-559,29.906,0.0,49.41,38.18,0.42,0.047,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),594.0,+118/-414,34.296,0.0,52.04,51.78,777.57,0.312,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),579.0,+128/-434,34.339,0.0,52.23,51.95,777.57,0.108,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),396.0,+82/-395,36.314,0.0,54.0,53.97,3.91,0.126,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_small/winrate_matrix.png.zip index f0c12b2f18ab5d04e0c9f80c4da0501d4ff062db..c1c495f1598d606e171e60deaa945acc87b0838b 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4113a666737bc59c010716a87d06d940d8bac923ac6b19e09cab8f01790dbe12 +oid sha256:5e931d4fe75c863b57de71fc1f359d7180afd7cdf73dae4c32e09cfcce06a3ca size 2146599 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 0b503294162f6e968e4eadb689c44d40cb813bca..e67b660ce9cf22f099ce685707d1676289a92411 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ed5c282d4440e740116db018c0e6ce515fc77c3f78479e08e5d802138655a14e +oid sha256:056f72436b3a73b9395b9b2829f949ad0cd9b6deef88bcd3986cc39a2af6fc8b size 453334 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip index 326387a2d746df0c301be8f746f7d975094de781..7a72c2f4ba8d52b97bfe90c17badb6036b752414 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c66c1389ce1248421f63c3339fd7264e2321c98a881cff87ef915b770df74faa -size 1046899 +oid sha256:0ca42b7b7f8ec97378c7127070aaff875caa06928d35dd0ab869b99eac501266 +size 1019357 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip index a9da4f243560b39a29a88b8ce770d54fda14aa34..7c25e1cd2b77e54422cfa6905c3df2e2abc8502c 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:82f091b46d1bd9488958a1d1f4ce0e034e01a582431cce998d54a1e02dbdf23d +oid sha256:bae81847e8d874f7724ee14d97aea388bece2409711ab183af81de997c4385f7 size 119094 diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/website_leaderboard.csv index 90377fef65b435ce248793b94ebd2127cb72e3d0..0e833c45528f5d3f913975bf8a03140d92d16394 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/website_leaderboard.csv @@ -1,56 +1,56 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1738.0,+227/-92,0.724,8.07,4.14,2.676,669.92,5.859,0.0,False,GPU -1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1729.0,+168/-38,0.767,8.36,3.52,3.492,4663.16,132.911,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1723.0,+332/-141,0.764,8.56,2.6,1.797,2166.18,6.341,0.0,False,GPU -3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1699.0,+128/-34,0.721,9.38,4.73,3.903,4663.16,18.295,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1682.0,+131/-57,0.676,10.01,5.56,3.736,4823.88,33.539,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1626.0,+176/-56,0.627,12.17,4.57,4.903,47.35,39.185,0.0,False,GPU -6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1607.0,+195/-116,0.601,13.01,5.36,3.411,6577.99,56.795,0.0,False,GPU -7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1592.0,+287/-164,0.619,13.66,4.39,2.725,2166.18,0.812,0.0,False,GPU -8,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1568.0,+116/-54,0.451,14.73,9.53,6.052,6530.9,4.363,0.0,False,CPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1565.0,+155/-64,0.462,14.89,8.5,4.84,4823.88,1.952,0.0,False,GPU -10,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1557.0,+236/-169,0.583,15.25,7.94,4.606,7.15,0.501,0.0,False,GPU -11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1540.0,+116/-42,0.392,16.08,10.73,6.578,3558.14,0.966,0.0,False,CPU -12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1520.0,+129/-52,0.354,17.06,12.43,6.697,3558.14,0.105,0.0,False,CPU -13,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1503.0,+250/-242,0.546,17.88,5.52,19.511,5.94,0.652,0.0,False,GPU -14,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1492.0,+191/-91,0.36,18.42,6.11,6.52,99.14,2.48,0.0,False,GPU -15,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1476.0,+131/-107,0.309,19.23,11.16,7.027,2045.39,1.388,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1475.0,+112/-68,0.303,19.3,15.45,7.822,1003.11,4.891,0.0,False,CPU -17,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1459.0,+217/-133,0.328,20.15,13.53,5.499,4158.29,1.946,0.0,False,GPU -18,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1440.0,+231/-181,0.391,21.16,9.88,5.382,6577.99,0.498,0.0,False,GPU -19,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1437.0,+138/-67,0.266,21.28,11.17,8.679,9308.22,7.885,0.0,False,GPU -20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1428.0,+88/-37,0.229,21.77,18.45,8.1,933.42,2.759,0.0,False,CPU -21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1412.0,+127/-98,0.246,22.61,16.94,7.406,2045.39,0.104,0.0,False,GPU -22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1401.0,+91/-46,0.192,23.23,19.21,8.265,933.42,0.388,0.0,False,CPU -23,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1399.0,+114/-70,0.201,23.35,18.34,8.659,1003.11,0.894,0.0,False,CPU -24,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1385.0,+140/-87,0.219,24.07,16.92,9.129,10.91,0.197,0.0,False,CPU -25,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1362.0,+218/-156,0.219,25.36,14.47,6.622,4158.29,0.208,0.0,False,GPU -26,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1354.0,+232/-206,0.284,25.77,13.92,7.982,9.1,0.85,0.0,False,GPU -27,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1299.0,+96/-60,0.073,28.81,20.91,10.902,9308.22,0.416,0.0,False,GPU -28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1293.0,+194/-163,0.171,29.15,18.85,13.262,623.85,1.144,0.0,False,CPU -29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1292.0,+215/-168,0.168,29.2,20.9,8.427,15.01,0.154,0.0,False,GPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1287.0,+170/-106,0.118,29.44,22.47,8.943,23.19,8.523,0.0,False,GPU -31,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1272.0,+170/-162,0.135,30.3,17.5,13.412,623.85,0.291,0.0,False,CPU -32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1250.0,+149/-168,0.091,31.47,21.51,12.707,2929.85,0.41,0.0,False,CPU -33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1228.0,+90/-90,0.015,32.61,24.17,12.445,27.82,0.299,0.0,False,GPU -34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1219.0,+163/-152,0.07,33.04,27.68,11.15,5692.02,1.354,0.0,False,CPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1204.0,+166/-180,0.074,33.82,22.52,13.286,2929.85,0.037,0.0,False,CPU -36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1202.0,+47/-44,0.0,33.94,32.18,11.45,4.57,0.275,0.0,False,CPU -37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1198.0,+126/-178,0.06,34.14,29.95,14.807,633.57,1.512,0.0,False,CPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1187.0,+188/-175,0.077,34.7,22.5,11.641,5692.02,0.131,0.0,False,CPU -39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1161.0,+120/-186,0.042,35.98,31.92,15.352,633.57,0.248,0.0,False,CPU -40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1142.0,+154/-178,0.055,36.87,25.15,14.894,10.27,3.409,0.0,False,GPU -41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1142.0,+46/-60,0.0,36.87,35.63,12.598,3.79,0.311,0.0,False,CPU -42,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1131.0,+130/-155,0.001,37.4,30.44,14.18,11.19,0.038,0.0,False,CPU -43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1128.0,+134/-196,0.002,37.53,32.07,15.73,0.47,0.117,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1085.0,+62/-94,0.0,39.46,38.31,14.849,2248.07,7.53,0.0,False,CPU -45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1025.0,+68/-114,0.0,41.94,41.02,15.336,2248.07,0.89,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1013.0,+116/-188,0.0,42.38,40.15,14.924,24.31,0.134,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+109/-207,0.0,42.87,40.52,17.325,0.53,0.118,0.0,False,CPU -48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,982.0,+155/-419,0.0,43.54,34.61,21.384,84.3,0.438,0.0,False,CPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,902.0,+53/-155,0.0,46.1,45.46,20.488,7.09,0.843,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,877.0,+150/-428,0.0,46.82,39.94,23.664,84.3,0.05,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,801.0,+167/-577,0.0,48.7,36.63,30.315,0.39,0.044,0.0,False,CPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,607.0,+111/-430,0.0,52.0,51.7,36.055,779.18,0.247,0.0,False,CPU -53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,594.0,+126/-470,0.0,52.17,51.85,36.078,779.18,0.089,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,421.0,+89/-422,0.0,53.9,53.88,37.463,3.97,0.122,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1738.0,+227/-92,2.676,0.724,8.07,4.14,669.92,5.859,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1729.0,+168/-38,3.492,0.767,8.36,3.52,4663.16,132.911,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1723.0,+332/-141,1.797,0.764,8.56,2.6,2166.18,6.341,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1699.0,+128/-34,3.903,0.721,9.38,4.73,4663.16,18.295,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1682.0,+131/-57,3.736,0.676,10.01,5.56,4823.88,33.539,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1626.0,+176/-56,4.903,0.627,12.17,4.57,47.35,39.185,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1607.0,+195/-116,3.411,0.601,13.01,5.36,6577.99,56.795,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1592.0,+287/-164,2.725,0.619,13.66,4.39,2166.18,0.812,✔️,0.0,False,GPU +8,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1568.0,+116/-54,6.052,0.451,14.73,9.53,6530.9,4.363,✔️,0.0,False,CPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1565.0,+155/-64,4.84,0.462,14.89,8.5,4823.88,1.952,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1557.0,+236/-169,4.606,0.583,15.25,7.94,7.15,0.501,✔️,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1540.0,+116/-42,6.578,0.392,16.08,10.73,3558.14,0.966,✔️,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1520.0,+129/-52,6.697,0.354,17.06,12.43,3558.14,0.105,✔️,0.0,False,CPU +13,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1503.0,+250/-242,19.511,0.546,17.88,5.52,5.94,0.652,➖,0.0,False,GPU +14,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1492.0,+191/-91,6.52,0.36,18.42,6.11,99.14,2.48,✔️,0.0,False,GPU +15,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1476.0,+131/-107,7.027,0.309,19.23,11.16,2045.39,1.388,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1475.0,+112/-68,7.822,0.303,19.3,15.45,1003.11,4.891,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1459.0,+217/-133,5.499,0.328,20.15,13.53,4158.29,1.946,✔️,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1440.0,+231/-181,5.382,0.391,21.16,9.88,6577.99,0.498,✔️,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1437.0,+138/-67,8.679,0.266,21.28,11.17,9308.22,7.885,✔️,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1428.0,+88/-37,8.1,0.229,21.77,18.45,933.42,2.759,✔️,0.0,False,CPU +21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1412.0,+127/-98,7.406,0.246,22.61,16.94,2045.39,0.104,✔️,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1401.0,+91/-46,8.265,0.192,23.23,19.21,933.42,0.388,✔️,0.0,False,CPU +23,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1399.0,+114/-70,8.659,0.201,23.35,18.34,1003.11,0.894,✔️,0.0,False,CPU +24,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1385.0,+140/-87,9.129,0.219,24.07,16.92,10.91,0.197,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1362.0,+218/-156,6.622,0.219,25.36,14.47,4158.29,0.208,✔️,0.0,False,GPU +26,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1354.0,+232/-206,7.982,0.284,25.77,13.92,9.1,0.85,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1299.0,+96/-60,10.902,0.073,28.81,20.91,9308.22,0.416,✔️,0.0,False,GPU +28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1293.0,+194/-163,13.262,0.171,29.15,18.85,623.85,1.144,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1292.0,+215/-168,8.427,0.168,29.2,20.9,15.01,0.154,✔️,0.0,False,GPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1287.0,+170/-106,8.943,0.118,29.44,22.47,23.19,8.523,✔️,0.0,False,GPU +31,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1272.0,+170/-162,13.412,0.135,30.3,17.5,623.85,0.291,✔️,0.0,False,CPU +32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1250.0,+149/-168,12.707,0.091,31.47,21.51,2929.85,0.41,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1228.0,+90/-90,12.445,0.015,32.61,24.17,27.82,0.299,✔️,0.0,False,GPU +34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1219.0,+163/-152,11.15,0.07,33.04,27.68,5692.02,1.354,✔️,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1204.0,+166/-180,13.286,0.074,33.82,22.52,2929.85,0.037,✔️,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1202.0,+47/-44,11.45,0.0,33.94,32.18,4.57,0.275,✔️,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1198.0,+126/-178,14.807,0.06,34.14,29.95,633.57,1.512,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1187.0,+188/-175,11.641,0.077,34.7,22.5,5692.02,0.131,✔️,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1161.0,+120/-186,15.352,0.042,35.98,31.92,633.57,0.248,✔️,0.0,False,CPU +40,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1142.0,+154/-178,14.894,0.055,36.87,25.15,10.27,3.409,✔️,0.0,False,GPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1142.0,+46/-60,12.598,0.0,36.87,35.63,3.79,0.311,✔️,0.0,False,CPU +42,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1131.0,+130/-155,14.18,0.001,37.4,30.44,11.19,0.038,✔️,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1128.0,+134/-196,15.73,0.002,37.53,32.07,0.47,0.117,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1085.0,+62/-94,14.849,0.0,39.46,38.31,2248.07,7.53,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1025.0,+68/-114,15.336,0.0,41.94,41.02,2248.07,0.89,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1013.0,+116/-188,14.924,0.0,42.38,40.15,24.31,0.134,✔️,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+109/-207,17.325,0.0,42.87,40.52,0.53,0.118,✔️,0.0,False,CPU +48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),982.0,+155/-419,21.384,0.0,43.54,34.61,84.3,0.438,✔️,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),902.0,+53/-155,20.488,0.0,46.1,45.46,7.09,0.843,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),877.0,+150/-428,23.664,0.0,46.82,39.94,84.3,0.05,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),801.0,+167/-577,30.315,0.0,48.7,36.63,0.39,0.044,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),607.0,+111/-430,36.055,0.0,52.0,51.7,779.18,0.247,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),594.0,+126/-470,36.078,0.0,52.17,51.85,779.18,0.089,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),421.0,+89/-422,37.463,0.0,53.9,53.88,3.97,0.122,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip index 0abb8e8d5d5b845c6382929c4fcfab4e55b7a5e7..f7a171a32be4692d56904c1ed060e1cacf197bb5 100644 --- a/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_all/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f585c63f5038a44eb83c2e5bbeb08389834c9dcc9d9404b58c0b4dec97477850 +oid sha256:81bebdd1a18ec60bb8b20509e0cead272022d5378d9b9e0939df7bae4104e9d2 size 2122484 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index bd929c4237d8f88f0f02daeee088af5596918860..3bcb78758a246a4eea9f9e134d9907232877699d 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1939f7264edc2ccde633fc0b7dac4b139e258e5e0212942f3eb65c9320c4004d +oid sha256:6ae087668d285948e91705c8f30f2d1f99db8a486ee593da20b787f26be1b8aa size 476815 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_all/pareto_n_configs_imp.png.zip index 9a1544fc424271f65b3e591ef1040911973a7e11..fd0f9973e548a7dea8e1b91455614589bc9ef600 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fcf74642b27f45732f55b52cca3d37769d7a620befd9da9c4b3a44e8276f36dd -size 1088958 +oid sha256:214b2abeaa8f6279a86d4b2088d37cbd84b376fe8555c0c0921a6806acd78e3e +size 1070652 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_all/tuning-impact-elo.png.zip index 663fdef44566d6beca448604a940675cf0398566..c0a9ccfd00d322d632c269a4109cd50feae45984 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:57ac7325b5752691a829f383e68eb3e8efe192db40ade299ae333749744dd213 +oid sha256:229a8f768f80a5b65881b401e85db03d6af5afd1229eccc8430009d0c042cc92 size 165567 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_all/datasets_all/website_leaderboard.csv index e2fc4c5d22965e135b7adca5f81003dd453b1905..d2e2c4a37ed3786c6e8ca2b43bb8b3214136ae63 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_all/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_all/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1628.0,+99/-65,0.679,8.55,2.79,3.917,2059.94,9.785,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1604.0,+80/-74,0.651,9.45,4.05,6.026,556.15,6.31,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1576.0,+84/-62,0.605,10.57,4.51,6.034,2059.94,1.03,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1543.0,+67/-62,0.556,12.0,4.98,7.8,1754.94,1.767,0.0,False,CPU -4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1539.0,+70/-51,0.559,12.17,6.36,6.68,5.71,0.611,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1507.0,+58/-50,0.496,13.71,8.01,8.918,2791.97,13.886,0.0,False,GPU -6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1436.0,+68/-58,0.449,17.55,5.1,9.317,6154.73,386.167,0.0,False,GPU -7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1432.0,+54/-55,0.401,17.77,9.48,10.444,2791.97,0.373,0.0,False,GPU -8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1411.0,+66/-44,0.365,19.02,9.32,10.905,3133.91,1.273,0.0,False,GPU -9,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1410.0,+39/-40,0.317,19.03,14.04,11.851,416.56,2.236,0.0,False,CPU -10,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1391.0,+75/-59,0.39,20.23,6.99,10.959,6154.73,39.452,0.0,False,GPU -11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1390.0,+58/-44,0.32,20.26,12.91,11.444,1665.53,0.559,0.0,False,CPU -12,🧠🔁,Neural Network,[LimiX (default) [11.76% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1389.0,+73/-61,0.396,20.35,6.09,12.972,3.88,0.594,11.76,True,GPU -13,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1383.0,+82/-60,0.378,20.73,8.09,11.741,4618.5,7.737,0.0,False,GPU -14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1366.0,+52/-46,0.296,21.79,13.57,11.867,1665.53,0.065,0.0,False,CPU -15,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1363.0,+39/-50,0.272,21.93,14.17,12.481,700.96,1.439,0.0,False,CPU -16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1350.0,+51/-47,0.258,22.76,17.23,12.755,416.56,0.381,0.0,False,CPU -17,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1347.0,+60/-58,0.298,22.96,11.99,12.351,4618.5,0.47,0.0,False,GPU -18,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1346.0,+62/-53,0.289,23.03,13.65,11.839,3133.91,0.13,0.0,False,GPU -19,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1344.0,+43/-45,0.256,23.18,13.07,12.455,6.7,0.088,0.0,False,CPU -20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1342.0,+45/-46,0.251,23.29,13.19,12.662,700.96,0.213,0.0,False,CPU -21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1340.0,+60/-46,0.279,23.39,14.52,12.633,866.11,2.007,0.0,False,GPU -22,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1334.0,+75/-72,0.334,23.77,7.85,12.812,2942.08,17.372,35.29,True,GPU -23,🧠🔁,Neural Network,[Mitra (default) [35.29% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1310.0,+56/-67,0.281,25.35,10.45,13.616,87.34,2.433,35.29,True,GPU -24,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1291.0,+56/-45,0.202,26.64,14.41,14.011,866.11,0.097,0.0,False,GPU -25,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1288.0,+71/-72,0.284,26.81,9.54,13.65,45.42,39.406,0.0,False,GPU -26,🧠⚡,Foundation Model,[TabICL (default) [29.41% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1284.0,+60/-55,0.248,27.1,9.5,13.464,6.86,1.52,29.41,True,GPU -27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1283.0,+51/-50,0.221,27.14,18.82,14.241,11.56,0.127,0.0,False,GPU -28,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1268.0,+70/-58,0.231,28.09,14.28,14.474,2942.08,0.262,35.29,True,GPU -29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1266.0,+51/-55,0.186,28.25,17.26,15.201,2961.52,0.482,0.0,False,CPU -30,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1257.0,+46/-56,0.151,28.84,22.85,13.981,2832.8,1.801,0.0,False,CPU -31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1256.0,+43/-48,0.136,28.9,20.14,14.615,10.44,1.714,0.0,False,GPU -32,🧠⚡,Foundation Model,[BetaTabPFN (default) [25.49% IMPUTED]](https://arxiv.org/abs/2502.02527),➖,1243.0,+68/-56,0.191,29.75,17.81,15.841,202.99,1.155,25.49,True,GPU -33,🧠⚡,Foundation Model,[TabPFNv2 (default) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1232.0,+67/-79,0.21,30.48,12.04,15.328,3.27,0.315,35.29,True,GPU -34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1218.0,+62/-50,0.132,31.4,16.55,16.847,13.74,0.316,0.0,False,GPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1217.0,+60/-59,0.141,31.47,16.17,16.032,2961.52,0.048,0.0,False,CPU -36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1205.0,+56/-60,0.116,32.22,23.68,17.265,191.44,0.76,0.0,False,CPU -37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1193.0,+55/-58,0.124,33.02,17.8,16.984,7.66,0.046,0.0,False,CPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1189.0,+48/-60,0.117,33.25,25.96,15.841,2832.8,0.112,0.0,False,CPU -39,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1186.0,+56/-52,0.114,33.47,22.61,15.887,2.06,0.122,0.0,False,CPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1176.0,+63/-66,0.11,34.11,24.29,17.624,594.95,4.651,0.0,False,CPU -41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1172.0,+64/-62,0.117,34.32,20.6,18.313,191.44,0.101,0.0,False,CPU -42,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1166.0,+56/-62,0.085,34.75,26.14,18.204,377.08,0.747,0.0,False,CPU -43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1154.0,+51/-47,0.081,35.44,30.91,16.682,2.2,0.171,0.0,False,CPU -44,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1122.0,+47/-47,0.049,37.4,31.35,19.017,377.08,0.091,0.0,False,CPU -45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1103.0,+59/-77,0.07,38.51,27.59,19.244,594.95,0.337,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1032.0,+55/-68,0.022,42.41,37.99,21.088,8.96,0.129,0.0,False,CPU -47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1032.0,+71/-72,0.057,42.43,27.72,23.862,3.14,0.741,0.0,False,GPU -48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+61/-56,0.014,44.01,36.11,23.881,0.43,0.053,0.0,False,CPU -49,🧠⚡,Foundation Model,[TabFlex (default) [25.49% IMPUTED]](https://arxiv.org/abs/2506.05584),➖,979.0,+77/-88,0.035,45.0,37.8,26.116,0.8,0.119,25.49,True,GPU -50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,970.0,+73/-77,0.019,45.39,41.48,25.472,0.26,0.054,0.0,False,CPU -51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,969.0,+63/-82,0.024,45.47,40.12,25.723,129.1,1.627,0.0,False,CPU -52,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,966.0,+67/-78,0.022,45.57,42.03,23.185,3.12,0.312,0.0,False,CPU -53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,898.0,+81/-109,0.024,48.35,24.73,32.354,240.73,0.308,0.0,False,CPU -54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,868.0,+82/-109,0.017,49.46,33.4,32.94,240.73,0.068,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,805.0,+91/-127,0.008,51.41,48.35,35.498,1.23,0.115,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,804.0,+69/-92,0.011,51.43,49.38,31.855,129.1,0.103,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,589.0,+81/-131,0.0,55.84,55.51,44.446,0.19,0.037,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1628.0,+99/-65,3.917,0.679,8.55,2.79,2059.94,9.785,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1604.0,+80/-74,6.026,0.651,9.45,4.05,556.15,6.31,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1576.0,+84/-62,6.034,0.605,10.57,4.51,2059.94,1.03,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1543.0,+67/-62,7.8,0.556,12.0,4.98,1754.94,1.767,✔️,0.0,False,CPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1539.0,+70/-51,6.68,0.559,12.17,6.36,5.71,0.611,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1507.0,+58/-50,8.918,0.496,13.71,8.01,2791.97,13.886,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1436.0,+68/-58,9.317,0.449,17.55,5.1,6154.73,386.167,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1432.0,+54/-55,10.444,0.401,17.77,9.48,2791.97,0.373,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1411.0,+66/-44,10.905,0.365,19.02,9.32,3133.91,1.273,✔️,0.0,False,GPU +9,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1410.0,+39/-40,11.851,0.317,19.03,14.04,416.56,2.236,✔️,0.0,False,CPU +10,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1391.0,+75/-59,10.959,0.39,20.23,6.99,6154.73,39.452,✔️,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1390.0,+58/-44,11.444,0.32,20.26,12.91,1665.53,0.559,✔️,0.0,False,CPU +12,🧠⚡,Foundation Model,[LimiX (default) [11.76% IMPUTED]](https://arxiv.org/abs/2509.03505),1389.0,+73/-61,12.972,0.396,20.35,6.09,3.88,0.594,➖,11.76,True,GPU +13,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1383.0,+82/-60,11.741,0.378,20.73,8.09,4618.5,7.737,✔️,0.0,False,GPU +14,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1366.0,+52/-46,11.867,0.296,21.79,13.57,1665.53,0.065,✔️,0.0,False,CPU +15,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1363.0,+39/-50,12.481,0.272,21.93,14.17,700.96,1.439,✔️,0.0,False,CPU +16,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1350.0,+51/-47,12.755,0.258,22.76,17.23,416.56,0.381,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1347.0,+60/-58,12.351,0.298,22.96,11.99,4618.5,0.47,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1346.0,+62/-53,11.839,0.289,23.03,13.65,3133.91,0.13,✔️,0.0,False,GPU +19,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1344.0,+43/-45,12.455,0.256,23.18,13.07,6.7,0.088,✔️,0.0,False,CPU +20,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1342.0,+45/-46,12.662,0.251,23.29,13.19,700.96,0.213,✔️,0.0,False,CPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1340.0,+60/-46,12.633,0.279,23.39,14.52,866.11,2.007,✔️,0.0,False,GPU +22,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1334.0,+75/-72,12.812,0.334,23.77,7.85,2942.08,17.372,✔️,35.29,True,GPU +23,🧠⚡,Foundation Model,[Mitra (default) [35.29% IMPUTED]](https://arxiv.org/abs/2510.21204),1310.0,+56/-67,13.616,0.281,25.35,10.45,87.34,2.433,✔️,35.29,True,GPU +24,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1291.0,+56/-45,14.011,0.202,26.64,14.41,866.11,0.097,✔️,0.0,False,GPU +25,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1288.0,+71/-72,13.65,0.284,26.81,9.54,45.42,39.406,✔️,0.0,False,GPU +26,🧠⚡,Foundation Model,[TabICL (default) [29.41% IMPUTED]](https://arxiv.org/abs/2502.05564),1284.0,+60/-55,13.464,0.248,27.1,9.5,6.86,1.52,✔️,29.41,True,GPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1283.0,+51/-50,14.241,0.221,27.14,18.82,11.56,0.127,✔️,0.0,False,GPU +28,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1268.0,+70/-58,14.474,0.231,28.09,14.28,2942.08,0.262,✔️,35.29,True,GPU +29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1266.0,+51/-55,15.201,0.186,28.25,17.26,2961.52,0.482,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1257.0,+46/-56,13.981,0.151,28.84,22.85,2832.8,1.801,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1256.0,+43/-48,14.615,0.136,28.9,20.14,10.44,1.714,✔️,0.0,False,GPU +32,🧠⚡,Foundation Model,[BetaTabPFN (default) [25.49% IMPUTED]](https://arxiv.org/abs/2502.02527),1243.0,+68/-56,15.841,0.191,29.75,17.81,202.99,1.155,➖,25.49,True,GPU +33,🧠⚡,Foundation Model,[TabPFNv2 (default) [35.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1232.0,+67/-79,15.328,0.21,30.48,12.04,3.27,0.315,✔️,35.29,True,GPU +34,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1218.0,+62/-50,16.847,0.132,31.4,16.55,13.74,0.316,✔️,0.0,False,GPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1217.0,+60/-59,16.032,0.141,31.47,16.17,2961.52,0.048,✔️,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1205.0,+56/-60,17.265,0.116,32.22,23.68,191.44,0.76,✔️,0.0,False,CPU +37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1193.0,+55/-58,16.984,0.124,33.02,17.8,7.66,0.046,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1189.0,+48/-60,15.841,0.117,33.25,25.96,2832.8,0.112,✔️,0.0,False,CPU +39,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1186.0,+56/-52,15.887,0.114,33.47,22.61,2.06,0.122,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1176.0,+63/-66,17.624,0.11,34.11,24.29,594.95,4.651,✔️,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1172.0,+64/-62,18.313,0.117,34.32,20.6,191.44,0.101,✔️,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1166.0,+56/-62,18.204,0.085,34.75,26.14,377.08,0.747,✔️,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1154.0,+51/-47,16.682,0.081,35.44,30.91,2.2,0.171,✔️,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1122.0,+47/-47,19.017,0.049,37.4,31.35,377.08,0.091,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1103.0,+59/-77,19.244,0.07,38.51,27.59,594.95,0.337,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1032.0,+55/-68,21.088,0.022,42.41,37.99,8.96,0.129,✔️,0.0,False,CPU +47,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1032.0,+71/-72,23.862,0.057,42.43,27.72,3.14,0.741,✔️,0.0,False,GPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+61/-56,23.881,0.014,44.01,36.11,0.43,0.053,✔️,0.0,False,CPU +49,🧠⚡,Foundation Model,[TabFlex (default) [25.49% IMPUTED]](https://arxiv.org/abs/2506.05584),979.0,+77/-88,26.116,0.035,45.0,37.8,0.8,0.119,➖,25.49,True,GPU +50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),970.0,+73/-77,25.472,0.019,45.39,41.48,0.26,0.054,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),969.0,+63/-82,25.723,0.024,45.47,40.12,129.1,1.627,✔️,0.0,False,CPU +52,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),966.0,+67/-78,23.185,0.022,45.57,42.03,3.12,0.312,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),898.0,+81/-109,32.354,0.024,48.35,24.73,240.73,0.308,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),868.0,+82/-109,32.94,0.017,49.46,33.4,240.73,0.068,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),805.0,+91/-127,35.498,0.008,51.41,48.35,1.23,0.115,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),804.0,+69/-92,31.855,0.011,51.43,49.38,129.1,0.103,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),589.0,+81/-131,44.446,0.0,55.84,55.51,0.19,0.037,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_all/winrate_matrix.png.zip index 3f654e8d14f56cd4f1dceda4215202d682e8a748..cbb8f48b7d273cd9dd59249613a26ebe4fb187f8 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fde2ce642997b850951dc514c346fab73d4295e1221de8e8b15e19cbb08526c9 +oid sha256:d63e8561929c0de79a37f81010df44932784b7cf6cd2a8680e8e27020ca461cf size 2541132 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index 9a66337ef74b7149ae3d9edd615b9018c1911502..9d714972b8fd91a086f46990162723390e35d3c1 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:caa44c98322dc9a7661b7b42a54df9e8c6aca73d02cdc57a6164d25423490879 +oid sha256:0866d69b3f69ef9c110afbf865021cac69fa77dcc9f120dc1abcfa6ab0abddcd size 459061 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip index 8c38275b4c62cd14a13141ca27fdfd12e2a199f3..959a07bc153cf66faac04b59ad8545427a9e7226 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:90b1af57de59828870a8623866abfcbbbd79bdb8f07a4c7d77b804b23ddc3f35 -size 1008317 +oid sha256:643a4b716f5ed89c6a4c50e2390695bef099d7560ad3ceb612ce72a643cc2e65 +size 991803 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/tuning-impact-elo.png.zip index 571e5f03c4d1f0cfdb417749c27fe0104660af4a..dc01388bbdc17e1b83378ca81525b74405d5c321 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:804bea56cec315135fbc9f1ff8ec2b332ead9138db3152430d211b96eaf6859e +oid sha256:4f66de16edb07fa6e75a6a3d8af7d0dc7d2c9eedcfea349fdb818cbfe14fd8bd size 138318 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/website_leaderboard.csv index c022edda34d3522856934c4ced369b3fdc212cf0..24a9dc4877f7820154e4b6de516f9844bfec4707 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/website_leaderboard.csv @@ -1,55 +1,55 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1884.0,+146/-62,0.804,4.67,2.45,2.12,432.14,4.214,0.0,False,CPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1841.0,+140/-68,0.774,5.53,2.88,2.589,302.05,2.868,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1732.0,+175/-116,0.666,8.4,3.18,3.925,739.1,11.65,0.0,False,GPU -3,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1689.0,+167/-130,0.642,9.8,4.58,4.338,2522.58,6.234,0.0,False,GPU -4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1681.0,+158/-108,0.613,10.07,4.61,5.359,739.1,1.134,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1621.0,+99/-84,0.532,12.33,8.51,5.94,1714.29,1.743,0.0,False,GPU -6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1582.0,+118/-80,0.461,13.93,8.65,6.784,763.16,0.246,0.0,False,CPU -7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1582.0,+86/-60,0.436,13.93,10.4,7.733,129.25,2.621,0.0,False,CPU -8,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1555.0,+139/-115,0.441,15.13,11.23,7.879,2522.58,0.318,0.0,False,GPU -9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1554.0,+109/-69,0.436,15.17,8.23,6.913,763.16,0.04,0.0,False,CPU -10,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1549.0,+95/-57,0.393,15.4,10.77,7.467,289.35,0.706,0.0,False,CPU -11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1540.0,+150/-93,0.408,15.8,6.85,7.635,1993.57,0.581,0.0,False,GPU -12,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1529.0,+100/-84,0.364,16.33,12.67,8.359,129.25,0.117,0.0,False,CPU -13,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1522.0,+152/-115,0.415,16.67,8.21,6.533,1.91,0.668,0.0,False,GPU -14,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1514.0,+132/-117,0.404,17.0,10.11,7.967,1714.29,0.079,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1512.0,+109/-84,0.41,17.1,7.62,7.185,2.93,0.045,0.0,False,CPU -16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1495.0,+104/-81,0.355,17.93,13.2,7.701,289.35,0.08,0.0,False,CPU -17,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1458.0,+133/-120,0.353,19.73,5.46,8.104,4532.14,387.927,0.0,False,GPU -18,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1458.0,+144/-86,0.313,19.73,11.0,8.363,1993.57,0.06,0.0,False,GPU -19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1414.0,+112/-107,0.304,21.93,15.24,9.218,559.64,0.88,0.0,False,GPU -20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1394.0,+152/-142,0.303,22.97,8.77,10.694,4532.14,39.452,0.0,False,GPU -21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1373.0,+104/-94,0.204,24.07,18.29,11.567,559.64,0.051,0.0,False,GPU -22,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1372.0,+124/-124,0.241,24.13,13.15,9.557,0.48,0.047,0.0,False,CPU -23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1368.0,+120/-107,0.205,24.33,20.34,10.953,5.42,0.078,0.0,False,GPU -24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1335.0,+83/-66,0.133,26.07,21.96,10.076,1803.23,1.199,0.0,False,CPU -25,🧠⚡,Foundation Model,[TabICL (default) [33.33% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1326.0,+149/-121,0.195,26.53,15.87,10.541,3.98,1.593,33.33,True,GPU -26,🧠🔁,Neural Network,[LimiX (default) [40.00% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1323.0,+159/-146,0.24,26.7,5.9,14.403,1.09,0.318,40.0,True,GPU -27,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1319.0,+134/-147,0.219,26.93,18.11,13.157,1535.55,0.182,0.0,False,CPU -28,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1282.0,+80/-74,0.064,28.87,26.1,12.919,6.54,0.294,0.0,False,GPU -29,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1281.0,+124/-138,0.197,28.9,13.45,13.622,41.5,39.095,0.0,False,GPU -30,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1273.0,+94/-95,0.117,29.33,25.0,11.636,9.07,0.224,0.0,False,GPU -31,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1234.0,+97/-92,0.049,31.33,25.01,12.885,65.73,0.23,0.0,False,CPU -32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1231.0,+82/-95,0.065,31.47,28.31,12.561,1803.23,0.051,0.0,False,CPU -33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1224.0,+95/-118,0.055,31.8,28.99,12.624,36.73,0.359,0.0,False,CPU -34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1218.0,+78/-94,0.044,32.13,30.42,12.051,0.34,0.07,0.0,False,CPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1210.0,+134/-142,0.131,32.53,25.81,14.45,1535.55,0.013,0.0,False,CPU -36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1201.0,+89/-83,0.038,32.97,28.86,13.221,65.73,0.025,0.0,False,CPU -37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1200.0,+111/-142,0.116,33.0,28.81,15.546,2.99,0.016,0.0,False,CPU -38,🧠⚡,Foundation Model,[BetaTabPFN (default) [33.33% IMPUTED]](https://arxiv.org/abs/2502.02527),➖,1181.0,+124/-115,0.078,33.93,24.31,14.165,66.06,0.61,33.33,True,GPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1175.0,+114/-140,0.067,34.2,27.87,15.053,354.91,1.636,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1154.0,+89/-111,0.031,35.2,32.74,15.004,36.73,0.038,0.0,False,CPU -41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1078.0,+85/-91,0.008,38.67,37.11,16.371,2.58,0.039,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1042.0,+91/-111,0.012,40.13,38.37,17.618,354.91,0.094,0.0,False,CPU -43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1014.0,+159/-215,0.022,41.27,35.06,25.313,2.55,0.157,0.0,False,GPU -44,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+65/-106,0.0,41.8,40.8,18.863,0.24,0.022,0.0,False,CPU -45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,969.0,+92/-124,0.012,42.93,40.74,22.593,150.58,4.952,0.0,False,CPU -46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,932.0,+109/-151,0.016,44.2,41.74,21.122,1.49,0.113,0.0,False,CPU -47,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,918.0,+111/-145,0.01,44.67,42.84,22.439,0.13,0.022,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,915.0,+167/-225,0.011,44.77,41.01,32.033,43.96,0.081,0.0,False,CPU -49,🧠⚡,Foundation Model,[TabFlex (default) [33.33% IMPUTED]](https://arxiv.org/abs/2506.05584),➖,874.0,+95/-141,0.0,46.0,45.16,27.553,0.27,0.043,33.33,True,GPU -50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,864.0,+152/-239,0.001,46.3,43.62,32.796,43.96,0.017,0.0,False,CPU -51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,785.0,+59/-123,0.0,48.33,48.08,29.049,150.58,0.546,0.0,False,CPU -52,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,725.0,+107/-185,0.0,49.6,49.2,34.849,0.2,0.02,0.0,False,CPU -53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,536.0,+93/-196,0.0,52.33,52.25,42.696,0.1,0.068,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1884.0,+146/-62,2.12,0.804,4.67,2.45,432.14,4.214,✔️,0.0,False,CPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1841.0,+140/-68,2.589,0.774,5.53,2.88,302.05,2.868,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1732.0,+175/-116,3.925,0.666,8.4,3.18,739.1,11.65,✔️,0.0,False,GPU +3,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1689.0,+167/-130,4.338,0.642,9.8,4.58,2522.58,6.234,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1681.0,+158/-108,5.359,0.613,10.07,4.61,739.1,1.134,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1621.0,+99/-84,5.94,0.532,12.33,8.51,1714.29,1.743,✔️,0.0,False,GPU +6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1582.0,+118/-80,6.784,0.461,13.93,8.65,763.16,0.246,✔️,0.0,False,CPU +7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1582.0,+86/-60,7.733,0.436,13.93,10.4,129.25,2.621,✔️,0.0,False,CPU +8,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1555.0,+139/-115,7.879,0.441,15.13,11.23,2522.58,0.318,✔️,0.0,False,GPU +9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1554.0,+109/-69,6.913,0.436,15.17,8.23,763.16,0.04,✔️,0.0,False,CPU +10,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1549.0,+95/-57,7.467,0.393,15.4,10.77,289.35,0.706,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1540.0,+150/-93,7.635,0.408,15.8,6.85,1993.57,0.581,✔️,0.0,False,GPU +12,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1529.0,+100/-84,8.359,0.364,16.33,12.67,129.25,0.117,✔️,0.0,False,CPU +13,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1522.0,+152/-115,6.533,0.415,16.67,8.21,1.91,0.668,✔️,0.0,False,GPU +14,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1514.0,+132/-117,7.967,0.404,17.0,10.11,1714.29,0.079,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1512.0,+109/-84,7.185,0.41,17.1,7.62,2.93,0.045,✔️,0.0,False,CPU +16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1495.0,+104/-81,7.701,0.355,17.93,13.2,289.35,0.08,✔️,0.0,False,CPU +17,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1458.0,+133/-120,8.104,0.353,19.73,5.46,4532.14,387.927,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1458.0,+144/-86,8.363,0.313,19.73,11.0,1993.57,0.06,✔️,0.0,False,GPU +19,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1414.0,+112/-107,9.218,0.304,21.93,15.24,559.64,0.88,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1394.0,+152/-142,10.694,0.303,22.97,8.77,4532.14,39.452,✔️,0.0,False,GPU +21,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1373.0,+104/-94,11.567,0.204,24.07,18.29,559.64,0.051,✔️,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1372.0,+124/-124,9.557,0.241,24.13,13.15,0.48,0.047,✔️,0.0,False,CPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1368.0,+120/-107,10.953,0.205,24.33,20.34,5.42,0.078,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1335.0,+83/-66,10.076,0.133,26.07,21.96,1803.23,1.199,✔️,0.0,False,CPU +25,🧠⚡,Foundation Model,[TabICL (default) [33.33% IMPUTED]](https://arxiv.org/abs/2502.05564),1326.0,+149/-121,10.541,0.195,26.53,15.87,3.98,1.593,✔️,33.33,True,GPU +26,🧠⚡,Foundation Model,[LimiX (default) [40.00% IMPUTED]](https://arxiv.org/abs/2509.03505),1323.0,+159/-146,14.403,0.24,26.7,5.9,1.09,0.318,➖,40.0,True,GPU +27,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1319.0,+134/-147,13.157,0.219,26.93,18.11,1535.55,0.182,✔️,0.0,False,CPU +28,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1282.0,+80/-74,12.919,0.064,28.87,26.1,6.54,0.294,✔️,0.0,False,GPU +29,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1281.0,+124/-138,13.622,0.197,28.9,13.45,41.5,39.095,✔️,0.0,False,GPU +30,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1273.0,+94/-95,11.636,0.117,29.33,25.0,9.07,0.224,✔️,0.0,False,GPU +31,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1234.0,+97/-92,12.885,0.049,31.33,25.01,65.73,0.23,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1231.0,+82/-95,12.561,0.065,31.47,28.31,1803.23,0.051,✔️,0.0,False,CPU +33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1224.0,+95/-118,12.624,0.055,31.8,28.99,36.73,0.359,✔️,0.0,False,CPU +34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1218.0,+78/-94,12.051,0.044,32.13,30.42,0.34,0.07,✔️,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1210.0,+134/-142,14.45,0.131,32.53,25.81,1535.55,0.013,✔️,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1201.0,+89/-83,13.221,0.038,32.97,28.86,65.73,0.025,✔️,0.0,False,CPU +37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1200.0,+111/-142,15.546,0.116,33.0,28.81,2.99,0.016,✔️,0.0,False,CPU +38,🧠⚡,Foundation Model,[BetaTabPFN (default) [33.33% IMPUTED]](https://arxiv.org/abs/2502.02527),1181.0,+124/-115,14.165,0.078,33.93,24.31,66.06,0.61,➖,33.33,True,GPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1175.0,+114/-140,15.053,0.067,34.2,27.87,354.91,1.636,✔️,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1154.0,+89/-111,15.004,0.031,35.2,32.74,36.73,0.038,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1078.0,+85/-91,16.371,0.008,38.67,37.11,2.58,0.039,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1042.0,+91/-111,17.618,0.012,40.13,38.37,354.91,0.094,✔️,0.0,False,CPU +43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1014.0,+159/-215,25.313,0.022,41.27,35.06,2.55,0.157,✔️,0.0,False,GPU +44,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+65/-106,18.863,0.0,41.8,40.8,0.24,0.022,✔️,0.0,False,CPU +45,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),969.0,+92/-124,22.593,0.012,42.93,40.74,150.58,4.952,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),932.0,+109/-151,21.122,0.016,44.2,41.74,1.49,0.113,✔️,0.0,False,CPU +47,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),918.0,+111/-145,22.439,0.01,44.67,42.84,0.13,0.022,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),915.0,+167/-225,32.033,0.011,44.77,41.01,43.96,0.081,✔️,0.0,False,CPU +49,🧠⚡,Foundation Model,[TabFlex (default) [33.33% IMPUTED]](https://arxiv.org/abs/2506.05584),874.0,+95/-141,27.553,0.0,46.0,45.16,0.27,0.043,➖,33.33,True,GPU +50,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),864.0,+152/-239,32.796,0.001,46.3,43.62,43.96,0.017,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),785.0,+59/-123,29.049,0.0,48.33,48.08,150.58,0.546,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),725.0,+107/-185,34.849,0.0,49.6,49.2,0.2,0.02,✔️,0.0,False,CPU +53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),536.0,+93/-196,42.696,0.0,52.33,52.25,0.1,0.068,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/winrate_matrix.png.zip index c1bd338ff97883925f84f47dd3ecc35db55b746c..28bf5b65594cfe68382863b29121d6364dbb7dcb 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ab0531860fb5a6a1c7ac3f06f669015e9ea18aeab6ac2a97f9c68423fe7097eb +oid sha256:b98f79e24ae6a96b2cd4a1f13ced754a7876d720081264c4d403c1270aa2b66c size 1860124 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 1157b8f972b95965c998ae0b1faeef5cccabc8e8..cf0c659d962313d797ccb3049bdd0a908817eddb 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cad364b8cc2dbab97aa740b921e9e194a41a91d29277db65667d1a4e092d7dac +oid sha256:6c0fd6c09905f93da237c4aa3428f177ea2161db3bc42cee13bf7cfe09b80ab5 size 471941 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_small/pareto_n_configs_imp.png.zip index 8de22d41f9b84bdbedc8046b218d8f7b27403126..d117c16c8105a7f494c16c201a4de383df9c5b40 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c7ec3ccc63dc1e8d0825467df296264478a36290a83298c69bc1aad7ca3a58be -size 1074214 +oid sha256:49d4cff54a4cd572129d5a414784d45f474d93974f5b5ff7abded3a4fa4ccf02 +size 1060558 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_small/tuning-impact-elo.png.zip index fe1d8ad24c4edb55b02aeb077797d7a7d6abea9c..4bc3bf16b98e4e22d11955d48b9c23fa9ca92ea3 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c2aca1dfc405bc95c07620d9719cb02274d59b8e30e2df8f70063ff977e55979 +oid sha256:e587becbfb7263378c73856bf63979e90018b5c56b4010d37fe62ead9339a3e4 size 174670 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_all/datasets_small/website_leaderboard.csv index 9bdb1545545b76158be4b8b45cd4b0056e1090cc..7eda4e21afb47688c8532fac603a5bc92e90d54c 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_small/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_small/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1628.0,+128/-83,0.678,8.61,2.65,3.914,2319.48,8.957,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1587.0,+101/-69,0.598,10.18,5.81,6.742,6.58,0.586,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1572.0,+108/-71,0.593,10.78,4.47,6.315,2319.48,0.942,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1565.0,+80/-67,0.591,11.08,4.88,7.459,657.21,7.989,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1500.0,+69/-57,0.466,14.17,7.81,10.159,3855.22,24.935,0.0,False,GPU -5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1482.0,+57/-52,0.444,15.06,8.72,10.167,3351.09,1.56,0.0,False,CPU -6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1475.0,+111/-81,0.474,15.44,5.86,10.288,3560.23,49.21,8.33,True,GPU -7,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1453.0,+94/-72,0.469,16.64,4.96,9.823,8059.77,364.288,0.0,False,GPU -8,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1440.0,+93/-72,0.454,17.38,6.16,12.376,5.11,0.624,0.0,False,GPU -9,🧠🔁,Neural Network,[Mitra (default) [8.33% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1435.0,+89/-50,0.398,17.67,7.95,11.429,128.8,2.999,8.33,True,GPU -10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1432.0,+71/-72,0.384,17.88,9.22,11.477,3855.22,0.954,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1414.0,+89/-80,0.419,18.92,6.44,11.069,8059.77,39.456,0.0,False,GPU -12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1390.0,+70/-56,0.336,20.36,10.97,12.268,3629.74,1.48,0.0,False,GPU -13,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1377.0,+46/-43,0.251,21.15,16.43,13.566,833.03,2.182,0.0,False,CPU -14,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1371.0,+88/-72,0.327,21.56,11.17,12.643,3560.23,0.519,8.33,True,GPU -15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1350.0,+61/-58,0.244,22.9,16.25,13.386,2448.79,0.814,0.0,False,CPU -16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1334.0,+72/-63,0.259,23.89,14.22,14.055,1346.37,2.149,0.0,False,GPU -17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1328.0,+73/-72,0.265,24.29,15.16,13.287,3629.74,0.189,0.0,False,GPU -18,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1324.0,+63/-55,0.218,24.56,18.59,13.932,2448.79,0.078,0.0,False,CPU -19,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1322.0,+57/-45,0.202,24.65,16.31,14.57,886.51,1.982,0.0,False,CPU -20,🧠⚡,Foundation Model,[TabPFNv2 (default) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1318.0,+79/-65,0.298,24.93,9.27,13.855,4.98,0.511,8.33,True,GPU -21,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1313.0,+67/-71,0.26,25.28,11.87,14.826,6341.76,8.556,0.0,False,GPU -22,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1310.0,+52/-46,0.196,25.44,20.28,14.586,833.03,0.435,0.0,False,CPU -23,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1310.0,+85/-82,0.312,25.5,8.49,13.661,51.78,39.927,0.0,False,GPU -24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1309.0,+60/-52,0.188,25.53,13.18,14.729,886.51,0.318,0.0,False,CPU -25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1306.0,+50/-44,0.172,25.71,18.61,14.65,8.39,0.139,0.0,False,CPU -26,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1298.0,+64/-64,0.224,26.22,12.34,14.214,6341.76,0.501,0.0,False,GPU -27,🧠⚡,Foundation Model,[TabICL (default) [27.78% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1286.0,+92/-100,0.256,27.03,8.13,14.682,7.56,1.494,27.78,True,GPU -28,🧠⚡,Foundation Model,[BetaTabPFN (default) [22.22% IMPUTED]](https://arxiv.org/abs/2502.02527),➖,1281.0,+77/-78,0.225,27.36,15.93,16.539,269.12,1.881,22.22,True,GPU -29,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1279.0,+65/-63,0.191,27.49,13.22,15.03,1346.37,0.103,0.0,False,GPU -30,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1270.0,+66/-61,0.211,28.08,18.21,15.611,13.71,0.149,0.0,False,GPU -31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1268.0,+68/-58,0.162,28.25,16.85,16.053,3249.24,0.548,0.0,False,CPU -32,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1262.0,+57/-60,0.149,28.58,18.33,15.321,16.6,4.299,0.0,False,GPU -33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1243.0,+51/-62,0.149,29.89,23.21,15.608,3693.32,2.14,0.0,False,CPU -34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1234.0,+65/-61,0.136,30.47,13.95,16.691,3249.24,0.059,0.0,False,CPU -35,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1208.0,+72/-71,0.119,32.15,14.49,19.018,14.9,0.323,0.0,False,GPU -36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1206.0,+79/-77,0.135,32.28,21.98,19.199,497.58,1.147,0.0,False,CPU -37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1203.0,+71/-60,0.113,32.47,15.3,17.583,9.93,0.052,0.0,False,CPU -38,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1188.0,+76/-91,0.125,33.4,22.89,18.695,1666.78,7.827,0.0,False,CPU -39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1186.0,+59/-70,0.13,33.56,24.95,17.208,3693.32,0.131,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1185.0,+82/-88,0.149,33.62,17.79,19.691,497.58,0.163,0.0,False,CPU -41,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1148.0,+77/-82,0.098,35.94,26.54,20.42,575.62,1.155,0.0,False,CPU -42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1137.0,+57/-66,0.078,36.6,30.99,18.612,3.63,0.218,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1129.0,+69/-91,0.092,37.06,24.49,19.921,1666.78,0.561,0.0,False,CPU -44,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1126.0,+53/-63,0.047,37.25,32.25,18.524,3.1,0.207,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1097.0,+63/-68,0.052,38.97,32.32,21.432,575.62,0.147,0.0,False,CPU -46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1043.0,+90/-101,0.067,41.92,25.27,23.257,6.09,0.951,0.0,False,GPU -47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1014.0,+64/-97,0.026,43.42,38.0,23.054,12.68,0.155,0.0,False,CPU -48,🧠⚡,Foundation Model,[TabFlex (default) [22.22% IMPUTED]](https://arxiv.org/abs/2506.05584),➖,1010.0,+75/-83,0.05,43.61,34.89,25.516,1.59,0.149,22.22,True,GPU -49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+69/-89,0.02,44.1,33.9,25.971,0.51,0.085,0.0,False,CPU -50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,992.0,+76/-81,0.021,44.47,40.13,26.736,0.43,0.081,0.0,False,CPU -51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,980.0,+58/-83,0.023,45.03,41.34,24.045,5.83,0.571,0.0,False,CPU -52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,967.0,+70/-121,0.029,45.64,39.24,27.027,116.3,1.243,0.0,False,CPU -53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,890.0,+85/-106,0.028,48.74,21.03,32.488,520.62,0.491,0.0,False,CPU -54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,864.0,+93/-116,0.023,49.67,30.07,33.0,520.62,0.097,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,824.0,+102/-147,0.011,50.94,46.99,35.769,2.59,0.129,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,801.0,+79/-133,0.016,51.61,48.91,33.024,116.3,0.075,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,608.0,+97/-193,0.0,55.64,55.2,45.176,0.37,0.035,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1628.0,+128/-83,3.914,0.678,8.61,2.65,2319.48,8.957,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1587.0,+101/-69,6.742,0.598,10.18,5.81,6.58,0.586,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1572.0,+108/-71,6.315,0.593,10.78,4.47,2319.48,0.942,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1565.0,+80/-67,7.459,0.591,11.08,4.88,657.21,7.989,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1500.0,+69/-57,10.159,0.466,14.17,7.81,3855.22,24.935,✔️,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1482.0,+57/-52,10.167,0.444,15.06,8.72,3351.09,1.56,✔️,0.0,False,CPU +6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1475.0,+111/-81,10.288,0.474,15.44,5.86,3560.23,49.21,✔️,8.33,True,GPU +7,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1453.0,+94/-72,9.823,0.469,16.64,4.96,8059.77,364.288,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1440.0,+93/-72,12.376,0.454,17.38,6.16,5.11,0.624,➖,0.0,False,GPU +9,🧠⚡,Foundation Model,[Mitra (default) [8.33% IMPUTED]](https://arxiv.org/abs/2510.21204),1435.0,+89/-50,11.429,0.398,17.67,7.95,128.8,2.999,✔️,8.33,True,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1432.0,+71/-72,11.477,0.384,17.88,9.22,3855.22,0.954,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1414.0,+89/-80,11.069,0.419,18.92,6.44,8059.77,39.456,✔️,0.0,False,GPU +12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1390.0,+70/-56,12.268,0.336,20.36,10.97,3629.74,1.48,✔️,0.0,False,GPU +13,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1377.0,+46/-43,13.566,0.251,21.15,16.43,833.03,2.182,✔️,0.0,False,CPU +14,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1371.0,+88/-72,12.643,0.327,21.56,11.17,3560.23,0.519,✔️,8.33,True,GPU +15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1350.0,+61/-58,13.386,0.244,22.9,16.25,2448.79,0.814,✔️,0.0,False,CPU +16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1334.0,+72/-63,14.055,0.259,23.89,14.22,1346.37,2.149,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1328.0,+73/-72,13.287,0.265,24.29,15.16,3629.74,0.189,✔️,0.0,False,GPU +18,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1324.0,+63/-55,13.932,0.218,24.56,18.59,2448.79,0.078,✔️,0.0,False,CPU +19,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1322.0,+57/-45,14.57,0.202,24.65,16.31,886.51,1.982,✔️,0.0,False,CPU +20,🧠⚡,Foundation Model,[TabPFNv2 (default) [8.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1318.0,+79/-65,13.855,0.298,24.93,9.27,4.98,0.511,✔️,8.33,True,GPU +21,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1313.0,+67/-71,14.826,0.26,25.28,11.87,6341.76,8.556,✔️,0.0,False,GPU +22,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1310.0,+52/-46,14.586,0.196,25.44,20.28,833.03,0.435,✔️,0.0,False,CPU +23,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1310.0,+85/-82,13.661,0.312,25.5,8.49,51.78,39.927,✔️,0.0,False,GPU +24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1309.0,+60/-52,14.729,0.188,25.53,13.18,886.51,0.318,✔️,0.0,False,CPU +25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1306.0,+50/-44,14.65,0.172,25.71,18.61,8.39,0.139,✔️,0.0,False,CPU +26,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1298.0,+64/-64,14.214,0.224,26.22,12.34,6341.76,0.501,✔️,0.0,False,GPU +27,🧠⚡,Foundation Model,[TabICL (default) [27.78% IMPUTED]](https://arxiv.org/abs/2502.05564),1286.0,+92/-100,14.682,0.256,27.03,8.13,7.56,1.494,✔️,27.78,True,GPU +28,🧠⚡,Foundation Model,[BetaTabPFN (default) [22.22% IMPUTED]](https://arxiv.org/abs/2502.02527),1281.0,+77/-78,16.539,0.225,27.36,15.93,269.12,1.881,➖,22.22,True,GPU +29,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1279.0,+65/-63,15.03,0.191,27.49,13.22,1346.37,0.103,✔️,0.0,False,GPU +30,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1270.0,+66/-61,15.611,0.211,28.08,18.21,13.71,0.149,✔️,0.0,False,GPU +31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1268.0,+68/-58,16.053,0.162,28.25,16.85,3249.24,0.548,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1262.0,+57/-60,15.321,0.149,28.58,18.33,16.6,4.299,✔️,0.0,False,GPU +33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1243.0,+51/-62,15.608,0.149,29.89,23.21,3693.32,2.14,✔️,0.0,False,CPU +34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1234.0,+65/-61,16.691,0.136,30.47,13.95,3249.24,0.059,✔️,0.0,False,CPU +35,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1208.0,+72/-71,19.018,0.119,32.15,14.49,14.9,0.323,✔️,0.0,False,GPU +36,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1206.0,+79/-77,19.199,0.135,32.28,21.98,497.58,1.147,✔️,0.0,False,CPU +37,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1203.0,+71/-60,17.583,0.113,32.47,15.3,9.93,0.052,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1188.0,+76/-91,18.695,0.125,33.4,22.89,1666.78,7.827,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1186.0,+59/-70,17.208,0.13,33.56,24.95,3693.32,0.131,✔️,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1185.0,+82/-88,19.691,0.149,33.62,17.79,497.58,0.163,✔️,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1148.0,+77/-82,20.42,0.098,35.94,26.54,575.62,1.155,✔️,0.0,False,CPU +42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1137.0,+57/-66,18.612,0.078,36.6,30.99,3.63,0.218,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1129.0,+69/-91,19.921,0.092,37.06,24.49,1666.78,0.561,✔️,0.0,False,CPU +44,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1126.0,+53/-63,18.524,0.047,37.25,32.25,3.1,0.207,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1097.0,+63/-68,21.432,0.052,38.97,32.32,575.62,0.147,✔️,0.0,False,CPU +46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1043.0,+90/-101,23.257,0.067,41.92,25.27,6.09,0.951,✔️,0.0,False,GPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1014.0,+64/-97,23.054,0.026,43.42,38.0,12.68,0.155,✔️,0.0,False,CPU +48,🧠⚡,Foundation Model,[TabFlex (default) [22.22% IMPUTED]](https://arxiv.org/abs/2506.05584),1010.0,+75/-83,25.516,0.05,43.61,34.89,1.59,0.149,➖,22.22,True,GPU +49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+69/-89,25.971,0.02,44.1,33.9,0.51,0.085,✔️,0.0,False,CPU +50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),992.0,+76/-81,26.736,0.021,44.47,40.13,0.43,0.081,✔️,0.0,False,CPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),980.0,+58/-83,24.045,0.023,45.03,41.34,5.83,0.571,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),967.0,+70/-121,27.027,0.029,45.64,39.24,116.3,1.243,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),890.0,+85/-106,32.488,0.028,48.74,21.03,520.62,0.491,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),864.0,+93/-116,33.0,0.023,49.67,30.07,520.62,0.097,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),824.0,+102/-147,35.769,0.011,50.94,46.99,2.59,0.129,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),801.0,+79/-133,33.024,0.016,51.61,48.91,116.3,0.075,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),608.0,+97/-193,45.176,0.0,55.64,55.2,0.37,0.035,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_small/winrate_matrix.png.zip index 107bfd04a4050b35b62c92c5e5c5a373d1eb5a28..652c1c7a59d32a4c2a8d0f2904cb5b65739b8497 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:718f6ffef5b3775c081b2649fd75a44f2ffc768cdc1029eec7298e4e7994c234 +oid sha256:faf8469be491dbfcf7401953d736c2ed1f5c39409c827b4a0c0acb6ce5cea3c2 size 2471898 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 54f7e40b419baa0a0cb5a59840bca37c1401cd56..806651db9e2856d756a75753d6105442019ef39d 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:541522fa0fecd41069da4c83a3aedb37c7f9c698b779d4934605451bde66e4ef +oid sha256:07f2ec8dd72f1bc6ae1c305cd03443745fa47a41e5f90085bc3219dbf5e65302 size 470545 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip index 5510e2572425d91e3a7eb9413f52bed9cb9b66b0..4ba4631c7d153c4ef61ed8815868a0ec97102e5e 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:61314dd30fcb4829f024fe83747e5f8b0d6b71c496e9f9825cc2d8715d5e4186 -size 1076521 +oid sha256:c4e27df66c0e772c355e2cde58b8fff8cacd141378fa055e706b6ab6dc57a12a +size 1068077 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip index 7a307c956acb2af1fb95bb3008d223bc498ceab0..4fd48b754dd3a016457f8b2831143b20dcc42d60 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fa0202a88554a0103f2ca0b92184cdb9c6bf98a9adc4737190f8bae625dd0b24 +oid sha256:410bba1c7c70862639a6df85afbf506e0b97ec1dd9339cf823c136e5ece3272a size 143442 diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/website_leaderboard.csv index 11a95648b532df4e405f1545d941e79afd4e1834..8d61b31c9252293053cf10b3133c17b4e3bcc9f6 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1650.0,+124/-84,0.683,8.42,2.71,4.175,2200.35,8.314,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1630.0,+84/-70,0.622,9.14,5.48,7.065,6.21,0.561,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1598.0,+113/-87,0.6,10.36,4.25,6.692,2200.35,0.882,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1570.0,+96/-80,0.581,11.52,4.86,7.982,625.63,7.662,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1536.0,+102/-76,0.517,13.08,5.44,9.632,3650.56,52.333,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1533.0,+74/-57,0.488,13.21,7.36,10.676,3565.03,23.942,0.0,False,GPU -6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1488.0,+62/-51,0.426,15.52,8.68,10.911,3035.75,1.498,0.0,False,CPU -7,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1486.0,+86/-68,0.434,15.59,7.42,10.885,137.76,3.046,0.0,False,GPU -8,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1469.0,+92/-79,0.461,16.5,5.8,13.013,4.66,0.611,0.0,False,GPU -9,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1462.0,+96/-75,0.458,16.94,4.96,10.452,7821.64,382.96,0.0,False,GPU -10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1456.0,+69/-67,0.4,17.23,8.75,12.039,3565.03,0.706,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1422.0,+108/-73,0.415,19.21,6.24,11.708,7821.64,39.414,0.0,False,GPU -12,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1413.0,+88/-73,0.357,19.74,10.49,12.201,3650.56,0.543,0.0,False,GPU -13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1401.0,+81/-72,0.33,20.48,10.84,12.887,3407.57,1.449,0.0,False,GPU -14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1370.0,+60/-45,0.213,22.35,18.6,14.631,780.12,2.129,0.0,False,CPU -15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1361.0,+70/-63,0.239,22.95,16.22,14.299,2234.83,0.653,0.0,False,CPU -16,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1352.0,+86/-61,0.325,23.52,8.67,13.532,4.97,0.545,0.0,False,GPU -17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1334.0,+82/-72,0.253,24.65,15.76,13.965,3407.57,0.173,0.0,False,GPU -18,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1329.0,+74/-58,0.227,24.94,17.24,15.058,1216.65,2.007,0.0,False,GPU -19,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1328.0,+56/-52,0.202,25.03,19.5,14.923,2234.83,0.071,0.0,False,CPU -20,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1327.0,+103/-84,0.33,25.09,7.99,14.25,51.52,40.033,0.0,False,GPU -21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1323.0,+51/-55,0.178,25.35,20.59,15.678,848.99,1.866,0.0,False,CPU -22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1323.0,+91/-73,0.253,25.36,16.7,15.733,6308.39,7.737,0.0,False,GPU -23,🧠⚡,Foundation Model,[TabICL (default) [21.21% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1316.0,+94/-98,0.279,25.8,7.59,14.435,7.68,1.615,21.21,True,GPU -24,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1313.0,+58/-46,0.161,25.95,18.68,15.66,8.13,0.126,0.0,False,CPU -25,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1312.0,+77/-83,0.223,26.06,12.6,14.889,6308.39,0.477,0.0,False,GPU -26,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1309.0,+59/-50,0.165,26.24,13.96,15.82,848.99,0.298,0.0,False,CPU -27,🧠⚡,Foundation Model,[BetaTabPFN (default) [21.21% IMPUTED]](https://arxiv.org/abs/2502.02527),➖,1306.0,+84/-85,0.235,26.42,15.22,16.996,277.28,1.727,21.21,True,GPU -28,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1302.0,+72/-63,0.16,26.67,21.7,15.671,780.12,0.381,0.0,False,CPU -29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1290.0,+73/-68,0.172,27.48,16.1,16.838,2961.52,0.521,0.0,False,CPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1287.0,+62/-54,0.163,27.67,17.51,15.815,16.29,4.082,0.0,False,GPU -31,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1281.0,+77/-76,0.211,28.03,17.74,16.232,13.3,0.146,0.0,False,GPU -32,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1272.0,+68/-66,0.16,28.65,20.28,16.037,1216.65,0.097,0.0,False,GPU -33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1248.0,+64/-62,0.144,30.15,23.3,16.297,3460.09,2.057,0.0,False,CPU -34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1246.0,+74/-66,0.138,30.3,13.32,17.567,2961.52,0.054,0.0,False,CPU -35,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1221.0,+66/-59,0.117,31.89,14.0,19.358,14.5,0.3,0.0,False,GPU -36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1220.0,+78/-78,0.11,31.94,14.62,18.183,9.77,0.046,0.0,False,CPU -37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1204.0,+86/-83,0.134,32.98,22.16,19.428,1453.07,6.678,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1195.0,+72/-79,0.107,33.56,26.02,20.536,418.39,0.972,0.0,False,CPU -39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1180.0,+71/-61,0.106,34.45,27.58,17.9,3460.09,0.121,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1176.0,+75/-82,0.13,34.76,19.79,20.959,418.39,0.147,0.0,False,CPU -41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1141.0,+77/-85,0.1,36.85,23.73,20.713,1453.07,0.637,0.0,False,CPU -42,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1134.0,+74/-98,0.073,37.24,27.69,21.857,563.93,0.997,0.0,False,CPU -43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1120.0,+51/-60,0.026,38.06,35.14,19.796,2.86,0.19,0.0,False,CPU -44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1112.0,+51/-65,0.037,38.53,35.23,20.033,3.28,0.179,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1092.0,+70/-84,0.046,39.67,32.65,22.761,563.93,0.149,0.0,False,CPU -46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1050.0,+89/-100,0.073,41.94,24.43,24.517,5.55,0.867,0.0,False,GPU -47,🧠⚡,Foundation Model,[TabFlex (default) [21.21% IMPUTED]](https://arxiv.org/abs/2506.05584),➖,1024.0,+82/-95,0.054,43.27,34.1,26.178,1.29,0.149,21.21,True,GPU -48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1012.0,+79/-77,0.028,43.85,38.13,24.002,9.39,0.15,0.0,False,CPU -49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+82/-107,0.021,44.42,33.54,26.75,0.48,0.074,0.0,False,CPU -50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,994.0,+86/-117,0.023,44.7,40.11,27.387,0.38,0.075,0.0,False,CPU -51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,984.0,+77/-97,0.025,45.18,41.29,24.643,5.2,0.522,0.0,False,CPU -52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,955.0,+74/-83,0.007,46.42,43.79,28.217,101.2,1.135,0.0,False,CPU -53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,912.0,+86/-109,0.03,48.15,19.91,32.259,497.63,0.481,0.0,False,CPU -54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,887.0,+84/-108,0.026,49.08,28.85,32.749,497.63,0.084,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,849.0,+98/-121,0.012,50.36,46.23,35.015,2.47,0.121,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,771.0,+84/-107,0.0,52.58,51.87,34.522,101.2,0.065,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,625.0,+106/-165,0.0,55.48,55.02,45.393,0.28,0.026,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1650.0,+124/-84,4.175,0.683,8.42,2.71,2200.35,8.314,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1630.0,+84/-70,7.065,0.622,9.14,5.48,6.21,0.561,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1598.0,+113/-87,6.692,0.6,10.36,4.25,2200.35,0.882,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1570.0,+96/-80,7.982,0.581,11.52,4.86,625.63,7.662,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1536.0,+102/-76,9.632,0.517,13.08,5.44,3650.56,52.333,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1533.0,+74/-57,10.676,0.488,13.21,7.36,3565.03,23.942,✔️,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1488.0,+62/-51,10.911,0.426,15.52,8.68,3035.75,1.498,✔️,0.0,False,CPU +7,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1486.0,+86/-68,10.885,0.434,15.59,7.42,137.76,3.046,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1469.0,+92/-79,13.013,0.461,16.5,5.8,4.66,0.611,➖,0.0,False,GPU +9,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1462.0,+96/-75,10.452,0.458,16.94,4.96,7821.64,382.96,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1456.0,+69/-67,12.039,0.4,17.23,8.75,3565.03,0.706,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1422.0,+108/-73,11.708,0.415,19.21,6.24,7821.64,39.414,✔️,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1413.0,+88/-73,12.201,0.357,19.74,10.49,3650.56,0.543,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1401.0,+81/-72,12.887,0.33,20.48,10.84,3407.57,1.449,✔️,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1370.0,+60/-45,14.631,0.213,22.35,18.6,780.12,2.129,✔️,0.0,False,CPU +15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1361.0,+70/-63,14.299,0.239,22.95,16.22,2234.83,0.653,✔️,0.0,False,CPU +16,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1352.0,+86/-61,13.532,0.325,23.52,8.67,4.97,0.545,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1334.0,+82/-72,13.965,0.253,24.65,15.76,3407.57,0.173,✔️,0.0,False,GPU +18,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1329.0,+74/-58,15.058,0.227,24.94,17.24,1216.65,2.007,✔️,0.0,False,GPU +19,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1328.0,+56/-52,14.923,0.202,25.03,19.5,2234.83,0.071,✔️,0.0,False,CPU +20,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1327.0,+103/-84,14.25,0.33,25.09,7.99,51.52,40.033,✔️,0.0,False,GPU +21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1323.0,+51/-55,15.678,0.178,25.35,20.59,848.99,1.866,✔️,0.0,False,CPU +22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1323.0,+91/-73,15.733,0.253,25.36,16.7,6308.39,7.737,✔️,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabICL (default) [21.21% IMPUTED]](https://arxiv.org/abs/2502.05564),1316.0,+94/-98,14.435,0.279,25.8,7.59,7.68,1.615,✔️,21.21,True,GPU +24,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1313.0,+58/-46,15.66,0.161,25.95,18.68,8.13,0.126,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1312.0,+77/-83,14.889,0.223,26.06,12.6,6308.39,0.477,✔️,0.0,False,GPU +26,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1309.0,+59/-50,15.82,0.165,26.24,13.96,848.99,0.298,✔️,0.0,False,CPU +27,🧠⚡,Foundation Model,[BetaTabPFN (default) [21.21% IMPUTED]](https://arxiv.org/abs/2502.02527),1306.0,+84/-85,16.996,0.235,26.42,15.22,277.28,1.727,➖,21.21,True,GPU +28,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1302.0,+72/-63,15.671,0.16,26.67,21.7,780.12,0.381,✔️,0.0,False,CPU +29,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1290.0,+73/-68,16.838,0.172,27.48,16.1,2961.52,0.521,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1287.0,+62/-54,15.815,0.163,27.67,17.51,16.29,4.082,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1281.0,+77/-76,16.232,0.211,28.03,17.74,13.3,0.146,✔️,0.0,False,GPU +32,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1272.0,+68/-66,16.037,0.16,28.65,20.28,1216.65,0.097,✔️,0.0,False,GPU +33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1248.0,+64/-62,16.297,0.144,30.15,23.3,3460.09,2.057,✔️,0.0,False,CPU +34,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1246.0,+74/-66,17.567,0.138,30.3,13.32,2961.52,0.054,✔️,0.0,False,CPU +35,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1221.0,+66/-59,19.358,0.117,31.89,14.0,14.5,0.3,✔️,0.0,False,GPU +36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1220.0,+78/-78,18.183,0.11,31.94,14.62,9.77,0.046,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1204.0,+86/-83,19.428,0.134,32.98,22.16,1453.07,6.678,✔️,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1195.0,+72/-79,20.536,0.107,33.56,26.02,418.39,0.972,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1180.0,+71/-61,17.9,0.106,34.45,27.58,3460.09,0.121,✔️,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1176.0,+75/-82,20.959,0.13,34.76,19.79,418.39,0.147,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1141.0,+77/-85,20.713,0.1,36.85,23.73,1453.07,0.637,✔️,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1134.0,+74/-98,21.857,0.073,37.24,27.69,563.93,0.997,✔️,0.0,False,CPU +43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1120.0,+51/-60,19.796,0.026,38.06,35.14,2.86,0.19,✔️,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1112.0,+51/-65,20.033,0.037,38.53,35.23,3.28,0.179,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1092.0,+70/-84,22.761,0.046,39.67,32.65,563.93,0.149,✔️,0.0,False,CPU +46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1050.0,+89/-100,24.517,0.073,41.94,24.43,5.55,0.867,✔️,0.0,False,GPU +47,🧠⚡,Foundation Model,[TabFlex (default) [21.21% IMPUTED]](https://arxiv.org/abs/2506.05584),1024.0,+82/-95,26.178,0.054,43.27,34.1,1.29,0.149,➖,21.21,True,GPU +48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1012.0,+79/-77,24.002,0.028,43.85,38.13,9.39,0.15,✔️,0.0,False,CPU +49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+82/-107,26.75,0.021,44.42,33.54,0.48,0.074,✔️,0.0,False,CPU +50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),994.0,+86/-117,27.387,0.023,44.7,40.11,0.38,0.075,✔️,0.0,False,CPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),984.0,+77/-97,24.643,0.025,45.18,41.29,5.2,0.522,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),955.0,+74/-83,28.217,0.007,46.42,43.79,101.2,1.135,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),912.0,+86/-109,32.259,0.03,48.15,19.91,497.63,0.481,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),887.0,+84/-108,32.749,0.026,49.08,28.85,497.63,0.084,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),849.0,+98/-121,35.015,0.012,50.36,46.23,2.47,0.121,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),771.0,+84/-107,34.522,0.0,52.58,51.87,101.2,0.065,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),625.0,+106/-165,45.393,0.0,55.48,55.02,0.28,0.026,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/winrate_matrix.png.zip index 3e9f0a42a61bff7e25a22da5071d8d9fa4e14ce4..68cb3aaecac0efe1c1a9a689cf93e39688365978 100644 --- a/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_all/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f12e7a9e0d7458bb5569a6d568573d8e0b8ac3a5c31b95b77c248e3d59fb9771 +oid sha256:d22360b40385e6c41366aca2b7b9302a934c01a500b3cf5fccdcf216391e6785 size 2416990 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index a2cca212f7d1a952f819750ab74749a9532d5dce..5b193f667ca433797f81f11f8f8cf1c59a2d2ae2 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:347ece1eab807333dda6cc4f437de798dbf715b69556ebe6dd3f51f39af0c99a +oid sha256:02dd24cd571aa73fb038155030805c59a3b85ac8289a17f13b55b415c4528588 size 492020 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip index 10fe889affac1bcfe1b079afd9e98a40a92e566d..9bde16ed3444ad947590c2b3a9bd840c00470b73 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7371fdfbb3a8c478a67a430f9b7ecb37635c048604130741c840d99b8623e64a -size 1062154 +oid sha256:dae95d50999e4d436a1f0dd33029bc78d5356fe64d0ef5021901bde4157757f0 +size 1046013 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/tuning-impact-elo.png.zip index 1b4dc0fc1c29b632da50c130bdd9e8179a8ed593..349855878157c987d99d0d22d310d8b06e80db5d 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5f045000dff91c57f8134ad8eb8bb3956854b487b51cf5a3c2b7cf1cbc2e1d2a +oid sha256:d2d25057404273423286e27d8748d667742b8a44c939e805d9f6ee66f8a28d96 size 183658 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/website_leaderboard.csv index 9bcca32f6b2b8a324fb2f08ae31bebfd2ab5c4be..3f928c74e3514929f8ce21c7398c165f2d063478 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1568.0,+105/-74,0.589,10.57,3.67,5.636,1877.76,10.745,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1562.0,+112/-85,0.623,10.8,3.7,6.142,442.6,6.737,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1547.0,+99/-70,0.567,11.47,4.88,7.435,1231.21,1.36,0.0,False,CPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1546.0,+95/-69,0.567,11.5,6.22,7.006,4.79,0.708,0.0,False,GPU -4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1537.0,+102/-63,0.551,11.92,5.24,6.667,1877.76,1.035,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1500.0,+69/-60,0.477,13.73,8.61,9.187,2721.85,9.311,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1424.0,+76/-72,0.388,17.92,8.05,11.154,2721.85,0.265,0.0,False,GPU -7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1412.0,+81/-64,0.382,18.67,7.44,11.026,2180.12,1.135,0.0,False,GPU -8,🧠⚡,Foundation Model,[TabICL (default) [3.33% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1394.0,+82/-68,0.349,19.78,7.26,10.012,8.05,2.008,3.33,True,GPU -9,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1388.0,+52/-50,0.292,20.12,14.44,12.615,328.64,0.774,0.0,False,CPU -10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1374.0,+71/-61,0.34,21.03,7.84,9.545,5890.6,409.986,0.0,False,GPU -11,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1368.0,+62/-51,0.304,21.4,10.27,12.209,3.84,0.068,0.0,False,CPU -12,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1367.0,+70/-70,0.302,21.45,11.5,13.18,462.92,0.613,0.0,False,CPU -13,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1366.0,+87/-76,0.331,21.52,10.11,12.046,3436.74,0.405,0.0,False,GPU -14,🧠🔁,Neural Network,[LimiX (default) [16.67% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1362.0,+93/-84,0.365,21.75,6.22,13.114,1.95,0.488,16.67,True,GPU -15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1358.0,+69/-73,0.302,22.05,12.29,12.343,1043.89,0.478,0.0,False,CPU -16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1350.0,+94/-95,0.353,22.52,8.36,12.196,3436.74,8.478,0.0,False,GPU -17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1340.0,+85/-73,0.305,23.22,12.12,12.161,2180.12,0.117,0.0,False,GPU -18,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1332.0,+56/-61,0.276,23.72,14.78,12.867,1043.89,0.041,0.0,False,CPU -19,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1332.0,+76/-80,0.281,23.72,9.4,11.754,5890.6,39.565,0.0,False,GPU -20,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1332.0,+95/-64,0.275,23.73,13.6,14.412,248.47,1.7,0.0,False,GPU -21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1331.0,+75/-80,0.262,23.77,10.5,13.559,462.92,0.108,0.0,False,CPU -22,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1320.0,+58/-62,0.23,24.53,19.55,13.786,328.64,0.089,0.0,False,CPU -23,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1315.0,+94/-79,0.304,24.85,9.85,14.164,2914.83,17.91,33.33,True,GPU -24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1311.0,+68/-57,0.213,25.1,15.6,14.601,1579.5,0.428,0.0,False,CPU -25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1304.0,+69/-82,0.224,25.6,17.15,13.468,559.77,1.513,0.0,False,GPU -26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1299.0,+58/-53,0.175,25.9,16.67,13.707,9.04,1.162,0.0,False,GPU -27,🧠🔁,Neural Network,[Mitra (default) [33.33% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1296.0,+102/-77,0.272,26.07,9.31,14.387,106.33,2.667,33.33,True,GPU -28,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1285.0,+89/-78,0.24,26.83,17.89,14.697,8.0,0.124,0.0,False,GPU -29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1273.0,+73/-55,0.179,27.63,20.43,13.865,2206.58,2.301,0.0,False,CPU -30,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1262.0,+91/-73,0.221,28.38,13.63,15.933,2914.83,0.225,33.33,True,GPU -31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1258.0,+63/-69,0.148,28.68,20.29,15.134,559.77,0.086,0.0,False,GPU -32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1257.0,+67/-57,0.14,28.73,13.78,15.369,1579.5,0.05,0.0,False,CPU -33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1251.0,+70/-60,0.15,29.1,13.32,16.377,4.79,0.045,0.0,False,CPU -34,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1222.0,+72/-90,0.179,31.08,11.5,14.951,47.89,39.446,0.0,False,GPU -35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1218.0,+76/-88,0.139,31.35,21.38,16.897,561.29,4.462,0.0,False,CPU -36,🧠⚡,Foundation Model,[TabPFNv2 (default) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1216.0,+78/-78,0.196,31.48,10.46,17.278,2.61,0.256,33.33,True,GPU -37,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1200.0,+85/-80,0.141,32.53,18.88,16.455,1.42,0.118,0.0,False,CPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1191.0,+77/-69,0.131,33.07,25.9,16.272,2206.58,0.107,0.0,False,CPU -39,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1188.0,+63/-62,0.111,33.25,13.68,17.287,11.15,0.312,0.0,False,GPU -40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1173.0,+74/-68,0.081,34.25,27.98,18.237,122.9,0.583,0.0,False,CPU -41,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1140.0,+68/-85,0.061,36.3,29.95,19.635,171.61,0.552,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1138.0,+79/-101,0.096,36.47,23.6,19.002,561.29,0.236,0.0,False,CPU -43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1135.0,+63/-66,0.082,36.65,32.0,17.467,0.93,0.094,0.0,False,CPU -44,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1130.0,+79/-75,0.086,36.97,21.52,19.849,122.9,0.072,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1106.0,+64/-69,0.021,38.42,34.53,20.285,171.61,0.053,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1044.0,+74/-69,0.035,41.93,36.06,21.546,4.97,0.092,0.0,False,CPU -47,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1010.0,+104/-121,0.059,43.7,33.56,26.685,0.81,0.115,0.0,False,GPU -48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+76/-92,0.024,44.18,32.57,24.447,0.29,0.031,0.0,False,CPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,998.0,+75/-93,0.031,44.27,40.03,22.018,2.79,0.259,0.0,False,CPU -50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,991.0,+63/-92,0.006,44.63,42.41,24.644,140.66,3.209,0.0,False,CPU -51,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,973.0,+93/-126,0.041,45.47,18.11,28.274,155.53,0.277,0.0,False,CPU -52,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,941.0,+96/-133,0.03,46.88,26.66,28.935,155.53,0.05,0.0,False,CPU -53,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,930.0,+91/-113,0.037,47.33,41.1,27.199,2.71,0.392,0.0,False,GPU -54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,924.0,+92/-110,0.016,47.6,43.29,26.269,0.18,0.035,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,904.0,+98/-113,0.013,48.4,44.34,30.667,0.8,0.088,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,827.0,+62/-96,0.0,51.03,50.24,30.916,140.66,0.188,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,586.0,+96/-207,0.0,56.0,55.7,43.999,0.15,0.024,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1568.0,+105/-74,5.636,0.589,10.57,3.67,1877.76,10.745,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1562.0,+112/-85,6.142,0.623,10.8,3.7,442.6,6.737,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1547.0,+99/-70,7.435,0.567,11.47,4.88,1231.21,1.36,✔️,0.0,False,CPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1546.0,+95/-69,7.006,0.567,11.5,6.22,4.79,0.708,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1537.0,+102/-63,6.667,0.551,11.92,5.24,1877.76,1.035,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1500.0,+69/-60,9.187,0.477,13.73,8.61,2721.85,9.311,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1424.0,+76/-72,11.154,0.388,17.92,8.05,2721.85,0.265,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1412.0,+81/-64,11.026,0.382,18.67,7.44,2180.12,1.135,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabICL (default) [3.33% IMPUTED]](https://arxiv.org/abs/2502.05564),1394.0,+82/-68,10.012,0.349,19.78,7.26,8.05,2.008,✔️,3.33,True,GPU +9,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1388.0,+52/-50,12.615,0.292,20.12,14.44,328.64,0.774,✔️,0.0,False,CPU +10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1374.0,+71/-61,9.545,0.34,21.03,7.84,5890.6,409.986,✔️,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1368.0,+62/-51,12.209,0.304,21.4,10.27,3.84,0.068,✔️,0.0,False,CPU +12,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1367.0,+70/-70,13.18,0.302,21.45,11.5,462.92,0.613,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1366.0,+87/-76,12.046,0.331,21.52,10.11,3436.74,0.405,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[LimiX (default) [16.67% IMPUTED]](https://arxiv.org/abs/2509.03505),1362.0,+93/-84,13.114,0.365,21.75,6.22,1.95,0.488,➖,16.67,True,GPU +15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1358.0,+69/-73,12.343,0.302,22.05,12.29,1043.89,0.478,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1350.0,+94/-95,12.196,0.353,22.52,8.36,3436.74,8.478,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1340.0,+85/-73,12.161,0.305,23.22,12.12,2180.12,0.117,✔️,0.0,False,GPU +18,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1332.0,+56/-61,12.867,0.276,23.72,14.78,1043.89,0.041,✔️,0.0,False,CPU +19,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1332.0,+76/-80,11.754,0.281,23.72,9.4,5890.6,39.565,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1332.0,+95/-64,14.412,0.275,23.73,13.6,248.47,1.7,➖,0.0,False,GPU +21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1331.0,+75/-80,13.559,0.262,23.77,10.5,462.92,0.108,✔️,0.0,False,CPU +22,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1320.0,+58/-62,13.786,0.23,24.53,19.55,328.64,0.089,✔️,0.0,False,CPU +23,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1315.0,+94/-79,14.164,0.304,24.85,9.85,2914.83,17.91,✔️,33.33,True,GPU +24,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1311.0,+68/-57,14.601,0.213,25.1,15.6,1579.5,0.428,✔️,0.0,False,CPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1304.0,+69/-82,13.468,0.224,25.6,17.15,559.77,1.513,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1299.0,+58/-53,13.707,0.175,25.9,16.67,9.04,1.162,✔️,0.0,False,GPU +27,🧠⚡,Foundation Model,[Mitra (default) [33.33% IMPUTED]](https://arxiv.org/abs/2510.21204),1296.0,+102/-77,14.387,0.272,26.07,9.31,106.33,2.667,✔️,33.33,True,GPU +28,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1285.0,+89/-78,14.697,0.24,26.83,17.89,8.0,0.124,✔️,0.0,False,GPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1273.0,+73/-55,13.865,0.179,27.63,20.43,2206.58,2.301,✔️,0.0,False,CPU +30,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1262.0,+91/-73,15.933,0.221,28.38,13.63,2914.83,0.225,✔️,33.33,True,GPU +31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1258.0,+63/-69,15.134,0.148,28.68,20.29,559.77,0.086,✔️,0.0,False,GPU +32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1257.0,+67/-57,15.369,0.14,28.73,13.78,1579.5,0.05,✔️,0.0,False,CPU +33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1251.0,+70/-60,16.377,0.15,29.1,13.32,4.79,0.045,✔️,0.0,False,CPU +34,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1222.0,+72/-90,14.951,0.179,31.08,11.5,47.89,39.446,✔️,0.0,False,GPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1218.0,+76/-88,16.897,0.139,31.35,21.38,561.29,4.462,✔️,0.0,False,CPU +36,🧠⚡,Foundation Model,[TabPFNv2 (default) [33.33% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1216.0,+78/-78,17.278,0.196,31.48,10.46,2.61,0.256,✔️,33.33,True,GPU +37,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1200.0,+85/-80,16.455,0.141,32.53,18.88,1.42,0.118,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1191.0,+77/-69,16.272,0.131,33.07,25.9,2206.58,0.107,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1188.0,+63/-62,17.287,0.111,33.25,13.68,11.15,0.312,✔️,0.0,False,GPU +40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1173.0,+74/-68,18.237,0.081,34.25,27.98,122.9,0.583,✔️,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1140.0,+68/-85,19.635,0.061,36.3,29.95,171.61,0.552,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1138.0,+79/-101,19.002,0.096,36.47,23.6,561.29,0.236,✔️,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1135.0,+63/-66,17.467,0.082,36.65,32.0,0.93,0.094,✔️,0.0,False,CPU +44,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1130.0,+79/-75,19.849,0.086,36.97,21.52,122.9,0.072,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1106.0,+64/-69,20.285,0.021,38.42,34.53,171.61,0.053,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1044.0,+74/-69,21.546,0.035,41.93,36.06,4.97,0.092,✔️,0.0,False,CPU +47,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1010.0,+104/-121,26.685,0.059,43.7,33.56,0.81,0.115,➖,0.0,False,GPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+76/-92,24.447,0.024,44.18,32.57,0.29,0.031,✔️,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),998.0,+75/-93,22.018,0.031,44.27,40.03,2.79,0.259,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),991.0,+63/-92,24.644,0.006,44.63,42.41,140.66,3.209,✔️,0.0,False,CPU +51,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),973.0,+93/-126,28.274,0.041,45.47,18.11,155.53,0.277,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),941.0,+96/-133,28.935,0.03,46.88,26.66,155.53,0.05,✔️,0.0,False,CPU +53,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),930.0,+91/-113,27.199,0.037,47.33,41.1,2.71,0.392,✔️,0.0,False,GPU +54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),924.0,+92/-110,26.269,0.016,47.6,43.29,0.18,0.035,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),904.0,+98/-113,30.667,0.013,48.4,44.34,0.8,0.088,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),827.0,+62/-96,30.916,0.0,51.03,50.24,140.66,0.188,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),586.0,+96/-207,43.999,0.0,56.0,55.7,0.15,0.024,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/winrate_matrix.png.zip index dd456d41fd4cecf92cabf16557d18bf051c996d2..2a57fa64073675d96ee4edc0a5e0e5ae48d88de6 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0981a5d5ab05f632ac0ae85de016bf0a95549ab0d5dfd2423f885cd6ad7bb030 +oid sha256:72788737937043034cfe107767f734cf0961225612a691ac783b36e76fc591a5 size 2329337 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index b88c38a35d7a3bedcd9481cab8c455f9452c1376..615ed59abbfe1f021a17bea9594a9c80a66cfb7f 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4c4da7f7b53bb4f19ba91748131ea59211a4eaf3b7ac7a4506dacdf44a2665cb +oid sha256:fed291c83460899264064b3f8e1047fb297987310c053a1760d0c448ec24b3a1 size 448234 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip index 263d5c1ab0148cfdd44222c98d71c1c01e3030c4..f2ce5133485bc61bdea30eb66cb5350bddc90b33 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:576e294afa617e55c7e28cae2cea0eb3a304e0eb873e1dca1d2ca7514aefbfb3 -size 1001999 +oid sha256:6b0b2cca46e709f2ab78a838fe985612b0010f15d74d7295af91f07b64415511 +size 933385 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/tuning-impact-elo.png.zip index e4277058d79cee575d51c48ea8af0db92abcf125..38618f877357fce214fe9817ed1dddd178adf0a7 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9aa94612b23003628b74e8131f5e534773f14abc331137384b2d26f0591203d7 +oid sha256:6fb355fddb9d884690ab134d30c7c8cbf493dad72c950773c495cc5070c66856 size 134935 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/website_leaderboard.csv index 944f59a8f5fd947d06096b4b0069d6ceec6ab4c8..8150f45cb95757e78b1004b7c2cf1a34d912397e 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/website_leaderboard.csv @@ -1,55 +1,55 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,2055.0,+219/-66,0.859,3.44,2.17,1.559,310.55,1.824,0.0,False,CPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1995.0,+270/-81,0.834,4.33,2.22,2.117,256.14,2.868,0.0,False,GPU -2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1747.0,+274/-161,0.619,10.78,3.85,4.475,2149.06,6.373,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1729.0,+219/-166,0.527,11.44,4.12,5.896,841.84,11.882,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1714.0,+130/-48,0.517,12.0,8.92,5.584,1655.56,2.483,0.0,False,GPU -5,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1708.0,+186/-105,0.46,12.22,8.36,7.366,114.48,0.404,0.0,False,CPU -6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1689.0,+204/-112,0.517,13.0,7.63,6.578,618.23,0.184,0.0,False,CPU -7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1675.0,+224/-137,0.452,13.56,7.63,7.894,841.84,1.263,0.0,False,GPU -8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1670.0,+201/-118,0.478,13.78,5.14,7.403,807.72,0.575,0.0,False,GPU -9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1661.0,+168/-101,0.489,14.17,8.72,6.676,618.23,0.031,0.0,False,CPU -10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1660.0,+186/-100,0.43,14.22,10.24,8.502,87.3,0.533,0.0,False,CPU -11,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1644.0,+181/-117,0.418,14.89,10.64,9.317,2149.06,0.318,0.0,False,GPU -12,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1635.0,+152/-84,0.46,15.28,5.61,6.592,2.12,0.043,0.0,False,CPU -13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1617.0,+163/-118,0.403,16.11,10.87,7.688,114.48,0.058,0.0,False,CPU -14,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1609.0,+189/-106,0.394,16.44,9.39,8.195,1655.56,0.086,0.0,False,GPU -15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1581.0,+177/-114,0.326,17.78,13.98,9.447,87.3,0.069,0.0,False,CPU -16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1578.0,+230/-158,0.396,17.89,8.37,8.051,807.72,0.07,0.0,False,GPU -17,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1548.0,+182/-140,0.333,19.33,9.01,7.804,2.32,0.889,0.0,False,GPU -18,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1537.0,+176/-155,0.29,19.89,11.67,9.501,9.96,4.847,0.0,False,GPU -19,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1474.0,+173/-195,0.249,23.0,18.82,10.609,5.77,0.078,0.0,False,GPU -20,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1461.0,+174/-196,0.26,23.67,10.33,10.169,0.48,0.047,0.0,False,CPU -21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1461.0,+158/-145,0.248,23.67,15.34,9.627,559.89,1.156,0.0,False,GPU -22,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1461.0,+119/-70,0.179,23.67,19.26,9.923,1128.19,1.674,0.0,False,CPU -23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1444.0,+183/-128,0.215,24.56,20.49,13.232,262.92,0.205,0.0,False,CPU -24,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1442.0,+169/-148,0.186,24.67,18.8,10.124,4560.06,421.968,0.0,False,GPU -25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1431.0,+163/-120,0.166,25.22,17.58,12.808,559.89,0.056,0.0,False,GPU -26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1412.0,+85/-76,0.087,26.22,23.87,11.884,6.55,0.295,0.0,False,GPU -27,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1380.0,+112/-153,0.112,27.89,22.51,12.385,368.88,2.507,0.0,False,CPU -28,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1366.0,+202/-171,0.132,28.61,19.08,13.822,4560.06,39.452,0.0,False,GPU -29,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1338.0,+181/-190,0.13,30.0,19.31,13.509,92.9,0.786,0.0,False,GPU -30,🧠🔁,Neural Network,[LimiX (default) [55.56% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1338.0,+244/-253,0.218,30.0,6.8,18.727,0.68,0.014,55.56,True,GPU -31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1323.0,+165/-119,0.077,30.78,27.72,16.195,0.92,0.018,0.0,False,CPU -32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1314.0,+200/-159,0.097,31.22,25.64,14.917,262.92,0.015,0.0,False,CPU -33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1307.0,+122/-138,0.083,31.56,27.8,13.503,1128.19,0.055,0.0,False,CPU -34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1300.0,+123/-154,0.036,31.89,28.72,13.474,40.45,0.331,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1294.0,+55/-72,0.018,32.22,31.24,14.752,43.93,0.204,0.0,False,CPU -36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1266.0,+109/-148,0.047,33.56,31.32,12.579,0.34,0.052,0.0,False,CPU -37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1251.0,+51/-62,0.0,34.28,33.94,15.173,43.93,0.018,0.0,False,CPU -38,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1230.0,+108/-121,0.015,35.22,33.77,14.121,9.93,0.224,0.0,False,GPU -39,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1210.0,+137/-213,0.022,36.17,32.49,18.236,41.8,39.131,0.0,False,GPU -40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1194.0,+128/-148,0.02,36.89,34.1,17.071,40.45,0.028,0.0,False,CPU -41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1178.0,+108/-183,0.02,37.56,35.52,16.535,368.88,0.084,0.0,False,CPU -42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1142.0,+101/-139,0.013,39.11,37.42,16.268,1.81,0.048,0.0,False,CPU -43,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1104.0,+173/-273,0.019,40.61,36.3,25.579,43.96,0.099,0.0,False,CPU -44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1102.0,+114/-195,0.02,40.67,37.79,20.929,514.21,52.341,0.0,False,CPU -45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1067.0,+137/-238,0.026,42.0,38.71,19.282,1.61,0.113,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1056.0,+170/-258,0.002,42.39,39.36,26.315,43.96,0.024,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+121/-272,0.0,44.28,43.24,22.467,0.11,0.012,0.0,False,CPU -48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,896.0,+238/-618,0.027,47.22,39.51,30.82,2.77,0.157,0.0,False,GPU -49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,887.0,+98/-215,0.0,47.44,47.08,28.058,0.22,0.022,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,859.0,+70/-166,0.0,48.11,47.78,28.846,514.21,2.49,0.0,False,CPU -51,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,859.0,+102/-228,0.0,48.11,47.82,25.047,0.1,0.02,0.0,False,CPU -52,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,823.0,+121/-262,0.0,48.89,48.33,33.192,0.2,0.035,0.0,False,GPU -53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,488.0,+95/-357,0.0,53.11,53.08,46.138,0.11,0.089,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",2055.0,+219/-66,1.559,0.859,3.44,2.17,310.55,1.824,✔️,0.0,False,CPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1995.0,+270/-81,2.117,0.834,4.33,2.22,256.14,2.868,✔️,0.0,False,GPU +2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1747.0,+274/-161,4.475,0.619,10.78,3.85,2149.06,6.373,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1729.0,+219/-166,5.896,0.527,11.44,4.12,841.84,11.882,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1714.0,+130/-48,5.584,0.517,12.0,8.92,1655.56,2.483,✔️,0.0,False,GPU +5,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1708.0,+186/-105,7.366,0.46,12.22,8.36,114.48,0.404,✔️,0.0,False,CPU +6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1689.0,+204/-112,6.578,0.517,13.0,7.63,618.23,0.184,✔️,0.0,False,CPU +7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1675.0,+224/-137,7.894,0.452,13.56,7.63,841.84,1.263,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1670.0,+201/-118,7.403,0.478,13.78,5.14,807.72,0.575,✔️,0.0,False,GPU +9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1661.0,+168/-101,6.676,0.489,14.17,8.72,618.23,0.031,✔️,0.0,False,CPU +10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1660.0,+186/-100,8.502,0.43,14.22,10.24,87.3,0.533,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1644.0,+181/-117,9.317,0.418,14.89,10.64,2149.06,0.318,✔️,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1635.0,+152/-84,6.592,0.46,15.28,5.61,2.12,0.043,✔️,0.0,False,CPU +13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1617.0,+163/-118,7.688,0.403,16.11,10.87,114.48,0.058,✔️,0.0,False,CPU +14,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1609.0,+189/-106,8.195,0.394,16.44,9.39,1655.56,0.086,✔️,0.0,False,GPU +15,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1581.0,+177/-114,9.447,0.326,17.78,13.98,87.3,0.069,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1578.0,+230/-158,8.051,0.396,17.89,8.37,807.72,0.07,✔️,0.0,False,GPU +17,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1548.0,+182/-140,7.804,0.333,19.33,9.01,2.32,0.889,✔️,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1537.0,+176/-155,9.501,0.29,19.89,11.67,9.96,4.847,✔️,0.0,False,GPU +19,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1474.0,+173/-195,10.609,0.249,23.0,18.82,5.77,0.078,✔️,0.0,False,GPU +20,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1461.0,+174/-196,10.169,0.26,23.67,10.33,0.48,0.047,✔️,0.0,False,CPU +21,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1461.0,+158/-145,9.627,0.248,23.67,15.34,559.89,1.156,✔️,0.0,False,GPU +22,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1461.0,+119/-70,9.923,0.179,23.67,19.26,1128.19,1.674,✔️,0.0,False,CPU +23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1444.0,+183/-128,13.232,0.215,24.56,20.49,262.92,0.205,✔️,0.0,False,CPU +24,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1442.0,+169/-148,10.124,0.186,24.67,18.8,4560.06,421.968,✔️,0.0,False,GPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1431.0,+163/-120,12.808,0.166,25.22,17.58,559.89,0.056,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1412.0,+85/-76,11.884,0.087,26.22,23.87,6.55,0.295,✔️,0.0,False,GPU +27,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1380.0,+112/-153,12.385,0.112,27.89,22.51,368.88,2.507,✔️,0.0,False,CPU +28,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1366.0,+202/-171,13.822,0.132,28.61,19.08,4560.06,39.452,✔️,0.0,False,GPU +29,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1338.0,+181/-190,13.509,0.13,30.0,19.31,92.9,0.786,➖,0.0,False,GPU +30,🧠⚡,Foundation Model,[LimiX (default) [55.56% IMPUTED]](https://arxiv.org/abs/2509.03505),1338.0,+244/-253,18.727,0.218,30.0,6.8,0.68,0.014,➖,55.56,True,GPU +31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1323.0,+165/-119,16.195,0.077,30.78,27.72,0.92,0.018,✔️,0.0,False,CPU +32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1314.0,+200/-159,14.917,0.097,31.22,25.64,262.92,0.015,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1307.0,+122/-138,13.503,0.083,31.56,27.8,1128.19,0.055,✔️,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1300.0,+123/-154,13.474,0.036,31.89,28.72,40.45,0.331,✔️,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1294.0,+55/-72,14.752,0.018,32.22,31.24,43.93,0.204,✔️,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1266.0,+109/-148,12.579,0.047,33.56,31.32,0.34,0.052,✔️,0.0,False,CPU +37,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1251.0,+51/-62,15.173,0.0,34.28,33.94,43.93,0.018,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1230.0,+108/-121,14.121,0.015,35.22,33.77,9.93,0.224,✔️,0.0,False,GPU +39,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1210.0,+137/-213,18.236,0.022,36.17,32.49,41.8,39.131,✔️,0.0,False,GPU +40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1194.0,+128/-148,17.071,0.02,36.89,34.1,40.45,0.028,✔️,0.0,False,CPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1178.0,+108/-183,16.535,0.02,37.56,35.52,368.88,0.084,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1142.0,+101/-139,16.268,0.013,39.11,37.42,1.81,0.048,✔️,0.0,False,CPU +43,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1104.0,+173/-273,25.579,0.019,40.61,36.3,43.96,0.099,✔️,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1102.0,+114/-195,20.929,0.02,40.67,37.79,514.21,52.341,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1067.0,+137/-238,19.282,0.026,42.0,38.71,1.61,0.113,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),1056.0,+170/-258,26.315,0.002,42.39,39.36,43.96,0.024,✔️,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+121/-272,22.467,0.0,44.28,43.24,0.11,0.012,✔️,0.0,False,CPU +48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),896.0,+238/-618,30.82,0.027,47.22,39.51,2.77,0.157,✔️,0.0,False,GPU +49,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),887.0,+98/-215,28.058,0.0,47.44,47.08,0.22,0.022,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),859.0,+70/-166,28.846,0.0,48.11,47.78,514.21,2.49,✔️,0.0,False,CPU +51,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),859.0,+102/-228,25.047,0.0,48.11,47.82,0.1,0.02,✔️,0.0,False,CPU +52,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),823.0,+121/-262,33.192,0.0,48.89,48.33,0.2,0.035,➖,0.0,False,GPU +53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),488.0,+95/-357,46.138,0.0,53.11,53.08,0.11,0.089,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/winrate_matrix.png.zip index e56ea89150a6c6c33aeadc625bc3f1892c566b74..9ba3e054d0850001f43e29c9121bb0b7d0902e10 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6941472102817838bcb51720593592be79ef6a4ee9b400ca3c905e5cb43a25f6 +oid sha256:21109011fe8ebcd72973055231212590179b02ef38fddaf5e6f1df1a53ba5009 size 1568808 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index c33175fabb9ae8f8367e4e8b2827b1eaeea2ba1b..be9d6e9439894c154d755e2e95498f96129508b5 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d0db8c3ca1b450190cc5a5fca59c91385075ab62dbe2a81a0a983b5fd64eb882 +oid sha256:86bb4df1cdfd81af44084f4ac48081b6a74088b02f509a42cff9b013ecb9e62a size 508699 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip index 5bd36e5f586d2f77cc688ab1c0d5c48ee8b9ee41..c76773acca9a5dafc2e5bc8be22102bce07d3673 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b96111c03fc5ef9f581c67a6fc330810645ae97ebde9b220693457565914cd78 -size 1029384 +oid sha256:3de61c20ed929d8cb707f15ea5588b90007245ee0a0136774c1d41d3aae841f0 +size 1027994 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/tuning-impact-elo.png.zip index 6262d304a2840429e4463ec39039b9f5c92a1c1a..e980fc5c4ffc6de3b5c14b69147d0e7fd1c72c9b 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a4a53f90fe84061c32a5d9b6b538b5e214ec4af58add43f9309860f9c7f08718 +oid sha256:87b52d1cf05d68b635f6a122d29b928c3401516bb6fff0491443da12e66368c1 size 163970 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/website_leaderboard.csv index a1f8028ab04dfc937cc06aa7fca357be2640081d..ce3f87366f5c688e1fc65a1f5fe93fcf02ef2eb2 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1622.0,+93/-51,0.644,7.95,5.49,6.664,5.53,0.55,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1562.0,+146/-85,0.606,10.19,3.5,5.525,2125.17,9.785,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1538.0,+139/-103,0.583,11.21,4.62,6.141,2125.17,0.882,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1488.0,+128/-92,0.525,13.57,5.17,7.867,613.33,8.472,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1470.0,+81/-86,0.441,14.48,8.48,10.731,2791.97,23.942,0.0,False,GPU -5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1462.0,+91/-82,0.435,14.9,10.49,9.953,1886.83,1.333,0.0,False,CPU -6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1447.0,+142/-74,0.434,15.69,7.38,10.606,3275.74,52.333,4.76,True,GPU -7,🧠🔁,Neural Network,[Mitra (default) [4.76% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1417.0,+139/-66,0.388,17.4,6.94,10.924,145.41,3.958,4.76,True,GPU -8,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1412.0,+107/-82,0.425,17.69,5.99,10.708,4.14,0.557,0.0,False,GPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1401.0,+111/-76,0.365,18.36,7.57,12.422,2791.97,0.544,0.0,False,GPU -10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1382.0,+100/-80,0.382,19.48,6.28,9.297,6616.87,408.881,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabICL (default) [4.76% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1378.0,+81/-87,0.353,19.74,6.25,10.231,7.68,1.811,4.76,True,GPU -12,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1363.0,+114/-83,0.314,20.67,12.04,14.798,298.33,2.036,0.0,False,GPU -13,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1362.0,+102/-68,0.315,20.74,10.49,13.132,3275.74,0.512,4.76,True,GPU -14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1361.0,+104/-87,0.334,20.76,9.2,12.578,2237.77,1.273,0.0,False,GPU -15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1351.0,+96/-92,0.339,21.43,7.72,10.868,6616.87,39.713,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1332.0,+66/-54,0.219,22.64,17.52,14.378,512.11,1.17,0.0,False,CPU -17,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1310.0,+81/-60,0.216,24.02,15.97,14.616,4.51,0.124,0.0,False,CPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1305.0,+138/-85,0.279,24.36,9.9,13.216,4618.5,0.426,0.0,False,GPU -19,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1296.0,+92/-85,0.196,24.95,14.11,15.188,1640.58,0.552,0.0,False,CPU -20,🧠⚡,Foundation Model,[TabPFNv2 (default) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1294.0,+109/-93,0.28,25.14,7.87,15.054,4.16,0.543,4.76,True,GPU -21,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1291.0,+114/-102,0.256,25.31,14.98,13.922,2237.77,0.147,0.0,False,GPU -22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1290.0,+88/-79,0.216,25.4,13.7,15.671,700.96,0.743,0.0,False,CPU -23,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1287.0,+80/-61,0.185,25.57,14.74,14.488,11.09,2.46,0.0,False,GPU -24,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1282.0,+89/-84,0.194,25.93,16.64,14.814,1103.84,0.559,0.0,False,CPU -25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1274.0,+82/-84,0.208,26.43,18.06,15.114,536.23,1.738,0.0,False,GPU -26,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1265.0,+79/-77,0.186,27.05,10.35,16.075,700.96,0.203,0.0,False,CPU -27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1261.0,+77/-68,0.147,27.29,11.48,15.562,1640.58,0.059,0.0,False,CPU -28,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1259.0,+68/-70,0.173,27.43,23.57,15.646,512.11,0.122,0.0,False,CPU -29,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1257.0,+114/-91,0.233,27.55,16.83,15.505,4618.5,9.218,0.0,False,GPU -30,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1253.0,+85/-82,0.168,27.81,21.05,15.521,1103.84,0.051,0.0,False,CPU -31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1250.0,+90/-86,0.162,28.0,10.87,16.455,7.16,0.046,0.0,False,CPU -32,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1246.0,+98/-87,0.219,28.29,17.5,16.449,11.3,0.149,0.0,False,GPU -33,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1245.0,+106/-102,0.239,28.33,8.98,13.544,52.04,39.82,0.0,False,GPU -34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1230.0,+98/-88,0.175,29.33,20.98,15.554,2535.0,2.546,0.0,False,CPU -35,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1221.0,+86/-94,0.137,29.98,21.67,16.131,536.23,0.097,0.0,False,GPU -36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1187.0,+108/-107,0.133,32.21,10.89,18.644,13.39,0.329,0.0,False,GPU -37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1180.0,+101/-110,0.147,32.64,20.89,18.831,650.63,5.616,0.0,False,CPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1170.0,+94/-93,0.146,33.33,25.03,17.459,2535.0,0.112,0.0,False,CPU -39,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1139.0,+87/-99,0.096,35.26,27.67,20.279,206.04,0.946,0.0,False,CPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1134.0,+104/-151,0.124,35.62,20.56,20.059,650.63,0.442,0.0,False,CPU -41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1125.0,+83/-84,0.081,36.14,29.19,19.149,1.98,0.122,0.0,False,CPU -42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1118.0,+108/-111,0.114,36.62,18.51,21.039,206.04,0.098,0.0,False,CPU -43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1098.0,+74/-90,0.078,37.79,32.21,19.563,2.47,0.168,0.0,False,CPU -44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1097.0,+93/-108,0.079,37.86,29.32,21.728,494.48,0.936,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1061.0,+86/-101,0.028,40.0,34.61,22.476,494.48,0.093,0.0,False,CPU -46,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1051.0,+115/-148,0.085,40.52,29.4,23.895,1.29,0.149,0.0,False,GPU -47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1010.0,+91/-135,0.044,42.76,35.3,23.809,7.76,0.147,0.0,False,CPU -48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+105/-148,0.034,43.29,29.07,25.296,0.39,0.041,0.0,False,CPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,976.0,+86/-127,0.029,44.48,40.11,23.191,3.48,0.427,0.0,False,CPU -50,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,947.0,+100/-143,0.041,45.86,40.9,25.646,2.66,0.702,0.0,False,GPU -51,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,947.0,+119/-146,0.023,45.86,40.57,26.793,0.33,0.044,0.0,False,CPU -52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,944.0,+72/-114,0.0,45.95,44.39,26.237,89.61,1.627,0.0,False,CPU -53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,926.0,+112/-152,0.048,46.79,14.84,29.429,280.42,0.382,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,900.0,+118/-175,0.018,47.86,42.54,31.785,1.48,0.127,0.0,False,CPU -55,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,895.0,+118/-156,0.04,48.05,23.27,30.058,280.42,0.072,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,793.0,+89/-139,0.0,51.52,50.65,31.804,89.61,0.08,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,608.0,+105/-184,0.0,55.52,55.12,43.082,0.24,0.02,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1622.0,+93/-51,6.664,0.644,7.95,5.49,5.53,0.55,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1562.0,+146/-85,5.525,0.606,10.19,3.5,2125.17,9.785,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1538.0,+139/-103,6.141,0.583,11.21,4.62,2125.17,0.882,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1488.0,+128/-92,7.867,0.525,13.57,5.17,613.33,8.472,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1470.0,+81/-86,10.731,0.441,14.48,8.48,2791.97,23.942,✔️,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1462.0,+91/-82,9.953,0.435,14.9,10.49,1886.83,1.333,✔️,0.0,False,CPU +6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1447.0,+142/-74,10.606,0.434,15.69,7.38,3275.74,52.333,✔️,4.76,True,GPU +7,🧠⚡,Foundation Model,[Mitra (default) [4.76% IMPUTED]](https://arxiv.org/abs/2510.21204),1417.0,+139/-66,10.924,0.388,17.4,6.94,145.41,3.958,✔️,4.76,True,GPU +8,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1412.0,+107/-82,10.708,0.425,17.69,5.99,4.14,0.557,➖,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1401.0,+111/-76,12.422,0.365,18.36,7.57,2791.97,0.544,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1382.0,+100/-80,9.297,0.382,19.48,6.28,6616.87,408.881,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabICL (default) [4.76% IMPUTED]](https://arxiv.org/abs/2502.05564),1378.0,+81/-87,10.231,0.353,19.74,6.25,7.68,1.811,✔️,4.76,True,GPU +12,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1363.0,+114/-83,14.798,0.314,20.67,12.04,298.33,2.036,➖,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1362.0,+102/-68,13.132,0.315,20.74,10.49,3275.74,0.512,✔️,4.76,True,GPU +14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1361.0,+104/-87,12.578,0.334,20.76,9.2,2237.77,1.273,✔️,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1351.0,+96/-92,10.868,0.339,21.43,7.72,6616.87,39.713,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1332.0,+66/-54,14.378,0.219,22.64,17.52,512.11,1.17,✔️,0.0,False,CPU +17,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1310.0,+81/-60,14.616,0.216,24.02,15.97,4.51,0.124,✔️,0.0,False,CPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1305.0,+138/-85,13.216,0.279,24.36,9.9,4618.5,0.426,✔️,0.0,False,GPU +19,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1296.0,+92/-85,15.188,0.196,24.95,14.11,1640.58,0.552,✔️,0.0,False,CPU +20,🧠⚡,Foundation Model,[TabPFNv2 (default) [4.76% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1294.0,+109/-93,15.054,0.28,25.14,7.87,4.16,0.543,✔️,4.76,True,GPU +21,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1291.0,+114/-102,13.922,0.256,25.31,14.98,2237.77,0.147,✔️,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1290.0,+88/-79,15.671,0.216,25.4,13.7,700.96,0.743,✔️,0.0,False,CPU +23,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1287.0,+80/-61,14.488,0.185,25.57,14.74,11.09,2.46,✔️,0.0,False,GPU +24,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1282.0,+89/-84,14.814,0.194,25.93,16.64,1103.84,0.559,✔️,0.0,False,CPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1274.0,+82/-84,15.114,0.208,26.43,18.06,536.23,1.738,✔️,0.0,False,GPU +26,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1265.0,+79/-77,16.075,0.186,27.05,10.35,700.96,0.203,✔️,0.0,False,CPU +27,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1261.0,+77/-68,15.562,0.147,27.29,11.48,1640.58,0.059,✔️,0.0,False,CPU +28,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1259.0,+68/-70,15.646,0.173,27.43,23.57,512.11,0.122,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1257.0,+114/-91,15.505,0.233,27.55,16.83,4618.5,9.218,✔️,0.0,False,GPU +30,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1253.0,+85/-82,15.521,0.168,27.81,21.05,1103.84,0.051,✔️,0.0,False,CPU +31,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1250.0,+90/-86,16.455,0.162,28.0,10.87,7.16,0.046,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1246.0,+98/-87,16.449,0.219,28.29,17.5,11.3,0.149,✔️,0.0,False,GPU +33,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1245.0,+106/-102,13.544,0.239,28.33,8.98,52.04,39.82,✔️,0.0,False,GPU +34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1230.0,+98/-88,15.554,0.175,29.33,20.98,2535.0,2.546,✔️,0.0,False,CPU +35,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1221.0,+86/-94,16.131,0.137,29.98,21.67,536.23,0.097,✔️,0.0,False,GPU +36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1187.0,+108/-107,18.644,0.133,32.21,10.89,13.39,0.329,✔️,0.0,False,GPU +37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1180.0,+101/-110,18.831,0.147,32.64,20.89,650.63,5.616,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1170.0,+94/-93,17.459,0.146,33.33,25.03,2535.0,0.112,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1139.0,+87/-99,20.279,0.096,35.26,27.67,206.04,0.946,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1134.0,+104/-151,20.059,0.124,35.62,20.56,650.63,0.442,✔️,0.0,False,CPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1125.0,+83/-84,19.149,0.081,36.14,29.19,1.98,0.122,✔️,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1118.0,+108/-111,21.039,0.114,36.62,18.51,206.04,0.098,✔️,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1098.0,+74/-90,19.563,0.078,37.79,32.21,2.47,0.168,✔️,0.0,False,CPU +44,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1097.0,+93/-108,21.728,0.079,37.86,29.32,494.48,0.936,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1061.0,+86/-101,22.476,0.028,40.0,34.61,494.48,0.093,✔️,0.0,False,CPU +46,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1051.0,+115/-148,23.895,0.085,40.52,29.4,1.29,0.149,➖,0.0,False,GPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1010.0,+91/-135,23.809,0.044,42.76,35.3,7.76,0.147,✔️,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+105/-148,25.296,0.034,43.29,29.07,0.39,0.041,✔️,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),976.0,+86/-127,23.191,0.029,44.48,40.11,3.48,0.427,✔️,0.0,False,CPU +50,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),947.0,+100/-143,25.646,0.041,45.86,40.9,2.66,0.702,✔️,0.0,False,GPU +51,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),947.0,+119/-146,26.793,0.023,45.86,40.57,0.33,0.044,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),944.0,+72/-114,26.237,0.0,45.95,44.39,89.61,1.627,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),926.0,+112/-152,29.429,0.048,46.79,14.84,280.42,0.382,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),900.0,+118/-175,31.785,0.018,47.86,42.54,1.48,0.127,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),895.0,+118/-156,30.058,0.04,48.05,23.27,280.42,0.072,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),793.0,+89/-139,31.804,0.0,51.52,50.65,89.61,0.08,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),608.0,+105/-184,43.082,0.0,55.52,55.12,0.24,0.02,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/winrate_matrix.png.zip index 115d16fdb282c66d79bdbe87e9e97c200ca1a79c..90aab53ce8fb826e16c54c297db7489c4ac92ec4 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5ef3f834103fbcaaca88f44b2c8937811053039c448ccccf07ef905a7f1e56a1 +oid sha256:baab5a61999f6d4b54e898f99fd248358ac7582727315823ff037b02103e8622 size 2224691 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index c5f786ed8931af32da1b1ceefd9a0f40cccfd79e..14023fc6c79e9904baec3842d26277a69dca93fd 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:54f2d456c5e55daf68dbb4d06fe05b21738409be336bff93efdc067ef59b02b9 +oid sha256:5d6ce6e272b4e836c71bb78b1cb5d125a165250ec2a62b167f214804abd0f95e size 495281 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip index 4f31edf0b96f16fb5caf66659deaa8395ab34e9e..5c198406cb7f71e78f2411cb8c463e2881bb3d81 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:69e553175190835e8dbe1afecee9bf7afc9a9cb75a17a5cf4476e9e22333ca64 -size 1035456 +oid sha256:246bf21b0ba13194a86dd66f40b576c97a7c436f97ece80ba84569d5ee70c022 +size 1020703 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip index 5294175492e4154e77b8ce6df9d901b36825dcae..f90f7802bdaf9696c349c281e97ec6b3afd15d90 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fa24b7ecdb3434e6e7af68075c969e22d0f6891e4ce5b323cdc3cd862e13c644 +oid sha256:21075d9a3c478af30efd8f013140ea58d94236bd6cff4b50b01e0895ba12e1b2 size 118622 diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/website_leaderboard.csv index c424e67f04943ef0c170f2786ba8b9c0ddad8d85..28185c72fd447069c3a0aa2fa1086235606fd735 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1644.0,+93/-60,0.637,7.8,5.36,6.865,5.25,0.517,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1567.0,+153/-98,0.586,10.65,4.0,5.801,2092.56,8.655,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1556.0,+140/-99,0.575,11.12,4.48,6.297,2092.56,0.867,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1493.0,+130/-78,0.506,14.1,5.36,8.203,598.6,8.421,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1492.0,+91/-82,0.441,14.2,8.25,10.934,2773.9,21.391,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1476.0,+124/-90,0.456,15.02,7.12,10.58,3361.32,65.579,0.0,False,GPU -6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1471.0,+85/-74,0.413,15.3,10.76,10.37,1881.65,1.26,0.0,False,CPU -7,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1443.0,+115/-72,0.407,16.82,6.69,10.913,145.75,4.071,0.0,False,GPU -8,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1426.0,+123/-84,0.419,17.77,5.81,10.972,4.09,0.544,0.0,False,GPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1420.0,+120/-99,0.371,18.18,7.33,12.618,2773.9,0.534,0.0,False,GPU -10,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1402.0,+115/-88,0.33,19.25,11.6,14.276,292.15,1.882,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1401.0,+118/-76,0.371,19.27,6.01,10.185,8.05,1.864,0.0,False,GPU -12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1399.0,+132/-82,0.38,19.4,6.06,9.42,6408.95,397.026,0.0,False,GPU -13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1389.0,+117/-88,0.351,20.05,8.88,12.618,2217.3,1.258,0.0,False,GPU -14,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1384.0,+124/-88,0.331,20.32,10.16,13.232,3361.32,0.527,0.0,False,GPU -15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1368.0,+131/-93,0.347,21.3,7.46,10.945,6408.95,39.606,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1334.0,+58/-48,0.186,23.52,20.03,15.024,457.0,1.064,0.0,False,CPU -17,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1333.0,+110/-76,0.293,23.58,9.54,13.112,4442.23,0.405,0.0,False,GPU -18,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1320.0,+84/-79,0.206,24.4,13.69,15.346,1618.79,0.502,0.0,False,CPU -19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1315.0,+112/-89,0.269,24.72,14.55,13.982,2217.3,0.137,0.0,False,GPU -20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1315.0,+77/-60,0.186,24.72,16.46,15.224,4.49,0.115,0.0,False,CPU -21,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1314.0,+80/-69,0.194,24.8,14.28,14.446,10.77,2.16,0.0,False,GPU -22,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1312.0,+104/-98,0.295,24.95,7.59,15.25,3.89,0.544,0.0,False,GPU -23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1287.0,+83/-75,0.178,26.58,19.37,16.445,673.88,0.681,0.0,False,CPU -24,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1284.0,+85/-81,0.162,26.78,17.38,15.452,1103.74,0.549,0.0,False,CPU -25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1281.0,+88/-92,0.186,27.0,18.24,15.662,516.54,1.513,0.0,False,GPU -26,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1278.0,+83/-91,0.17,27.2,10.48,16.268,6.77,0.046,0.0,False,CPU -27,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1276.0,+99/-108,0.245,27.28,16.43,15.707,4442.23,8.478,0.0,False,GPU -28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1275.0,+83/-79,0.154,27.4,11.18,15.789,1618.79,0.055,0.0,False,CPU -29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1272.0,+120/-112,0.23,27.55,17.0,16.267,10.18,0.149,0.0,False,GPU -30,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1268.0,+136/-120,0.251,27.85,8.65,13.464,51.78,39.866,0.0,False,GPU -31,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1264.0,+74/-64,0.146,28.1,24.41,16.249,457.0,0.111,0.0,False,CPU -32,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1262.0,+76/-68,0.15,28.2,11.24,16.819,673.88,0.177,0.0,False,CPU -33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1255.0,+98/-87,0.183,28.7,20.47,15.396,2533.55,2.301,0.0,False,CPU -34,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1252.0,+87/-54,0.133,28.9,24.07,16.217,1103.74,0.05,0.0,False,CPU -35,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1227.0,+80/-86,0.12,30.58,21.9,16.637,516.54,0.086,0.0,False,GPU -36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1212.0,+84/-118,0.139,31.52,10.49,18.467,11.78,0.312,0.0,False,GPU -37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1204.0,+96/-119,0.154,32.02,20.34,18.721,622.79,5.389,0.0,False,CPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1196.0,+97/-94,0.153,32.6,24.44,17.128,2533.55,0.11,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1157.0,+98/-138,0.13,35.05,20.0,19.935,622.79,0.398,0.0,False,CPU -40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1141.0,+94/-87,0.076,36.08,28.32,20.989,204.47,0.853,0.0,False,CPU -41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1121.0,+117/-112,0.109,37.3,18.33,21.645,204.47,0.091,0.0,False,CPU -42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1117.0,+81/-93,0.043,37.55,33.65,20.016,1.88,0.121,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1094.0,+98/-125,0.057,38.9,30.42,22.532,448.77,0.841,0.0,False,CPU -44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1091.0,+80/-100,0.043,39.08,35.18,20.402,2.39,0.159,0.0,False,CPU -45,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1078.0,+103/-162,0.089,39.85,28.74,23.197,1.17,0.148,0.0,False,GPU -46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1065.0,+93/-110,0.03,40.55,34.95,23.044,448.77,0.092,0.0,False,CPU -47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1033.0,+109/-126,0.046,42.3,34.74,23.554,7.36,0.138,0.0,False,CPU -48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+98/-164,0.035,44.0,29.07,26.004,0.37,0.04,0.0,False,CPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,997.0,+107/-123,0.031,44.15,39.68,22.94,3.3,0.395,0.0,False,CPU -50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,963.0,+65/-111,0.0,45.75,44.14,26.281,89.46,1.579,0.0,False,CPU -51,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,953.0,+121/-156,0.044,46.2,41.0,26.17,2.66,0.553,0.0,False,GPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,949.0,+114/-129,0.05,46.38,14.32,29.0,276.0,0.368,0.0,False,CPU -53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,948.0,+121/-161,0.024,46.45,40.96,27.548,0.29,0.041,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,926.0,+106/-145,0.019,47.35,41.98,31.115,1.45,0.123,0.0,False,CPU -55,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,920.0,+122/-145,0.042,47.6,22.6,29.565,276.0,0.065,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,810.0,+84/-146,0.0,51.45,50.54,31.853,89.46,0.075,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,624.0,+110/-269,0.0,55.5,55.08,43.289,0.22,0.02,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1644.0,+93/-60,6.865,0.637,7.8,5.36,5.25,0.517,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1567.0,+153/-98,5.801,0.586,10.65,4.0,2092.56,8.655,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1556.0,+140/-99,6.297,0.575,11.12,4.48,2092.56,0.867,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1493.0,+130/-78,8.203,0.506,14.1,5.36,598.6,8.421,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1492.0,+91/-82,10.934,0.441,14.2,8.25,2773.9,21.391,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1476.0,+124/-90,10.58,0.456,15.02,7.12,3361.32,65.579,✔️,0.0,False,GPU +6,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1471.0,+85/-74,10.37,0.413,15.3,10.76,1881.65,1.26,✔️,0.0,False,CPU +7,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1443.0,+115/-72,10.913,0.407,16.82,6.69,145.75,4.071,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1426.0,+123/-84,10.972,0.419,17.77,5.81,4.09,0.544,➖,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1420.0,+120/-99,12.618,0.371,18.18,7.33,2773.9,0.534,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1402.0,+115/-88,14.276,0.33,19.25,11.6,292.15,1.882,➖,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1401.0,+118/-76,10.185,0.371,19.27,6.01,8.05,1.864,✔️,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1399.0,+132/-82,9.42,0.38,19.4,6.06,6408.95,397.026,✔️,0.0,False,GPU +13,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1389.0,+117/-88,12.618,0.351,20.05,8.88,2217.3,1.258,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1384.0,+124/-88,13.232,0.331,20.32,10.16,3361.32,0.527,✔️,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1368.0,+131/-93,10.945,0.347,21.3,7.46,6408.95,39.606,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1334.0,+58/-48,15.024,0.186,23.52,20.03,457.0,1.064,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1333.0,+110/-76,13.112,0.293,23.58,9.54,4442.23,0.405,✔️,0.0,False,GPU +18,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1320.0,+84/-79,15.346,0.206,24.4,13.69,1618.79,0.502,✔️,0.0,False,CPU +19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1315.0,+112/-89,13.982,0.269,24.72,14.55,2217.3,0.137,✔️,0.0,False,GPU +20,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1315.0,+77/-60,15.224,0.186,24.72,16.46,4.49,0.115,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1314.0,+80/-69,14.446,0.194,24.8,14.28,10.77,2.16,✔️,0.0,False,GPU +22,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1312.0,+104/-98,15.25,0.295,24.95,7.59,3.89,0.544,✔️,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1287.0,+83/-75,16.445,0.178,26.58,19.37,673.88,0.681,✔️,0.0,False,CPU +24,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1284.0,+85/-81,15.452,0.162,26.78,17.38,1103.74,0.549,✔️,0.0,False,CPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1281.0,+88/-92,15.662,0.186,27.0,18.24,516.54,1.513,✔️,0.0,False,GPU +26,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1278.0,+83/-91,16.268,0.17,27.2,10.48,6.77,0.046,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1276.0,+99/-108,15.707,0.245,27.28,16.43,4442.23,8.478,✔️,0.0,False,GPU +28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1275.0,+83/-79,15.789,0.154,27.4,11.18,1618.79,0.055,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1272.0,+120/-112,16.267,0.23,27.55,17.0,10.18,0.149,✔️,0.0,False,GPU +30,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1268.0,+136/-120,13.464,0.251,27.85,8.65,51.78,39.866,✔️,0.0,False,GPU +31,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1264.0,+74/-64,16.249,0.146,28.1,24.41,457.0,0.111,✔️,0.0,False,CPU +32,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1262.0,+76/-68,16.819,0.15,28.2,11.24,673.88,0.177,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1255.0,+98/-87,15.396,0.183,28.7,20.47,2533.55,2.301,✔️,0.0,False,CPU +34,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1252.0,+87/-54,16.217,0.133,28.9,24.07,1103.74,0.05,✔️,0.0,False,CPU +35,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1227.0,+80/-86,16.637,0.12,30.58,21.9,516.54,0.086,✔️,0.0,False,GPU +36,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1212.0,+84/-118,18.467,0.139,31.52,10.49,11.78,0.312,✔️,0.0,False,GPU +37,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1204.0,+96/-119,18.721,0.154,32.02,20.34,622.79,5.389,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1196.0,+97/-94,17.128,0.153,32.6,24.44,2533.55,0.11,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1157.0,+98/-138,19.935,0.13,35.05,20.0,622.79,0.398,✔️,0.0,False,CPU +40,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1141.0,+94/-87,20.989,0.076,36.08,28.32,204.47,0.853,✔️,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1121.0,+117/-112,21.645,0.109,37.3,18.33,204.47,0.091,✔️,0.0,False,CPU +42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1117.0,+81/-93,20.016,0.043,37.55,33.65,1.88,0.121,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1094.0,+98/-125,22.532,0.057,38.9,30.42,448.77,0.841,✔️,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1091.0,+80/-100,20.402,0.043,39.08,35.18,2.39,0.159,✔️,0.0,False,CPU +45,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1078.0,+103/-162,23.197,0.089,39.85,28.74,1.17,0.148,➖,0.0,False,GPU +46,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1065.0,+93/-110,23.044,0.03,40.55,34.95,448.77,0.092,✔️,0.0,False,CPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1033.0,+109/-126,23.554,0.046,42.3,34.74,7.36,0.138,✔️,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+98/-164,26.004,0.035,44.0,29.07,0.37,0.04,✔️,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),997.0,+107/-123,22.94,0.031,44.15,39.68,3.3,0.395,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),963.0,+65/-111,26.281,0.0,45.75,44.14,89.46,1.579,✔️,0.0,False,CPU +51,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),953.0,+121/-156,26.17,0.044,46.2,41.0,2.66,0.553,✔️,0.0,False,GPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),949.0,+114/-129,29.0,0.05,46.38,14.32,276.0,0.368,✔️,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),948.0,+121/-161,27.548,0.024,46.45,40.96,0.29,0.041,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),926.0,+106/-145,31.115,0.019,47.35,41.98,1.45,0.123,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),920.0,+122/-145,29.565,0.042,47.6,22.6,276.0,0.065,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),810.0,+84/-146,31.853,0.0,51.45,50.54,89.46,0.075,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),624.0,+110/-269,43.289,0.0,55.5,55.08,0.22,0.02,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip index 711594a5a08091f45d1ddde524750d80cc59f30f..a38c97493472f144f3cac754be7a9530f1853b29 100644 --- a/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_binary/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9348f5b45a35aff4ebe40e59f58cbe2981206a9e9cd51cb980f4a2a0897d1ed8 +oid sha256:52a0c714e4310e9214f6cd09f32813abb9f538deaa4b26a1dbb368b8cab31d36 size 2250455 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index aa766d692d52efca9b981957412542b8650dd1cc..258f800e1f5f6a56785fe5b110269007e06da238 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2b130297b67c3e5f46ba1c4098b5ed92eac7adb86888b248ace20c3e67a672a2 +oid sha256:57f1be2af2aaddc57542aa565f43865bd4981375bd1b23692d849a1057bc5055 size 488004 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip index 9d2bd597eeb265e325bda157308cb1e54fedb008..7f51178b43ecb249d45467145bc7c512a0128ec3 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fdd380009c6b6b95b7e3ecb69692e40263e270bdb21072b7d9e7598e08c64c3d -size 1059023 +oid sha256:61e44cde308ff17f8f809983b69a11d2b30091b184fc7fb3e76aecee5da8bed4 +size 1021494 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/tuning-impact-elo.png.zip index e9d665c49a3f6edb4f26654db34e9fd4cf821918..feee90328ae54403ff2c48e944aa2468ca09da65 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3c669cc744b65e418aa7d2c48d99bbad906d3680ad754ac08d807cc8a049a4e9 +oid sha256:556e7d4f401c6d869bacb8535295606afb961d8936bd3cd5cec58f8d4eec6a64 size 175033 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/website_leaderboard.csv index 852127de0d77c8b9725f0ada6d968b728cc86864..ea7ee08add81b5e52ede347a0b3eb4f1459328a9 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1590.0,+102/-84,0.632,9.61,2.92,4.83,2063.62,9.938,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1577.0,+90/-78,0.643,10.13,3.95,7.182,521.51,7.239,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1555.0,+94/-77,0.577,11.08,5.09,7.35,2063.62,1.087,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1537.0,+80/-70,0.559,11.88,6.08,7.714,5.62,0.771,0.0,False,GPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1525.0,+94/-61,0.552,12.42,4.74,8.832,1815.7,1.456,0.0,False,CPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1480.0,+56/-49,0.461,14.68,7.76,10.513,2773.9,16.363,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1415.0,+53/-51,0.381,18.38,8.94,12.166,2773.9,0.39,0.0,False,GPU -7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1404.0,+75/-60,0.369,19.08,8.25,12.641,2461.81,1.283,0.0,False,GPU -8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1389.0,+56/-44,0.297,19.96,14.18,13.379,391.23,1.243,0.0,False,CPU -9,🧠🔁,Neural Network,[LimiX (default) [15.79% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1384.0,+73/-74,0.384,20.29,6.54,12.947,3.46,0.517,15.79,True,GPU -10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1373.0,+95/-58,0.347,21.0,8.39,11.63,6067.13,404.225,0.0,False,GPU -11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1369.0,+72/-56,0.314,21.22,12.73,12.907,1326.58,0.531,0.0,False,CPU -12,🧠⚡,Foundation Model,[TabICL (default) [5.26% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1367.0,+79/-72,0.332,21.37,7.5,12.807,9.07,2.008,5.26,True,GPU -13,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1360.0,+69/-72,0.284,21.83,12.73,13.973,682.62,0.971,0.0,False,CPU -14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1350.0,+64/-68,0.3,22.47,11.37,13.402,4853.07,0.473,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1347.0,+66/-56,0.289,22.67,14.9,13.432,1326.58,0.051,0.0,False,CPU -16,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1346.0,+50/-54,0.272,22.74,11.73,13.55,5.74,0.107,0.0,False,CPU -17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1344.0,+83/-62,0.307,22.86,12.22,13.609,2461.81,0.138,0.0,False,GPU -18,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1342.0,+93/-83,0.326,22.97,9.57,13.448,4853.07,9.386,0.0,False,GPU -19,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1334.0,+93/-80,0.334,23.51,8.96,15.007,2914.83,17.597,31.58,True,GPU -20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1333.0,+64/-47,0.249,23.58,17.69,14.32,391.23,0.186,0.0,False,CPU -21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1331.0,+63/-69,0.256,23.68,11.74,14.222,682.62,0.183,0.0,False,CPU -22,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1323.0,+75/-74,0.285,24.22,8.48,13.66,6067.13,39.565,0.0,False,GPU -23,🧠🔁,Neural Network,[Mitra (default) [31.58% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1316.0,+86/-83,0.295,24.66,9.93,15.339,118.41,2.667,31.58,True,GPU -24,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1312.0,+66/-58,0.256,24.92,14.82,15.997,250.94,2.036,0.0,False,GPU -25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1310.0,+69/-58,0.239,25.11,14.61,14.71,1053.72,2.067,0.0,False,GPU -26,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1303.0,+63/-51,0.207,25.53,15.73,15.784,2399.23,0.558,0.0,False,CPU -27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1282.0,+72/-66,0.237,26.95,17.43,16.264,10.31,0.136,0.0,False,GPU -28,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1273.0,+76/-76,0.233,27.54,13.68,16.721,2914.83,0.271,31.58,True,GPU -29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1264.0,+54/-51,0.169,28.13,21.55,15.515,2480.32,2.175,0.0,False,CPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1260.0,+67/-50,0.152,28.42,18.64,16.137,10.77,1.787,0.0,False,GPU -31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1258.0,+70/-56,0.166,28.59,13.64,16.368,1053.72,0.105,0.0,False,GPU -32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1249.0,+66/-51,0.146,29.18,15.17,16.707,2399.23,0.056,0.0,False,CPU -33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1242.0,+59/-54,0.147,29.63,14.98,17.609,6.77,0.052,0.0,False,CPU -34,🧠⚡,Foundation Model,[TabPFNv2 (default) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1239.0,+82/-80,0.216,29.83,10.67,17.473,3.3,0.305,31.58,True,GPU -35,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1212.0,+81/-70,0.186,31.62,11.33,16.869,47.89,39.446,0.0,False,GPU -36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1210.0,+68/-91,0.145,31.75,21.46,18.74,622.79,5.037,0.0,False,CPU -37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1194.0,+64/-63,0.114,32.82,23.16,18.948,197.16,0.721,0.0,False,CPU -38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1193.0,+65/-71,0.126,32.84,20.53,17.316,1.88,0.121,0.0,False,CPU -39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1192.0,+69/-62,0.13,32.89,24.82,17.8,2480.32,0.117,0.0,False,CPU -40,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1170.0,+65/-60,0.092,34.36,15.73,19.393,13.35,0.353,0.0,False,GPU -41,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1167.0,+67/-75,0.104,34.53,24.3,19.874,327.31,0.708,0.0,False,CPU -42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1155.0,+66/-76,0.113,35.3,19.39,20.247,197.16,0.083,0.0,False,CPU -43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1142.0,+57/-62,0.092,36.12,30.52,18.628,2.25,0.159,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1127.0,+68/-78,0.093,37.03,24.79,20.748,622.79,0.346,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1124.0,+58/-60,0.056,37.2,30.05,20.709,327.31,0.085,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1030.0,+62/-79,0.028,42.55,37.41,23.445,5.73,0.148,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+61/-86,0.019,44.07,34.16,26.787,0.37,0.038,0.0,False,CPU -48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,995.0,+74/-102,0.032,44.29,38.03,27.556,140.66,2.876,0.0,False,CPU -49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,986.0,+72/-69,0.029,44.76,40.61,24.653,3.12,0.334,0.0,False,CPU -50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,972.0,+82/-138,0.047,45.39,36.21,29.787,1.04,0.192,0.0,False,GPU -51,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,966.0,+90/-91,0.042,45.66,39.33,27.581,3.48,0.677,0.0,False,GPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,961.0,+83/-99,0.032,45.88,20.78,31.022,256.15,0.36,0.0,False,CPU -53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,932.0,+89/-110,0.023,47.13,29.33,31.745,256.15,0.07,0.0,False,CPU -54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,917.0,+77/-108,0.014,47.74,43.73,29.31,0.24,0.037,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,872.0,+91/-108,0.01,49.42,45.92,34.338,1.32,0.117,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,819.0,+93/-118,0.015,51.11,48.48,35.052,140.66,0.188,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,540.0,+117/-145,0.0,56.42,56.17,49.668,0.18,0.028,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1590.0,+102/-84,4.83,0.632,9.61,2.92,2063.62,9.938,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1577.0,+90/-78,7.182,0.643,10.13,3.95,521.51,7.239,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1555.0,+94/-77,7.35,0.577,11.08,5.09,2063.62,1.087,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1537.0,+80/-70,7.714,0.559,11.88,6.08,5.62,0.771,✔️,0.0,False,GPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1525.0,+94/-61,8.832,0.552,12.42,4.74,1815.7,1.456,✔️,0.0,False,CPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1480.0,+56/-49,10.513,0.461,14.68,7.76,2773.9,16.363,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1415.0,+53/-51,12.166,0.381,18.38,8.94,2773.9,0.39,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1404.0,+75/-60,12.641,0.369,19.08,8.25,2461.81,1.283,✔️,0.0,False,GPU +8,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1389.0,+56/-44,13.379,0.297,19.96,14.18,391.23,1.243,✔️,0.0,False,CPU +9,🧠⚡,Foundation Model,[LimiX (default) [15.79% IMPUTED]](https://arxiv.org/abs/2509.03505),1384.0,+73/-74,12.947,0.384,20.29,6.54,3.46,0.517,➖,15.79,True,GPU +10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1373.0,+95/-58,11.63,0.347,21.0,8.39,6067.13,404.225,✔️,0.0,False,GPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1369.0,+72/-56,12.907,0.314,21.22,12.73,1326.58,0.531,✔️,0.0,False,CPU +12,🧠⚡,Foundation Model,[TabICL (default) [5.26% IMPUTED]](https://arxiv.org/abs/2502.05564),1367.0,+79/-72,12.807,0.332,21.37,7.5,9.07,2.008,✔️,5.26,True,GPU +13,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1360.0,+69/-72,13.973,0.284,21.83,12.73,682.62,0.971,✔️,0.0,False,CPU +14,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1350.0,+64/-68,13.402,0.3,22.47,11.37,4853.07,0.473,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1347.0,+66/-56,13.432,0.289,22.67,14.9,1326.58,0.051,✔️,0.0,False,CPU +16,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1346.0,+50/-54,13.55,0.272,22.74,11.73,5.74,0.107,✔️,0.0,False,CPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1344.0,+83/-62,13.609,0.307,22.86,12.22,2461.81,0.138,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1342.0,+93/-83,13.448,0.326,22.97,9.57,4853.07,9.386,✔️,0.0,False,GPU +19,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1334.0,+93/-80,15.007,0.334,23.51,8.96,2914.83,17.597,✔️,31.58,True,GPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1333.0,+64/-47,14.32,0.249,23.58,17.69,391.23,0.186,✔️,0.0,False,CPU +21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1331.0,+63/-69,14.222,0.256,23.68,11.74,682.62,0.183,✔️,0.0,False,CPU +22,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1323.0,+75/-74,13.66,0.285,24.22,8.48,6067.13,39.565,✔️,0.0,False,GPU +23,🧠⚡,Foundation Model,[Mitra (default) [31.58% IMPUTED]](https://arxiv.org/abs/2510.21204),1316.0,+86/-83,15.339,0.295,24.66,9.93,118.41,2.667,✔️,31.58,True,GPU +24,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1312.0,+66/-58,15.997,0.256,24.92,14.82,250.94,2.036,➖,0.0,False,GPU +25,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1310.0,+69/-58,14.71,0.239,25.11,14.61,1053.72,2.067,✔️,0.0,False,GPU +26,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1303.0,+63/-51,15.784,0.207,25.53,15.73,2399.23,0.558,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1282.0,+72/-66,16.264,0.237,26.95,17.43,10.31,0.136,✔️,0.0,False,GPU +28,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1273.0,+76/-76,16.721,0.233,27.54,13.68,2914.83,0.271,✔️,31.58,True,GPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1264.0,+54/-51,15.515,0.169,28.13,21.55,2480.32,2.175,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1260.0,+67/-50,16.137,0.152,28.42,18.64,10.77,1.787,✔️,0.0,False,GPU +31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1258.0,+70/-56,16.368,0.166,28.59,13.64,1053.72,0.105,✔️,0.0,False,GPU +32,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1249.0,+66/-51,16.707,0.146,29.18,15.17,2399.23,0.056,✔️,0.0,False,CPU +33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1242.0,+59/-54,17.609,0.147,29.63,14.98,6.77,0.052,✔️,0.0,False,CPU +34,🧠⚡,Foundation Model,[TabPFNv2 (default) [31.58% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1239.0,+82/-80,17.473,0.216,29.83,10.67,3.3,0.305,✔️,31.58,True,GPU +35,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1212.0,+81/-70,16.869,0.186,31.62,11.33,47.89,39.446,✔️,0.0,False,GPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1210.0,+68/-91,18.74,0.145,31.75,21.46,622.79,5.037,✔️,0.0,False,CPU +37,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1194.0,+64/-63,18.948,0.114,32.82,23.16,197.16,0.721,✔️,0.0,False,CPU +38,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1193.0,+65/-71,17.316,0.126,32.84,20.53,1.88,0.121,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1192.0,+69/-62,17.8,0.13,32.89,24.82,2480.32,0.117,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1170.0,+65/-60,19.393,0.092,34.36,15.73,13.35,0.353,✔️,0.0,False,GPU +41,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1167.0,+67/-75,19.874,0.104,34.53,24.3,327.31,0.708,✔️,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1155.0,+66/-76,20.247,0.113,35.3,19.39,197.16,0.083,✔️,0.0,False,CPU +43,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1142.0,+57/-62,18.628,0.092,36.12,30.52,2.25,0.159,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1127.0,+68/-78,20.748,0.093,37.03,24.79,622.79,0.346,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1124.0,+58/-60,20.709,0.056,37.2,30.05,327.31,0.085,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1030.0,+62/-79,23.445,0.028,42.55,37.41,5.73,0.148,✔️,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+61/-86,26.787,0.019,44.07,34.16,0.37,0.038,✔️,0.0,False,CPU +48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),995.0,+74/-102,27.556,0.032,44.29,38.03,140.66,2.876,✔️,0.0,False,CPU +49,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),986.0,+72/-69,24.653,0.029,44.76,40.61,3.12,0.334,✔️,0.0,False,CPU +50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),972.0,+82/-138,29.787,0.047,45.39,36.21,1.04,0.192,➖,0.0,False,GPU +51,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),966.0,+90/-91,27.581,0.042,45.66,39.33,3.48,0.677,✔️,0.0,False,GPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),961.0,+83/-99,31.022,0.032,45.88,20.78,256.15,0.36,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),932.0,+89/-110,31.745,0.023,47.13,29.33,256.15,0.07,✔️,0.0,False,CPU +54,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),917.0,+77/-108,29.31,0.014,47.74,43.73,0.24,0.037,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),872.0,+91/-108,34.338,0.01,49.42,45.92,1.32,0.117,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),819.0,+93/-118,35.052,0.015,51.11,48.48,140.66,0.188,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),540.0,+117/-145,49.668,0.0,56.42,56.17,0.18,0.028,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/winrate_matrix.png.zip index 75bfb93719a7bb3d8ede5fd6f77df6381a08aea6..c0c23bef329c91b1b2db85cdc6219b478b292983 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6eaeef65c81a4ffb8dcdaa9f12c8756cc18139f7e288c01c13699f3895c073a1 +oid sha256:a413a48103ee85da34baa22e24b81a4cb173674a3ff108e90319ba96ee6e04e7 size 2410880 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index 9a1a6bf053b00b49ca2112e6602b214f987e795d..bc2541c40f01f960ac2694aad9401898acdb23dd 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3b8b5c0fa95416cd837832e3dd764fcb30899666cfa80da3612e60fd8bb5524f +oid sha256:8dce2ac911711eccf79ff9be6354e33cff14b3f36ad02eb84c71e63a9b317b22 size 444464 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip index 082a11eaa1c273be69588fb7ac5ba21f62cafbff..537ed23db875cf43de1dcc0fff06f3fc8593459e 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:35877a053515c3156deffd9b9d92b1bfedff657eb1d515c8161ea82c50e6faf4 -size 1021051 +oid sha256:dcc442e3aafecb3f06c824ca97efb971aa165a3590b58f606e023e22b84da031 +size 975039 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/tuning-impact-elo.png.zip index 543421cebe4e5c3072378c19e85320258f045e4e..eaf8c900a6bc89a8ea181c7a33fd34c063f74b5e 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a7e3b95acfd3931bbbf4b71cc2b64849dde90be9d0968a7bd4e071d999bef443 +oid sha256:3696de77e0a739df957caaf9b09aab43f3de8881a09c37fab94769f256ae5b10 size 133309 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/website_leaderboard.csv index ec5b948ba88c15e72207926674febfe2839aeec6..3c9a8357f37c04c30f185c15b3568180dbb5e0c0 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/website_leaderboard.csv @@ -1,55 +1,55 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,2023.0,+176/-73,0.867,3.3,2.15,1.576,307.34,2.186,0.0,False,CPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1959.0,+218/-71,0.842,4.2,2.28,2.126,266.49,2.963,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1712.0,+263/-137,0.575,10.4,3.14,5.306,780.95,11.64,0.0,False,GPU -3,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1684.0,+240/-162,0.601,11.4,4.17,5.366,2335.82,14.145,0.0,False,GPU -4,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1661.0,+156/-106,0.467,12.3,8.67,7.774,202.53,0.43,0.0,False,CPU -5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1654.0,+240/-119,0.495,12.6,6.99,7.407,780.95,1.493,0.0,False,GPU -6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1646.0,+177/-92,0.519,12.9,7.92,7.059,688.25,0.192,0.0,False,CPU -7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1632.0,+173/-79,0.445,13.5,9.79,8.682,91.02,0.568,0.0,False,CPU -8,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1626.0,+168/-86,0.494,13.75,8.83,7.132,688.25,0.036,0.0,False,CPU -9,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1615.0,+129/-89,0.466,14.2,9.63,7.485,1635.73,2.146,0.0,False,GPU -10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1592.0,+183/-102,0.43,15.2,5.6,8.859,1400.65,0.571,0.0,False,GPU -11,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1590.0,+170/-111,0.417,15.3,11.13,9.79,2335.82,0.502,0.0,False,GPU -12,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1589.0,+136/-69,0.459,15.35,6.0,7.247,2.2,0.044,0.0,False,CPU -13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1576.0,+176/-103,0.414,15.9,11.12,8.091,202.53,0.055,0.0,False,CPU -14,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1557.0,+174/-105,0.352,16.8,13.01,9.535,91.02,0.074,0.0,False,CPU -15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1525.0,+159/-111,0.355,18.3,10.13,9.894,1635.73,0.082,0.0,False,GPU -16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1511.0,+196/-166,0.356,19.0,9.01,9.451,1400.65,0.065,0.0,False,GPU -17,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1502.0,+191/-126,0.337,19.4,9.54,8.488,2.3,0.818,0.0,False,GPU -18,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1490.0,+167/-135,0.292,20.0,12.21,10.137,8.46,3.879,0.0,False,GPU -19,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1439.0,+205/-127,0.253,22.6,15.64,12.923,899.24,0.221,0.0,False,CPU -20,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1435.0,+228/-155,0.285,22.8,10.66,10.324,0.47,0.047,0.0,False,CPU -21,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1412.0,+111/-69,0.164,24.0,19.83,11.033,1128.73,1.573,0.0,False,CPU -22,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1400.0,+180/-172,0.224,24.6,19.84,12.728,5.79,0.084,0.0,False,GPU -23,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1400.0,+137/-131,0.176,24.6,19.22,11.114,4546.1,406.11,0.0,False,GPU -24,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1393.0,+156/-139,0.223,25.0,16.3,11.586,713.0,2.317,0.0,False,GPU -25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1364.0,+159/-138,0.15,26.5,18.58,14.611,713.0,0.155,0.0,False,GPU -26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1340.0,+93/-103,0.078,27.8,24.94,14.578,6.53,0.294,0.0,False,GPU -27,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1318.0,+162/-158,0.118,28.95,19.88,14.846,4546.1,39.323,0.0,False,GPU -28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1313.0,+170/-130,0.141,29.2,22.63,14.555,899.24,0.015,0.0,False,CPU -29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1307.0,+107/-154,0.101,29.5,23.67,15.444,387.73,2.446,0.0,False,CPU -30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1307.0,+156/-121,0.11,29.5,26.3,15.979,1.95,0.02,0.0,False,CPU -31,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1305.0,+130/-103,0.074,29.6,21.99,14.301,52.7,0.202,0.0,False,CPU -32,🧠🔁,Neural Network,[LimiX (default) [60.00% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1296.0,+236/-216,0.196,30.05,7.37,19.254,0.46,0.013,60.0,True,GPU -33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1279.0,+119/-104,0.08,30.9,27.5,14.214,1128.73,0.054,0.0,False,CPU -34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1279.0,+136/-144,0.06,30.9,27.87,13.793,38.59,0.337,0.0,False,CPU -35,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1276.0,+179/-173,0.117,31.1,20.39,15.572,82.52,0.97,0.0,False,GPU -36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1262.0,+124/-96,0.057,31.75,26.58,14.723,52.7,0.017,0.0,False,CPU -37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1220.0,+104/-127,0.042,33.8,31.74,13.965,0.33,0.05,0.0,False,CPU -38,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1210.0,+91/-114,0.016,34.3,32.79,14.804,11.03,0.489,0.0,False,GPU -39,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1184.0,+145/-162,0.031,35.5,32.53,17.278,38.59,0.028,0.0,False,CPU -40,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1176.0,+131/-135,0.02,35.85,32.54,18.831,41.63,39.259,0.0,False,GPU -41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1125.0,+100/-144,0.018,38.1,36.15,19.014,387.73,0.089,0.0,False,CPU -42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1099.0,+96/-108,0.011,39.2,37.66,17.823,1.84,0.047,0.0,False,CPU -43,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,1046.0,+175/-253,0.017,41.25,37.15,28.308,43.41,0.111,0.0,False,CPU -44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1037.0,+113/-202,0.018,41.6,38.74,24.557,332.4,33.481,0.0,False,CPU -45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1014.0,+145/-195,0.024,42.4,39.33,22.583,1.63,0.11,0.0,False,CPU -46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+112/-243,0.0,42.9,41.5,22.62,0.11,0.012,0.0,False,CPU -47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,998.0,+162/-268,0.002,42.95,40.08,29.28,43.41,0.02,0.0,False,CPU -48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,837.0,+96/-216,0.0,47.6,47.26,30.878,0.21,0.019,0.0,False,CPU -49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,837.0,+242/-553,0.024,47.6,40.42,33.515,2.96,0.183,0.0,False,GPU -50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,828.0,+85/-214,0.0,47.8,47.52,27.598,0.1,0.02,0.0,False,CPU -51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,792.0,+67/-191,0.0,48.6,48.26,33.031,332.4,1.631,0.0,False,CPU -52,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,762.0,+124/-263,0.0,49.2,48.67,35.655,0.18,0.032,0.0,False,GPU -53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,424.0,+108/-391,0.0,53.2,53.17,49.897,0.12,0.094,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",2023.0,+176/-73,1.576,0.867,3.3,2.15,307.34,2.186,✔️,0.0,False,CPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1959.0,+218/-71,2.126,0.842,4.2,2.28,266.49,2.963,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1712.0,+263/-137,5.306,0.575,10.4,3.14,780.95,11.64,✔️,0.0,False,GPU +3,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1684.0,+240/-162,5.366,0.601,11.4,4.17,2335.82,14.145,✔️,0.0,False,GPU +4,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1661.0,+156/-106,7.774,0.467,12.3,8.67,202.53,0.43,✔️,0.0,False,CPU +5,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1654.0,+240/-119,7.407,0.495,12.6,6.99,780.95,1.493,✔️,0.0,False,GPU +6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1646.0,+177/-92,7.059,0.519,12.9,7.92,688.25,0.192,✔️,0.0,False,CPU +7,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1632.0,+173/-79,8.682,0.445,13.5,9.79,91.02,0.568,✔️,0.0,False,CPU +8,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1626.0,+168/-86,7.132,0.494,13.75,8.83,688.25,0.036,✔️,0.0,False,CPU +9,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1615.0,+129/-89,7.485,0.466,14.2,9.63,1635.73,2.146,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1592.0,+183/-102,8.859,0.43,15.2,5.6,1400.65,0.571,✔️,0.0,False,GPU +11,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1590.0,+170/-111,9.79,0.417,15.3,11.13,2335.82,0.502,✔️,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1589.0,+136/-69,7.247,0.459,15.35,6.0,2.2,0.044,✔️,0.0,False,CPU +13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1576.0,+176/-103,8.091,0.414,15.9,11.12,202.53,0.055,✔️,0.0,False,CPU +14,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1557.0,+174/-105,9.535,0.352,16.8,13.01,91.02,0.074,✔️,0.0,False,CPU +15,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1525.0,+159/-111,9.894,0.355,18.3,10.13,1635.73,0.082,✔️,0.0,False,GPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1511.0,+196/-166,9.451,0.356,19.0,9.01,1400.65,0.065,✔️,0.0,False,GPU +17,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1502.0,+191/-126,8.488,0.337,19.4,9.54,2.3,0.818,✔️,0.0,False,GPU +18,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1490.0,+167/-135,10.137,0.292,20.0,12.21,8.46,3.879,✔️,0.0,False,GPU +19,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1439.0,+205/-127,12.923,0.253,22.6,15.64,899.24,0.221,✔️,0.0,False,CPU +20,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1435.0,+228/-155,10.324,0.285,22.8,10.66,0.47,0.047,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1412.0,+111/-69,11.033,0.164,24.0,19.83,1128.73,1.573,✔️,0.0,False,CPU +22,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1400.0,+180/-172,12.728,0.224,24.6,19.84,5.79,0.084,✔️,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1400.0,+137/-131,11.114,0.176,24.6,19.22,4546.1,406.11,✔️,0.0,False,GPU +24,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1393.0,+156/-139,11.586,0.223,25.0,16.3,713.0,2.317,✔️,0.0,False,GPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1364.0,+159/-138,14.611,0.15,26.5,18.58,713.0,0.155,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1340.0,+93/-103,14.578,0.078,27.8,24.94,6.53,0.294,✔️,0.0,False,GPU +27,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1318.0,+162/-158,14.846,0.118,28.95,19.88,4546.1,39.323,✔️,0.0,False,GPU +28,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1313.0,+170/-130,14.555,0.141,29.2,22.63,899.24,0.015,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1307.0,+107/-154,15.444,0.101,29.5,23.67,387.73,2.446,✔️,0.0,False,CPU +30,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1307.0,+156/-121,15.979,0.11,29.5,26.3,1.95,0.02,✔️,0.0,False,CPU +31,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1305.0,+130/-103,14.301,0.074,29.6,21.99,52.7,0.202,✔️,0.0,False,CPU +32,🧠⚡,Foundation Model,[LimiX (default) [60.00% IMPUTED]](https://arxiv.org/abs/2509.03505),1296.0,+236/-216,19.254,0.196,30.05,7.37,0.46,0.013,➖,60.0,True,GPU +33,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1279.0,+119/-104,14.214,0.08,30.9,27.5,1128.73,0.054,✔️,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1279.0,+136/-144,13.793,0.06,30.9,27.87,38.59,0.337,✔️,0.0,False,CPU +35,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1276.0,+179/-173,15.572,0.117,31.1,20.39,82.52,0.97,➖,0.0,False,GPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1262.0,+124/-96,14.723,0.057,31.75,26.58,52.7,0.017,✔️,0.0,False,CPU +37,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1220.0,+104/-127,13.965,0.042,33.8,31.74,0.33,0.05,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1210.0,+91/-114,14.804,0.016,34.3,32.79,11.03,0.489,✔️,0.0,False,GPU +39,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1184.0,+145/-162,17.278,0.031,35.5,32.53,38.59,0.028,✔️,0.0,False,CPU +40,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1176.0,+131/-135,18.831,0.02,35.85,32.54,41.63,39.259,✔️,0.0,False,GPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1125.0,+100/-144,19.014,0.018,38.1,36.15,387.73,0.089,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1099.0,+96/-108,17.823,0.011,39.2,37.66,1.84,0.047,✔️,0.0,False,CPU +43,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),1046.0,+175/-253,28.308,0.017,41.25,37.15,43.41,0.111,✔️,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1037.0,+113/-202,24.557,0.018,41.6,38.74,332.4,33.481,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),1014.0,+145/-195,22.583,0.024,42.4,39.33,1.63,0.11,✔️,0.0,False,CPU +46,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+112/-243,22.62,0.0,42.9,41.5,0.11,0.012,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),998.0,+162/-268,29.28,0.002,42.95,40.08,43.41,0.02,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),837.0,+96/-216,30.878,0.0,47.6,47.26,0.21,0.019,✔️,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),837.0,+242/-553,33.515,0.024,47.6,40.42,2.96,0.183,✔️,0.0,False,GPU +50,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),828.0,+85/-214,27.598,0.0,47.8,47.52,0.1,0.02,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),792.0,+67/-191,33.031,0.0,48.6,48.26,332.4,1.631,✔️,0.0,False,CPU +52,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),762.0,+124/-263,35.655,0.0,49.2,48.67,0.18,0.032,➖,0.0,False,GPU +53,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),424.0,+108/-391,49.897,0.0,53.2,53.17,0.12,0.094,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/winrate_matrix.png.zip index ce1e2f54f8b09beeb5c2add968e8eeff82b5acb4..41b09f8b8fed655f1ce918139e9a2083d1bcbbb2 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9f8e9ad1973094f40746081f5f1223a893bbc0d209670aad6b6890a80d1e30b8 +oid sha256:b6fefa7813c3765b91e4fffb19866742ff952318f962d00c019381b21b958f27 size 1795058 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index d3d45d00858326bab4de9bc2261b5fec5e2588e1..a50c399ce5ad735254f462395c82662a04bad72b 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1f3158320802329628380b20cf3983f8686e135140eaa0272de60d1478cef63a +oid sha256:10a7198765b50a0cb683b00e28ad2a4fa8aeccae2fe0d83aea7cfdd6e6f14280 size 473361 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip index 1b3975015b422a573b8e87088f7e7f658314b630..1456b35535e51545230faa4072a2bbc548b13bc5 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4b4e05f4251199c1edfe9547e7eef11c4516e162b8ad0f774ec2a7cfb44ba414 -size 1036578 +oid sha256:5d78bdd8bae05a2ba7d11979fd2de96071783b2325367e044f678aacf92600ad +size 1031428 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/tuning-impact-elo.png.zip index 13af61e65915224f687e47c51502b3eee8feebbb..c52aea9e00f56ddf6e84741a932cbd0f8cabbad9 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3fb9c854f9a6ea3a7638f7c5d77ee8c3172ccb1a5898573da47cf9e9108b19cf +oid sha256:8a8ff0e7c0fabb1c4e5c23a75048ffb83e3da4ced9015351dfb48d575802da44 size 179441 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/website_leaderboard.csv index 494483cf0a96d894192d94511cdb42327ee09829..566bd9e01dc146fbd7bc9f4e129039e8490607b3 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1603.0,+95/-74,0.618,9.05,5.38,7.437,6.03,0.703,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1596.0,+129/-95,0.646,9.32,2.85,4.66,2279.85,9.666,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1565.0,+125/-84,0.598,10.54,4.64,7.33,2279.85,0.968,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1526.0,+107/-86,0.567,12.25,5.35,8.987,635.11,8.569,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1476.0,+84/-64,0.445,14.71,7.26,11.594,3486.44,22.867,0.0,False,GPU -5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1459.0,+89/-66,0.434,15.68,8.29,11.423,2993.17,1.401,0.0,False,CPU -6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1457.0,+132/-93,0.454,15.79,6.98,12.285,3226.71,49.21,7.14,True,GPU -7,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1446.0,+87/-62,0.449,16.38,6.27,10.694,4.53,0.611,0.0,False,GPU -8,🧠🔁,Neural Network,[Mitra (default) [7.14% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1427.0,+107/-80,0.401,17.46,7.78,12.738,145.75,4.071,7.14,True,GPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1416.0,+74/-72,0.375,18.12,8.57,12.977,3486.44,1.002,0.0,False,GPU -10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1389.0,+84/-72,0.387,19.71,6.98,11.814,7432.95,404.225,0.0,False,GPU -11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1377.0,+76/-76,0.34,20.46,9.93,13.992,3083.36,1.48,0.0,False,GPU -12,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1365.0,+108/-81,0.316,21.25,10.98,14.612,3226.71,0.494,7.14,True,GPU -13,🧠⚡,Foundation Model,[TabICL (default) [7.14% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1355.0,+72/-64,0.329,21.86,6.59,13.761,9.07,1.864,7.14,True,GPU -14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1349.0,+62/-52,0.233,22.27,16.88,15.057,774.51,1.455,0.0,False,CPU -15,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1348.0,+96/-74,0.289,22.29,13.46,16.148,303.47,2.324,0.0,False,GPU -16,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1348.0,+85/-89,0.341,22.32,7.03,13.237,7432.95,39.797,0.0,False,GPU -17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1320.0,+89/-67,0.281,24.09,13.98,15.094,3083.36,0.189,0.0,False,GPU -18,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1319.0,+71/-61,0.228,24.2,16.26,14.996,1612.87,0.724,0.0,False,CPU -19,🧠⚡,Foundation Model,[TabPFNv2 (default) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1314.0,+94/-71,0.293,24.48,8.41,15.634,4.32,0.469,7.14,True,GPU -20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1307.0,+78/-74,0.241,25.0,14.07,15.826,1165.19,2.067,0.0,False,GPU -21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1306.0,+82/-77,0.246,25.04,11.45,14.692,6286.73,0.473,0.0,False,GPU -22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1303.0,+68/-70,0.205,25.23,15.29,16.187,829.32,1.642,0.0,False,CPU -23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1301.0,+57/-56,0.188,25.38,17.8,15.801,6.97,0.144,0.0,False,CPU -24,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1294.0,+67/-47,0.202,25.86,19.76,15.683,1612.87,0.061,0.0,False,CPU -25,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1291.0,+70/-66,0.199,26.0,20.3,16.028,774.51,0.282,0.0,False,CPU -26,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1287.0,+72/-64,0.178,26.29,15.72,16.805,2989.36,0.609,0.0,False,CPU -27,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1284.0,+69/-67,0.186,26.46,11.97,16.412,829.32,0.241,0.0,False,CPU -28,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1275.0,+83/-79,0.222,27.11,17.78,16.335,6286.73,9.386,0.0,False,GPU -29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1269.0,+81/-80,0.229,27.5,16.66,17.527,11.86,0.149,0.0,False,GPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1256.0,+70/-70,0.157,28.36,17.05,16.694,14.59,4.035,0.0,False,GPU -31,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1248.0,+60/-67,0.138,28.89,13.54,17.476,2989.36,0.063,0.0,False,CPU -32,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1246.0,+78/-82,0.17,29.05,12.43,16.995,1165.19,0.105,0.0,False,GPU -33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1240.0,+80/-64,0.145,29.39,12.97,18.192,8.12,0.053,0.0,False,CPU -34,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1238.0,+86/-106,0.24,29.54,9.17,16.168,51.78,39.927,0.0,False,GPU -35,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1237.0,+81/-73,0.165,29.61,22.23,17.116,2894.94,2.42,0.0,False,CPU -36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1197.0,+102/-91,0.158,32.27,20.71,19.917,1425.28,7.355,0.0,False,CPU -37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1180.0,+87/-76,0.141,33.32,23.9,19.081,2894.94,0.145,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1178.0,+83/-85,0.129,33.5,21.84,20.789,402.38,1.373,0.0,False,CPU -39,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1166.0,+69/-85,0.102,34.23,13.26,21.032,14.22,0.353,0.0,False,GPU -40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1155.0,+84/-85,0.139,34.95,16.9,21.307,402.38,0.163,0.0,False,CPU -41,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1136.0,+80/-98,0.113,36.14,25.19,21.865,562.0,1.155,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1135.0,+91/-92,0.117,36.21,22.19,21.368,1425.28,0.561,0.0,False,CPU -43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1134.0,+67/-66,0.061,36.29,30.61,19.814,2.85,0.19,0.0,False,CPU -44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1128.0,+64/-81,0.095,36.66,29.97,20.294,3.27,0.193,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1088.0,+72/-83,0.054,39.0,31.4,22.847,562.0,0.138,0.0,False,CPU -46,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1012.0,+95/-122,0.064,43.18,32.86,27.691,2.45,0.519,0.0,False,GPU -47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1010.0,+82/-100,0.033,43.32,37.04,25.453,9.06,0.196,0.0,False,CPU -48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1002.0,+96/-132,0.048,43.68,38.27,25.461,5.43,0.836,0.0,False,GPU -49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+85/-113,0.025,43.8,31.74,28.276,0.46,0.07,0.0,False,CPU -50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,979.0,+92/-121,0.037,44.82,37.52,28.627,123.01,1.579,0.0,False,CPU -51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,978.0,+84/-106,0.029,44.89,40.58,25.392,5.03,0.601,0.0,False,CPU -52,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,944.0,+94/-123,0.018,46.43,41.59,29.921,0.38,0.072,0.0,False,CPU -53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,934.0,+89/-116,0.036,46.82,17.87,31.991,479.28,0.495,0.0,False,CPU -54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,908.0,+89/-126,0.03,47.91,26.58,32.625,479.28,0.098,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,873.0,+104/-131,0.014,49.21,44.74,35.574,2.34,0.132,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,809.0,+108/-156,0.02,51.29,47.97,35.773,123.01,0.092,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,565.0,+114/-194,0.0,56.14,55.81,49.586,0.28,0.024,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1603.0,+95/-74,7.437,0.618,9.05,5.38,6.03,0.703,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1596.0,+129/-95,4.66,0.646,9.32,2.85,2279.85,9.666,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1565.0,+125/-84,7.33,0.598,10.54,4.64,2279.85,0.968,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1526.0,+107/-86,8.987,0.567,12.25,5.35,635.11,8.569,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1476.0,+84/-64,11.594,0.445,14.71,7.26,3486.44,22.867,✔️,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1459.0,+89/-66,11.423,0.434,15.68,8.29,2993.17,1.401,✔️,0.0,False,CPU +6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1457.0,+132/-93,12.285,0.454,15.79,6.98,3226.71,49.21,✔️,7.14,True,GPU +7,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1446.0,+87/-62,10.694,0.449,16.38,6.27,4.53,0.611,➖,0.0,False,GPU +8,🧠⚡,Foundation Model,[Mitra (default) [7.14% IMPUTED]](https://arxiv.org/abs/2510.21204),1427.0,+107/-80,12.738,0.401,17.46,7.78,145.75,4.071,✔️,7.14,True,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1416.0,+74/-72,12.977,0.375,18.12,8.57,3486.44,1.002,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1389.0,+84/-72,11.814,0.387,19.71,6.98,7432.95,404.225,✔️,0.0,False,GPU +11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1377.0,+76/-76,13.992,0.34,20.46,9.93,3083.36,1.48,✔️,0.0,False,GPU +12,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1365.0,+108/-81,14.612,0.316,21.25,10.98,3226.71,0.494,✔️,7.14,True,GPU +13,🧠⚡,Foundation Model,[TabICL (default) [7.14% IMPUTED]](https://arxiv.org/abs/2502.05564),1355.0,+72/-64,13.761,0.329,21.86,6.59,9.07,1.864,✔️,7.14,True,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1349.0,+62/-52,15.057,0.233,22.27,16.88,774.51,1.455,✔️,0.0,False,CPU +15,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1348.0,+96/-74,16.148,0.289,22.29,13.46,303.47,2.324,➖,0.0,False,GPU +16,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1348.0,+85/-89,13.237,0.341,22.32,7.03,7432.95,39.797,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1320.0,+89/-67,15.094,0.281,24.09,13.98,3083.36,0.189,✔️,0.0,False,GPU +18,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1319.0,+71/-61,14.996,0.228,24.2,16.26,1612.87,0.724,✔️,0.0,False,CPU +19,🧠⚡,Foundation Model,[TabPFNv2 (default) [7.14% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1314.0,+94/-71,15.634,0.293,24.48,8.41,4.32,0.469,✔️,7.14,True,GPU +20,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1307.0,+78/-74,15.826,0.241,25.0,14.07,1165.19,2.067,✔️,0.0,False,GPU +21,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1306.0,+82/-77,14.692,0.246,25.04,11.45,6286.73,0.473,✔️,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1303.0,+68/-70,16.187,0.205,25.23,15.29,829.32,1.642,✔️,0.0,False,CPU +23,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1301.0,+57/-56,15.801,0.188,25.38,17.8,6.97,0.144,✔️,0.0,False,CPU +24,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1294.0,+67/-47,15.683,0.202,25.86,19.76,1612.87,0.061,✔️,0.0,False,CPU +25,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1291.0,+70/-66,16.028,0.199,26.0,20.3,774.51,0.282,✔️,0.0,False,CPU +26,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1287.0,+72/-64,16.805,0.178,26.29,15.72,2989.36,0.609,✔️,0.0,False,CPU +27,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1284.0,+69/-67,16.412,0.186,26.46,11.97,829.32,0.241,✔️,0.0,False,CPU +28,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1275.0,+83/-79,16.335,0.222,27.11,17.78,6286.73,9.386,✔️,0.0,False,GPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1269.0,+81/-80,17.527,0.229,27.5,16.66,11.86,0.149,✔️,0.0,False,GPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1256.0,+70/-70,16.694,0.157,28.36,17.05,14.59,4.035,✔️,0.0,False,GPU +31,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1248.0,+60/-67,17.476,0.138,28.89,13.54,2989.36,0.063,✔️,0.0,False,CPU +32,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1246.0,+78/-82,16.995,0.17,29.05,12.43,1165.19,0.105,✔️,0.0,False,GPU +33,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1240.0,+80/-64,18.192,0.145,29.39,12.97,8.12,0.053,✔️,0.0,False,CPU +34,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1238.0,+86/-106,16.168,0.24,29.54,9.17,51.78,39.927,✔️,0.0,False,GPU +35,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1237.0,+81/-73,17.116,0.165,29.61,22.23,2894.94,2.42,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1197.0,+102/-91,19.917,0.158,32.27,20.71,1425.28,7.355,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1180.0,+87/-76,19.081,0.141,33.32,23.9,2894.94,0.145,✔️,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1178.0,+83/-85,20.789,0.129,33.5,21.84,402.38,1.373,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1166.0,+69/-85,21.032,0.102,34.23,13.26,14.22,0.353,✔️,0.0,False,GPU +40,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1155.0,+84/-85,21.307,0.139,34.95,16.9,402.38,0.163,✔️,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1136.0,+80/-98,21.865,0.113,36.14,25.19,562.0,1.155,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1135.0,+91/-92,21.368,0.117,36.21,22.19,1425.28,0.561,✔️,0.0,False,CPU +43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1134.0,+67/-66,19.814,0.061,36.29,30.61,2.85,0.19,✔️,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1128.0,+64/-81,20.294,0.095,36.66,29.97,3.27,0.193,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1088.0,+72/-83,22.847,0.054,39.0,31.4,562.0,0.138,✔️,0.0,False,CPU +46,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1012.0,+95/-122,27.691,0.064,43.18,32.86,2.45,0.519,➖,0.0,False,GPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1010.0,+82/-100,25.453,0.033,43.32,37.04,9.06,0.196,✔️,0.0,False,CPU +48,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1002.0,+96/-132,25.461,0.048,43.68,38.27,5.43,0.836,✔️,0.0,False,GPU +49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+85/-113,28.276,0.025,43.8,31.74,0.46,0.07,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),979.0,+92/-121,28.627,0.037,44.82,37.52,123.01,1.579,✔️,0.0,False,CPU +51,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),978.0,+84/-106,25.392,0.029,44.89,40.58,5.03,0.601,✔️,0.0,False,CPU +52,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),944.0,+94/-123,29.921,0.018,46.43,41.59,0.38,0.072,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),934.0,+89/-116,31.991,0.036,46.82,17.87,479.28,0.495,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),908.0,+89/-126,32.625,0.03,47.91,26.58,479.28,0.098,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),873.0,+104/-131,35.574,0.014,49.21,44.74,2.34,0.132,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),809.0,+108/-156,35.773,0.02,51.29,47.97,123.01,0.092,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),565.0,+114/-194,49.586,0.0,56.14,55.81,0.28,0.024,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/winrate_matrix.png.zip index 60a71a7cf219376263f40e674247b710020bb94f..bf53bf2673e41905932c647be0d77e6b2f6fd592 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6f35a4ce1cdc4d296be798638f97b52e11e9bd34e90c56ec42fa4a2750d5e407 +oid sha256:4009c2691e3a3a767625bde867c12c6f3b19d60e22b8c8e88b81226da5967ca2 size 2402746 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 7f62c7bd1c985d943e492b8f831d12355fa72bb9..5ef1882182376979e2af8064911f4df016c7156f 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4a77ba8244c373ba077f7fd8b3fc037bec886d67eb997f47375a4ca5ba374dcb +oid sha256:3660b9a319ecbb36b1bc8d75e5c01c63b3a33a712f8025022e7ff2c27fb888f4 size 481575 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip index e6a35f19914097e935eed4fbfbd98675bf75b383..c5d35aba0b8430d6c4c66e3b77bf9a03e78544db 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:caa0fbfc23a316e4ea64fe9d5bb4aade532ea12b0f6f084ecdaf58e75e135f5e -size 1038842 +oid sha256:66ac13813be4eaa8dbc496584a35acd482640c60a3b49abcd863aff40c9fafcc +size 1023492 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip index ce4b8166d249bad8358abb84ef8445a274a1f9de..ea17501ee0a4ce45fe7f4aa59f86c57319feb69d 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fa56c5b223191a4ea45d028c9f764cf9d11450c8b889d4b83927101c54682afb +oid sha256:eb2ec143bac9618492218a8f500c0fb3ecc5f0c4f2e388985062f3d813b2871e size 121062 diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/website_leaderboard.csv index c5cdb97720afb21c722cc84043868ea75819c1b0..1fe395cd30ca8ffe789f7e6c60ff47aaab3d9097 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1648.0,+88/-68,0.635,8.1,5.12,7.839,5.78,0.586,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1624.0,+124/-100,0.653,8.92,2.96,4.968,2162.76,8.93,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1601.0,+124/-104,0.61,9.81,4.39,7.727,2162.76,0.894,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1533.0,+113/-82,0.548,12.73,5.43,9.618,620.85,8.569,0.0,False,GPU -4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1510.0,+93/-63,0.463,13.81,6.88,12.159,3159.9,21.159,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1509.0,+107/-89,0.488,13.87,6.57,11.53,3361.32,51.669,0.0,False,GPU -6,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1472.0,+108/-68,0.432,15.79,7.34,12.029,150.4,4.403,0.0,False,GPU -7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1467.0,+82/-70,0.41,16.08,8.22,12.218,2628.27,1.401,0.0,False,CPU -8,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1460.0,+100/-66,0.439,16.44,6.0,11.286,4.3,0.584,0.0,False,GPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1446.0,+89/-57,0.394,17.25,8.14,13.535,3159.9,0.646,0.0,False,GPU -10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1403.0,+84/-80,0.341,19.75,10.41,14.036,3361.32,0.527,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1403.0,+103/-85,0.381,19.77,6.66,12.433,6883.9,405.331,0.0,False,GPU -12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1392.0,+106/-73,0.335,20.42,9.82,14.604,2899.02,1.461,0.0,False,GPU -13,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1390.0,+84/-96,0.354,20.52,6.2,13.131,9.07,2.008,0.0,False,GPU -14,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1377.0,+96/-83,0.298,21.31,12.92,16.382,303.47,2.098,0.0,False,GPU -15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1360.0,+81/-99,0.341,22.42,6.69,13.867,6883.9,39.606,0.0,False,GPU -16,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1345.0,+83/-90,0.316,23.35,7.94,15.148,4.28,0.511,0.0,False,GPU -17,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1345.0,+53/-41,0.192,23.37,19.01,16.138,691.46,1.455,0.0,False,CPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1336.0,+73/-88,0.265,23.88,10.86,15.115,6020.89,0.454,0.0,False,GPU -19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1330.0,+102/-70,0.27,24.33,14.64,15.756,2899.02,0.171,0.0,False,GPU -20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1326.0,+85/-72,0.213,24.52,16.47,16.007,1394.45,0.618,0.0,False,CPU -21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1308.0,+69/-57,0.182,25.71,19.95,17.318,785.9,1.281,0.0,False,CPU -22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1306.0,+76/-53,0.169,25.83,18.07,16.868,5.74,0.139,0.0,False,CPU -23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1305.0,+73/-64,0.185,25.92,15.16,17.587,2399.23,0.565,0.0,False,CPU -24,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1300.0,+79/-73,0.2,26.27,18.27,16.877,1053.72,1.872,0.0,False,GPU -25,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1297.0,+91/-84,0.239,26.42,17.12,17.031,6020.89,8.478,0.0,False,GPU -26,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1293.0,+74/-51,0.173,26.69,21.51,16.785,1394.45,0.053,0.0,False,CPU -27,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1290.0,+72/-60,0.166,26.88,12.61,17.51,785.9,0.213,0.0,False,CPU -28,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1288.0,+71/-52,0.163,27.0,21.23,17.102,691.46,0.264,0.0,False,CPU -29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1285.0,+102/-73,0.23,27.19,16.18,18.07,11.6,0.148,0.0,False,GPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1284.0,+83/-65,0.17,27.27,16.3,17.108,12.75,3.502,0.0,False,GPU -31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1257.0,+98/-127,0.259,29.04,8.67,16.756,51.52,40.126,0.0,False,GPU -32,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1254.0,+90/-93,0.139,29.23,12.47,18.783,7.41,0.052,0.0,False,CPU -33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1254.0,+60/-73,0.136,29.27,13.11,18.357,2399.23,0.061,0.0,False,CPU -34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1253.0,+73/-74,0.169,29.31,21.75,17.668,2792.14,2.175,0.0,False,CPU -35,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1234.0,+85/-76,0.126,30.56,21.73,18.071,1053.72,0.103,0.0,False,GPU -36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1221.0,+93/-108,0.17,31.37,19.9,20.456,1350.04,6.57,0.0,False,CPU -37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1191.0,+72/-113,0.109,33.29,12.56,21.041,13.67,0.339,0.0,False,GPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1183.0,+80/-82,0.119,33.81,26.42,19.614,2792.14,0.133,0.0,False,CPU -39,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1160.0,+84/-81,0.086,35.21,27.55,22.147,378.34,1.373,0.0,False,CPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1154.0,+96/-109,0.126,35.58,21.37,21.966,1350.04,0.613,0.0,False,CPU -41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1137.0,+91/-92,0.107,36.62,19.58,22.595,378.34,0.154,0.0,False,CPU -42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1128.0,+60/-59,0.033,37.19,33.98,21.116,2.66,0.17,0.0,False,CPU -43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1118.0,+107/-106,0.078,37.77,26.65,23.308,527.27,1.155,0.0,False,CPU -44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1103.0,+69/-69,0.047,38.63,34.62,21.73,3.01,0.177,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1079.0,+91/-89,0.045,40.04,32.03,24.138,527.27,0.128,0.0,False,CPU -46,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,1029.0,+86/-152,0.069,42.69,32.05,28.036,2.28,0.454,0.0,False,GPU -47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1013.0,+91/-94,0.036,43.5,36.97,26.238,8.93,0.172,0.0,False,CPU -48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+86/-108,0.027,44.15,31.42,28.762,0.44,0.065,0.0,False,CPU -49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,999.0,+93/-122,0.051,44.19,38.38,26.76,5.0,0.753,0.0,False,GPU -50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,990.0,+94/-109,0.031,44.62,40.08,25.606,4.68,0.547,0.0,False,CPU -51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,959.0,+80/-107,0.009,46.04,43.24,29.843,118.4,1.479,0.0,False,CPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,956.0,+94/-99,0.038,46.17,16.98,31.197,429.17,0.495,0.0,False,CPU -53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,946.0,+87/-146,0.02,46.62,41.57,30.28,0.36,0.068,0.0,False,CPU -54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,930.0,+99/-111,0.032,47.27,25.54,31.807,429.17,0.081,0.0,False,CPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,897.0,+108/-120,0.015,48.58,43.99,34.214,2.16,0.129,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,766.0,+80/-151,0.0,52.58,51.76,37.314,118.4,0.082,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,576.0,+118/-291,0.0,56.08,55.73,49.46,0.27,0.024,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1648.0,+88/-68,7.839,0.635,8.1,5.12,5.78,0.586,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1624.0,+124/-100,4.968,0.653,8.92,2.96,2162.76,8.93,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1601.0,+124/-104,7.727,0.61,9.81,4.39,2162.76,0.894,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1533.0,+113/-82,9.618,0.548,12.73,5.43,620.85,8.569,✔️,0.0,False,GPU +4,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1510.0,+93/-63,12.159,0.463,13.81,6.88,3159.9,21.159,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1509.0,+107/-89,11.53,0.488,13.87,6.57,3361.32,51.669,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1472.0,+108/-68,12.029,0.432,15.79,7.34,150.4,4.403,✔️,0.0,False,GPU +7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1467.0,+82/-70,12.218,0.41,16.08,8.22,2628.27,1.401,✔️,0.0,False,CPU +8,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1460.0,+100/-66,11.286,0.439,16.44,6.0,4.3,0.584,➖,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1446.0,+89/-57,13.535,0.394,17.25,8.14,3159.9,0.646,✔️,0.0,False,GPU +10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1403.0,+84/-80,14.036,0.341,19.75,10.41,3361.32,0.527,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1403.0,+103/-85,12.433,0.381,19.77,6.66,6883.9,405.331,✔️,0.0,False,GPU +12,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1392.0,+106/-73,14.604,0.335,20.42,9.82,2899.02,1.461,✔️,0.0,False,GPU +13,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1390.0,+84/-96,13.131,0.354,20.52,6.2,9.07,2.008,✔️,0.0,False,GPU +14,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1377.0,+96/-83,16.382,0.298,21.31,12.92,303.47,2.098,➖,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1360.0,+81/-99,13.867,0.341,22.42,6.69,6883.9,39.606,✔️,0.0,False,GPU +16,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1345.0,+83/-90,15.148,0.316,23.35,7.94,4.28,0.511,✔️,0.0,False,GPU +17,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1345.0,+53/-41,16.138,0.192,23.37,19.01,691.46,1.455,✔️,0.0,False,CPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1336.0,+73/-88,15.115,0.265,23.88,10.86,6020.89,0.454,✔️,0.0,False,GPU +19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1330.0,+102/-70,15.756,0.27,24.33,14.64,2899.02,0.171,✔️,0.0,False,GPU +20,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1326.0,+85/-72,16.007,0.213,24.52,16.47,1394.45,0.618,✔️,0.0,False,CPU +21,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1308.0,+69/-57,17.318,0.182,25.71,19.95,785.9,1.281,✔️,0.0,False,CPU +22,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1306.0,+76/-53,16.868,0.169,25.83,18.07,5.74,0.139,✔️,0.0,False,CPU +23,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1305.0,+73/-64,17.587,0.185,25.92,15.16,2399.23,0.565,✔️,0.0,False,CPU +24,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1300.0,+79/-73,16.877,0.2,26.27,18.27,1053.72,1.872,✔️,0.0,False,GPU +25,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1297.0,+91/-84,17.031,0.239,26.42,17.12,6020.89,8.478,✔️,0.0,False,GPU +26,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1293.0,+74/-51,16.785,0.173,26.69,21.51,1394.45,0.053,✔️,0.0,False,CPU +27,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1290.0,+72/-60,17.51,0.166,26.88,12.61,785.9,0.213,✔️,0.0,False,CPU +28,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1288.0,+71/-52,17.102,0.163,27.0,21.23,691.46,0.264,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1285.0,+102/-73,18.07,0.23,27.19,16.18,11.6,0.148,✔️,0.0,False,GPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1284.0,+83/-65,17.108,0.17,27.27,16.3,12.75,3.502,✔️,0.0,False,GPU +31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1257.0,+98/-127,16.756,0.259,29.04,8.67,51.52,40.126,✔️,0.0,False,GPU +32,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1254.0,+90/-93,18.783,0.139,29.23,12.47,7.41,0.052,✔️,0.0,False,CPU +33,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1254.0,+60/-73,18.357,0.136,29.27,13.11,2399.23,0.061,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1253.0,+73/-74,17.668,0.169,29.31,21.75,2792.14,2.175,✔️,0.0,False,CPU +35,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1234.0,+85/-76,18.071,0.126,30.56,21.73,1053.72,0.103,✔️,0.0,False,GPU +36,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1221.0,+93/-108,20.456,0.17,31.37,19.9,1350.04,6.57,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1191.0,+72/-113,21.041,0.109,33.29,12.56,13.67,0.339,✔️,0.0,False,GPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1183.0,+80/-82,19.614,0.119,33.81,26.42,2792.14,0.133,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1160.0,+84/-81,22.147,0.086,35.21,27.55,378.34,1.373,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1154.0,+96/-109,21.966,0.126,35.58,21.37,1350.04,0.613,✔️,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1137.0,+91/-92,22.595,0.107,36.62,19.58,378.34,0.154,✔️,0.0,False,CPU +42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1128.0,+60/-59,21.116,0.033,37.19,33.98,2.66,0.17,✔️,0.0,False,CPU +43,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1118.0,+107/-106,23.308,0.078,37.77,26.65,527.27,1.155,✔️,0.0,False,CPU +44,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1103.0,+69/-69,21.73,0.047,38.63,34.62,3.01,0.177,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1079.0,+91/-89,24.138,0.045,40.04,32.03,527.27,0.128,✔️,0.0,False,CPU +46,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),1029.0,+86/-152,28.036,0.069,42.69,32.05,2.28,0.454,➖,0.0,False,GPU +47,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1013.0,+91/-94,26.238,0.036,43.5,36.97,8.93,0.172,✔️,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+86/-108,28.762,0.027,44.15,31.42,0.44,0.065,✔️,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),999.0,+93/-122,26.76,0.051,44.19,38.38,5.0,0.753,✔️,0.0,False,GPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),990.0,+94/-109,25.606,0.031,44.62,40.08,4.68,0.547,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),959.0,+80/-107,29.843,0.009,46.04,43.24,118.4,1.479,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),956.0,+94/-99,31.197,0.038,46.17,16.98,429.17,0.495,✔️,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),946.0,+87/-146,30.28,0.02,46.62,41.57,0.36,0.068,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),930.0,+99/-111,31.807,0.032,47.27,25.54,429.17,0.081,✔️,0.0,False,CPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),897.0,+108/-120,34.214,0.015,48.58,43.99,2.16,0.129,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),766.0,+80/-151,37.314,0.0,52.58,51.76,118.4,0.082,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),576.0,+118/-291,49.46,0.0,56.08,55.73,0.27,0.024,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip index 3b9bc6c0dd70956c31f7ad678ce566558d97efff..65dbd80994f0ff2942dd25b37c99ac0133805069 100644 --- a/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_classification/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d1ccaff088036697bfb009fbf65c0f4219c2bdc58f19ac73fe5f642de08f15e7 +oid sha256:c668582b4db6db0ea6b4685d4f1e16f2c54f5a894dc20c85607658823a7bc2f0 size 2336052 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index 165cdf251befd47a2c0356804d9129c9d965c70b..b23a5a8143368347b56a7d1e05146d7d8df155c0 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9beae876e2d8e4ddd581bcedfdb26cf9c2f4772ae1f7ec09c74184c00926e9ca +oid sha256:c41dfb1ce8978e97fb425592288545d25aeace828f367d002f0014fd97098778 size 466924 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip index c86657c322f132d65120eec987e7ee40fc035291..b28b654a37bdf4d1c043741fef7b98de9cecdc63 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b95acc32536124ad75cd15f63cdbd2c8093b34c8c014cd6fefa8cc3374f7f3eb -size 1027006 +oid sha256:e6e5faf6f26a0a508e2d9ef7a144442b343171e7089cde1e45d5dd466c0191fb +size 1007719 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip index 952decdb436cb48489551aae752b7a523db0447c..ea164860b05c7b92ddd8d519f89bce38c4daf84c 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:86f32f6a1dfa8c0b9af8304c47654e0b347cb6c75896a0b962b8b35232f3776c +oid sha256:46cb4af198af897080ad8f1eaac66ac6551c563b91f5a795d894335eb142641b size 169156 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/website_leaderboard.csv index 9d1bcb26db617ed948b4876d4ff97b16e8aa7b00..da5264ea182bfefca3100fbc2173060f912ef7a0 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1724.0,+707/-194,0.796,6.0,1.66,1.808,2793.65,8.93,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1666.0,+231/-99,0.72,7.62,5.34,11.082,1008.06,9.121,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1657.0,+269/-152,0.675,7.94,4.58,9.911,2793.65,1.513,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1523.0,+311/-140,0.527,13.31,5.61,10.367,9.88,0.825,0.0,False,GPU -4,🧠🔁,Neural Network,[LimiX (default) [12.50% IMPUTED]](https://arxiv.org/abs/2509.03505),➖,1492.0,+207/-87,0.456,14.81,8.07,12.322,5.93,0.989,12.5,True,GPU -5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1470.0,+196/-107,0.496,16.0,4.26,14.071,5546.65,3.268,0.0,False,CPU -6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1432.0,+176/-88,0.359,18.12,14.73,15.024,3307.58,1.182,0.0,False,CPU -7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1430.0,+204/-111,0.401,18.25,5.67,15.484,4334.8,21.159,0.0,False,GPU -8,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1425.0,+318/-180,0.447,18.5,6.69,18.166,2584.13,12.372,25.0,True,GPU -9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1421.0,+173/-68,0.336,18.75,15.39,15.552,3307.58,0.14,0.0,False,CPU -10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1410.0,+136/-93,0.316,19.38,13.27,16.245,1280.01,4.078,0.0,False,CPU -11,🧠🔁,Neural Network,[Mitra (default) [25.00% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1410.0,+215/-171,0.384,19.38,13.23,18.908,159.43,3.545,25.0,True,GPU -12,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1400.0,+134/-112,0.32,20.0,13.04,16.321,1280.01,1.051,0.0,False,CPU -13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1398.0,+193/-119,0.354,20.12,15.39,15.962,4334.8,3.529,0.0,False,GPU -14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1389.0,+142/-108,0.32,20.62,13.94,18.699,5568.31,1.781,0.0,False,GPU -15,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1385.0,+215/-130,0.374,20.88,11.34,19.449,9411.86,344.009,0.0,False,GPU -16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1375.0,+148/-106,0.317,21.5,12.59,19.038,5568.31,0.372,0.0,False,GPU -17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1347.0,+112/-85,0.219,23.25,21.3,16.949,2029.77,4.113,0.0,False,CPU -18,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1347.0,+193/-161,0.296,23.25,9.4,19.369,2079.21,3.347,0.0,False,GPU -19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1345.0,+124/-90,0.234,23.38,21.0,16.708,2029.77,1.04,0.0,False,CPU -20,🧠⚡,Foundation Model,[TabPFNv2 (default) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1341.0,+193/-143,0.291,23.62,11.54,18.203,5.48,0.354,25.0,True,GPU -21,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1330.0,+214/-150,0.28,24.38,13.89,19.679,2584.13,0.408,25.0,True,GPU -22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1325.0,+143/-104,0.225,24.69,20.75,18.145,6684.65,9.59,0.0,False,GPU -23,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1303.0,+157/-112,0.184,26.06,21.31,18.488,6684.65,0.752,0.0,False,GPU -24,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1302.0,+213/-134,0.302,26.12,6.19,20.809,9411.86,39.507,0.0,False,GPU -25,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1287.0,+146/-134,0.185,27.12,16.25,20.217,27108.48,0.742,0.0,False,CPU -26,🧠⚡,Foundation Model,[TabICL (default) [12.50% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1284.0,+236/-185,0.269,27.31,8.56,23.291,11.51,1.948,12.5,True,GPU -27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1283.0,+144/-138,0.227,27.38,15.88,22.14,17.09,0.145,0.0,False,GPU -28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1282.0,+193/-166,0.24,27.44,14.07,21.612,728.32,2.441,0.0,False,CPU -29,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1277.0,+114/-76,0.153,27.75,24.97,18.577,43.1,0.254,0.0,False,CPU -30,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1275.0,+224/-214,0.266,27.88,14.24,20.771,729.17,1.833,0.0,False,CPU -31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1269.0,+242/-179,0.234,28.25,6.12,20.994,2079.21,0.319,0.0,False,GPU -32,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1256.0,+205/-148,0.211,29.06,14.14,21.742,728.32,0.356,0.0,False,CPU -33,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1251.0,+123/-146,0.187,29.38,22.35,21.941,283.57,4.104,0.0,False,GPU -34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1241.0,+123/-109,0.132,30.0,27.09,21.705,3646.83,2.162,0.0,False,CPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1228.0,+122/-158,0.169,30.88,24.3,21.727,27108.48,0.066,0.0,False,CPU -36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1216.0,+122/-136,0.139,31.62,28.2,22.231,17.86,0.099,0.0,False,CPU -37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1206.0,+148/-164,0.126,32.25,21.47,23.531,3646.83,0.194,0.0,False,CPU -38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1200.0,+198/-186,0.191,32.62,20.2,22.297,729.17,0.325,0.0,False,CPU -39,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1190.0,+204/-185,0.168,33.25,21.78,25.651,2721.87,12.593,0.0,False,CPU -40,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1184.0,+252/-168,0.213,33.62,10.77,24.06,46.36,39.812,0.0,False,GPU -41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1178.0,+130/-119,0.07,34.0,30.54,20.547,4.93,0.592,0.0,False,CPU -42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1176.0,+155/-160,0.13,34.12,26.02,22.982,5.12,0.444,0.0,False,CPU -43,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1112.0,+125/-146,0.066,37.88,33.48,25.251,19.12,6.922,0.0,False,GPU -44,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1101.0,+125/-156,0.022,38.5,35.99,27.29,17.24,0.565,0.0,False,GPU -45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1089.0,+164/-171,0.085,39.12,30.54,27.296,2721.87,1.083,0.0,False,CPU -46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1085.0,+163/-237,0.058,39.38,33.84,29.013,8.99,1.372,0.0,False,GPU -47,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1013.0,+264/-406,0.128,43.0,27.42,38.476,155.52,2.046,0.0,False,CPU -48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+99/-219,0.0,43.62,41.77,35.562,0.74,0.148,0.0,False,CPU -49,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,972.0,+107/-162,0.0,44.88,43.55,30.565,14.37,0.363,0.0,False,CPU -50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,929.0,+156/-270,0.024,46.62,42.92,34.532,8.37,0.659,0.0,False,CPU -51,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,908.0,+73/-232,0.0,47.44,46.53,41.326,909.67,0.609,0.0,False,CPU -52,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,890.0,+83/-217,0.0,48.06,47.0,42.282,909.67,0.107,0.0,False,CPU -53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,885.0,+164/-240,0.003,48.25,45.44,40.712,0.76,0.153,0.0,False,CPU -54,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,780.0,+283/-1302,0.071,51.38,42.87,50.559,155.52,0.191,0.0,False,CPU -55,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,765.0,+104/-224,0.0,51.75,51.44,41.42,4.67,0.855,0.0,False,GPU -56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,698.0,+93/-331,0.0,53.25,52.96,48.104,4.62,0.135,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-283.0,+165/-1047,0.0,58.0,58.0,70.927,0.52,0.075,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1724.0,+707/-194,1.808,0.796,6.0,1.66,2793.65,8.93,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1666.0,+231/-99,11.082,0.72,7.62,5.34,1008.06,9.121,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1657.0,+269/-152,9.911,0.675,7.94,4.58,2793.65,1.513,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1523.0,+311/-140,10.367,0.527,13.31,5.61,9.88,0.825,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[LimiX (default) [12.50% IMPUTED]](https://arxiv.org/abs/2509.03505),1492.0,+207/-87,12.322,0.456,14.81,8.07,5.93,0.989,➖,12.5,True,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1470.0,+196/-107,14.071,0.496,16.0,4.26,5546.65,3.268,✔️,0.0,False,CPU +6,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1432.0,+176/-88,15.024,0.359,18.12,14.73,3307.58,1.182,✔️,0.0,False,CPU +7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1430.0,+204/-111,15.484,0.401,18.25,5.67,4334.8,21.159,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1425.0,+318/-180,18.166,0.447,18.5,6.69,2584.13,12.372,✔️,25.0,True,GPU +9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1421.0,+173/-68,15.552,0.336,18.75,15.39,3307.58,0.14,✔️,0.0,False,CPU +10,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1410.0,+136/-93,16.245,0.316,19.38,13.27,1280.01,4.078,✔️,0.0,False,CPU +11,🧠⚡,Foundation Model,[Mitra (default) [25.00% IMPUTED]](https://arxiv.org/abs/2510.21204),1410.0,+215/-171,18.908,0.384,19.38,13.23,159.43,3.545,✔️,25.0,True,GPU +12,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1400.0,+134/-112,16.321,0.32,20.0,13.04,1280.01,1.051,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1398.0,+193/-119,15.962,0.354,20.12,15.39,4334.8,3.529,✔️,0.0,False,GPU +14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1389.0,+142/-108,18.699,0.32,20.62,13.94,5568.31,1.781,✔️,0.0,False,GPU +15,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1385.0,+215/-130,19.449,0.374,20.88,11.34,9411.86,344.009,✔️,0.0,False,GPU +16,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1375.0,+148/-106,19.038,0.317,21.5,12.59,5568.31,0.372,✔️,0.0,False,GPU +17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1347.0,+112/-85,16.949,0.219,23.25,21.3,2029.77,4.113,✔️,0.0,False,CPU +18,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1347.0,+193/-161,19.369,0.296,23.25,9.4,2079.21,3.347,✔️,0.0,False,GPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1345.0,+124/-90,16.708,0.234,23.38,21.0,2029.77,1.04,✔️,0.0,False,CPU +20,🧠⚡,Foundation Model,[TabPFNv2 (default) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1341.0,+193/-143,18.203,0.291,23.62,11.54,5.48,0.354,✔️,25.0,True,GPU +21,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [25.00% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1330.0,+214/-150,19.679,0.28,24.38,13.89,2584.13,0.408,✔️,25.0,True,GPU +22,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1325.0,+143/-104,18.145,0.225,24.69,20.75,6684.65,9.59,✔️,0.0,False,GPU +23,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1303.0,+157/-112,18.488,0.184,26.06,21.31,6684.65,0.752,✔️,0.0,False,GPU +24,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1302.0,+213/-134,20.809,0.302,26.12,6.19,9411.86,39.507,✔️,0.0,False,GPU +25,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1287.0,+146/-134,20.217,0.185,27.12,16.25,27108.48,0.742,✔️,0.0,False,CPU +26,🧠⚡,Foundation Model,[TabICL (default) [12.50% IMPUTED]](https://arxiv.org/abs/2502.05564),1284.0,+236/-185,23.291,0.269,27.31,8.56,11.51,1.948,✔️,12.5,True,GPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1283.0,+144/-138,22.14,0.227,27.38,15.88,17.09,0.145,✔️,0.0,False,GPU +28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1282.0,+193/-166,21.612,0.24,27.44,14.07,728.32,2.441,✔️,0.0,False,CPU +29,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1277.0,+114/-76,18.577,0.153,27.75,24.97,43.1,0.254,✔️,0.0,False,CPU +30,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1275.0,+224/-214,20.771,0.266,27.88,14.24,729.17,1.833,✔️,0.0,False,CPU +31,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1269.0,+242/-179,20.994,0.234,28.25,6.12,2079.21,0.319,✔️,0.0,False,GPU +32,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1256.0,+205/-148,21.742,0.211,29.06,14.14,728.32,0.356,✔️,0.0,False,CPU +33,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1251.0,+123/-146,21.941,0.187,29.38,22.35,283.57,4.104,➖,0.0,False,GPU +34,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1241.0,+123/-109,21.705,0.132,30.0,27.09,3646.83,2.162,✔️,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1228.0,+122/-158,21.727,0.169,30.88,24.3,27108.48,0.066,✔️,0.0,False,CPU +36,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1216.0,+122/-136,22.231,0.139,31.62,28.2,17.86,0.099,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1206.0,+148/-164,23.531,0.126,32.25,21.47,3646.83,0.194,✔️,0.0,False,CPU +38,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1200.0,+198/-186,22.297,0.191,32.62,20.2,729.17,0.325,✔️,0.0,False,CPU +39,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1190.0,+204/-185,25.651,0.168,33.25,21.78,2721.87,12.593,✔️,0.0,False,CPU +40,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1184.0,+252/-168,24.06,0.213,33.62,10.77,46.36,39.812,✔️,0.0,False,GPU +41,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1178.0,+130/-119,20.547,0.07,34.0,30.54,4.93,0.592,✔️,0.0,False,CPU +42,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1176.0,+155/-160,22.982,0.13,34.12,26.02,5.12,0.444,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1112.0,+125/-146,25.251,0.066,37.88,33.48,19.12,6.922,✔️,0.0,False,GPU +44,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1101.0,+125/-156,27.29,0.022,38.5,35.99,17.24,0.565,✔️,0.0,False,GPU +45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1089.0,+164/-171,27.296,0.085,39.12,30.54,2721.87,1.083,✔️,0.0,False,CPU +46,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1085.0,+163/-237,29.013,0.058,39.38,33.84,8.99,1.372,✔️,0.0,False,GPU +47,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1013.0,+264/-406,38.476,0.128,43.0,27.42,155.52,2.046,✔️,0.0,False,CPU +48,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+99/-219,35.562,0.0,43.62,41.77,0.74,0.148,✔️,0.0,False,CPU +49,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),972.0,+107/-162,30.565,0.0,44.88,43.55,14.37,0.363,✔️,0.0,False,CPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),929.0,+156/-270,34.532,0.024,46.62,42.92,8.37,0.659,✔️,0.0,False,CPU +51,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),908.0,+73/-232,41.326,0.0,47.44,46.53,909.67,0.609,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),890.0,+83/-217,42.282,0.0,48.06,47.0,909.67,0.107,✔️,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),885.0,+164/-240,40.712,0.003,48.25,45.44,0.76,0.153,✔️,0.0,False,CPU +54,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),780.0,+283/-1302,50.559,0.071,51.38,42.87,155.52,0.191,✔️,0.0,False,CPU +55,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),765.0,+104/-224,41.42,0.0,51.75,51.44,4.67,0.855,➖,0.0,False,GPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),698.0,+93/-331,48.104,0.0,53.25,52.96,4.62,0.135,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-283.0,+165/-1047,70.927,0.0,58.0,58.0,0.52,0.075,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/winrate_matrix.png.zip index ffeaf6f1a5c0ae8c51a2c5267db381427c7c2b23..68d95d1cbdfe193f2191496de484f58e51919f18 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bb16a715a0f5f5dcfbbc7a0f4c65c94776aab8f68562b31d5f782689b08b143a +oid sha256:4c179e7e2c619545fb7a3953c137ad7953f31b7de66374a5e7a4cb9672eb0694 size 1788887 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index 032193a1de40318e74f2abb61794382a84bc4bb7..502f9965ec35ce1828d9e48438bd7cebbace2e1a 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:009b6cb81ef3de594a819fbf56a75afa7f313664c5bdc519192d3e9ec56fdacd +oid sha256:134a614466a30a212c4dfab23d74e399f90ed8d3d055621a4f9858988bab216f size 451105 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip index 3a4ee0ea943bc5e8dfe9425f2de3082de817c8db..44a02f1b113e9d79778ba384c02adfd72e79446f 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:83cb622eff48677f2ce2f9d550d58c9737f520bf60536e21a0a6465d3e3b8a39 -size 957364 +oid sha256:0a6d78d440415703b94c598e6f2551520cb543b71edc689143987ebc44807b3b +size 948654 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip index cb8cf85c3ed1e4005aafef7196f4e8a1928f909e..344d8cb5b00a349dfe02f719d089f1eb0a7a6889 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5028f09f92049a1c238298072cdba70e78de0446128528f17c6fa13c08379f4b +oid sha256:c3100cd7f8a801aac04d2c58754b8b5767e466c2f9adb9f5598c30335f2b8fb8 size 107975 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/website_leaderboard.csv index ed900c3d3f2df0c96ed91727ee80cc17ac49c7f9..f680a6b8392e345fe931fd78494d7ac947fec6fd 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/website_leaderboard.csv @@ -1,54 +1,54 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,25228.0,+0/-0,1.0,1.0,1.0,0.0,134.59,8.308,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,23917.0,+0/-0,0.934,2.0,2.0,1.729,276.98,4.897,0.0,False,CPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,22310.0,+0/-0,0.915,3.0,3.0,2.208,276.84,3.303,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,20483.0,+0/-0,0.883,4.0,4.0,3.024,134.59,2.819,0.0,False,GPU -4,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,19432.0,+0/-0,0.576,5.0,5.0,10.141,7855.1,0.565,0.0,False,CPU -5,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,18258.0,+0/-0,0.572,6.0,6.0,10.238,61.47,0.154,0.0,False,CPU -6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,17510.0,+0/-0,0.569,7.0,7.0,10.303,157.4,4.579,0.0,False,CPU -7,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,16643.0,+0/-0,0.567,8.0,8.0,10.333,157.4,0.401,0.0,False,CPU -8,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,15885.0,+0/-0,0.551,9.0,9.0,10.67,61.47,0.015,0.0,False,CPU -9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,15137.0,+0/-0,0.525,10.0,10.0,11.231,2687.46,0.361,0.0,False,CPU -10,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,14345.0,+0/-0,0.522,11.0,11.0,11.295,7855.1,0.048,0.0,False,CPU -11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,13528.0,+0/-0,0.517,12.0,12.0,11.389,2687.46,0.793,0.0,False,CPU -12,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,12768.0,+0/-0,0.515,13.0,13.0,11.441,456.33,1.171,0.0,False,CPU -13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,12036.0,+0/-0,0.501,14.0,14.0,11.722,456.33,0.047,0.0,False,CPU -14,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,11311.0,+0/-0,0.501,15.0,15.0,11.722,0.32,0.047,0.0,False,CPU -15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,10518.0,+0/-0,0.432,16.0,16.0,13.143,8.22,0.049,0.0,False,CPU -16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,9807.0,+0/-0,0.42,17.0,17.0,13.383,4378.82,25.595,0.0,False,GPU -17,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,9111.0,+0/-0,0.387,18.0,18.0,14.026,20.73,0.079,0.0,False,CPU -18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,8390.0,+0/-0,0.386,19.0,19.0,14.047,4378.82,2.216,0.0,False,GPU -19,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,7635.0,+0/-0,0.356,20.0,20.0,14.639,1.48,0.535,0.0,False,GPU -20,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,6953.0,+0/-0,0.292,21.0,21.0,15.857,2.79,1.057,0.0,False,GPU -21,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,6264.0,+0/-0,0.249,22.0,22.0,16.662,20.88,0.359,0.0,False,CPU -22,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,5562.0,+0/-0,0.111,23.0,23.0,19.143,20.88,0.026,0.0,False,CPU -23,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,5007.0,+0/-0,0.059,24.0,24.0,20.028,4390.96,275.957,0.0,False,GPU -24,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,4306.0,+0/-0,0.024,25.0,25.0,20.621,2341.91,0.027,0.0,False,CPU -25,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,3678.0,+0/-0,0.004,26.0,26.0,20.955,14.53,1.747,0.0,False,GPU -26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,3004.0,+0/-0,0.0,27.0,27.0,21.025,2341.91,0.172,0.0,False,CPU -27,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,2329.0,+0/-0,0.0,28.0,28.0,21.961,1993.57,0.469,0.0,False,GPU -28,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1701.0,+0/-0,0.0,29.0,29.0,22.049,1993.57,0.055,0.0,False,GPU -29,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+0/-0,0.0,30.0,30.0,24.001,0.18,0.007,0.0,False,CPU -30,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,445.0,+0/-0,0.0,31.0,31.0,24.06,4390.96,38.525,0.0,False,GPU -31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,-256.0,+0/-0,0.0,32.0,32.0,24.187,41.21,39.406,0.0,False,GPU -32,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,-946.0,+0/-0,0.0,33.0,33.0,24.591,1322.48,0.546,0.0,False,GPU -33,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,-1628.0,+0/-0,0.0,34.0,34.0,25.183,1322.48,0.055,0.0,False,GPU -34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,-2382.0,+0/-0,0.0,35.0,35.0,26.439,0.21,0.047,0.0,False,CPU -35,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,-3104.0,+0/-0,0.0,36.0,36.0,29.215,2009.11,3.767,0.0,False,GPU -36,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,-3800.0,+0/-0,0.0,37.0,37.0,30.84,2009.11,0.974,0.0,False,GPU -37,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,-4511.0,+0/-0,0.0,38.0,38.0,31.796,6.84,0.09,0.0,False,GPU -38,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,-5304.0,+0/-0,0.0,39.0,39.0,31.813,3.34,0.028,0.0,False,CPU -39,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,-6029.0,+0/-0,0.0,40.0,40.0,34.136,66.06,1.783,0.0,False,GPU -40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,-6760.0,+0/-0,0.0,41.0,41.0,38.819,6.51,0.162,0.0,False,GPU -41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,-7521.0,+0/-0,0.0,42.0,42.0,41.321,553.48,0.174,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,-8338.0,+0/-0,0.0,43.0,43.0,42.976,553.48,1.636,0.0,False,CPU -43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,-9130.0,+0/-0,0.0,44.0,44.0,50.562,0.08,0.017,0.0,False,CPU -44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,-9878.0,+0/-0,0.0,45.0,45.0,52.296,2.73,0.083,0.0,False,CPU -45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-10636.0,+0/-0,0.0,46.0,46.0,52.871,30.19,0.134,0.0,False,CPU -46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-11503.0,+0/-0,0.0,47.0,47.0,55.965,30.19,0.017,0.0,False,CPU -47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-12250.0,+0/-0,0.0,48.0,48.0,56.26,0.13,0.015,0.0,False,CPU -48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-13425.0,+0/-0,0.0,49.0,49.0,57.211,76.41,2.853,0.0,False,CPU -49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,-14476.0,+0/-0,0.0,50.0,50.0,57.769,3.14,0.921,0.0,False,GPU -50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,-16303.0,+0/-0,0.0,51.0,51.0,57.822,0.11,0.023,0.0,False,GPU -51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-17910.0,+0/-0,0.0,52.0,52.0,70.703,76.41,0.258,0.0,False,CPU -52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-19221.0,+0/-0,0.0,53.0,53.0,83.729,0.14,0.123,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),25228.0,+0/-0,0.0,1.0,1.0,1.0,134.59,8.308,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",23917.0,+0/-0,1.729,0.934,2.0,2.0,276.98,4.897,✔️,0.0,False,CPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",22310.0,+0/-0,2.208,0.915,3.0,3.0,276.84,3.303,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),20483.0,+0/-0,3.024,0.883,4.0,4.0,134.59,2.819,✔️,0.0,False,GPU +4,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),19432.0,+0/-0,10.141,0.576,5.0,5.0,7855.1,0.565,✔️,0.0,False,CPU +5,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),18258.0,+0/-0,10.238,0.572,6.0,6.0,61.47,0.154,✔️,0.0,False,CPU +6,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),17510.0,+0/-0,10.303,0.569,7.0,7.0,157.4,4.579,✔️,0.0,False,CPU +7,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),16643.0,+0/-0,10.333,0.567,8.0,8.0,157.4,0.401,✔️,0.0,False,CPU +8,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),15885.0,+0/-0,10.67,0.551,9.0,9.0,61.47,0.015,✔️,0.0,False,CPU +9,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),15137.0,+0/-0,11.231,0.525,10.0,10.0,2687.46,0.361,✔️,0.0,False,CPU +10,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),14345.0,+0/-0,11.295,0.522,11.0,11.0,7855.1,0.048,✔️,0.0,False,CPU +11,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),13528.0,+0/-0,11.389,0.517,12.0,12.0,2687.46,0.793,✔️,0.0,False,CPU +12,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),12768.0,+0/-0,11.441,0.515,13.0,13.0,456.33,1.171,✔️,0.0,False,CPU +13,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),12036.0,+0/-0,11.722,0.501,14.0,14.0,456.33,0.047,✔️,0.0,False,CPU +14,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),11311.0,+0/-0,11.722,0.501,15.0,15.0,0.32,0.047,✔️,0.0,False,CPU +15,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),10518.0,+0/-0,13.143,0.432,16.0,16.0,8.22,0.049,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),9807.0,+0/-0,13.383,0.42,17.0,17.0,4378.82,25.595,✔️,0.0,False,GPU +17,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),9111.0,+0/-0,14.026,0.387,18.0,18.0,20.73,0.079,✔️,0.0,False,CPU +18,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),8390.0,+0/-0,14.047,0.386,19.0,19.0,4378.82,2.216,✔️,0.0,False,GPU +19,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),7635.0,+0/-0,14.639,0.356,20.0,20.0,1.48,0.535,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),6953.0,+0/-0,15.857,0.292,21.0,21.0,2.79,1.057,✔️,0.0,False,GPU +21,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),6264.0,+0/-0,16.662,0.249,22.0,22.0,20.88,0.359,✔️,0.0,False,CPU +22,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),5562.0,+0/-0,19.143,0.111,23.0,23.0,20.88,0.026,✔️,0.0,False,CPU +23,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),5007.0,+0/-0,20.028,0.059,24.0,24.0,4390.96,275.957,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),4306.0,+0/-0,20.621,0.024,25.0,25.0,2341.91,0.027,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),3678.0,+0/-0,20.955,0.004,26.0,26.0,14.53,1.747,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),3004.0,+0/-0,21.025,0.0,27.0,27.0,2341.91,0.172,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),2329.0,+0/-0,21.961,0.0,28.0,28.0,1993.57,0.469,✔️,0.0,False,GPU +28,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1701.0,+0/-0,22.049,0.0,29.0,29.0,1993.57,0.055,✔️,0.0,False,GPU +29,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+0/-0,24.001,0.0,30.0,30.0,0.18,0.007,✔️,0.0,False,CPU +30,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),445.0,+0/-0,24.06,0.0,31.0,31.0,4390.96,38.525,✔️,0.0,False,GPU +31,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),-256.0,+0/-0,24.187,0.0,32.0,32.0,41.21,39.406,✔️,0.0,False,GPU +32,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),-946.0,+0/-0,24.591,0.0,33.0,33.0,1322.48,0.546,✔️,0.0,False,GPU +33,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),-1628.0,+0/-0,25.183,0.0,34.0,34.0,1322.48,0.055,✔️,0.0,False,GPU +34,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),-2382.0,+0/-0,26.439,0.0,35.0,35.0,0.21,0.047,✔️,0.0,False,CPU +35,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),-3104.0,+0/-0,29.215,0.0,36.0,36.0,2009.11,3.767,✔️,0.0,False,GPU +36,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),-3800.0,+0/-0,30.84,0.0,37.0,37.0,2009.11,0.974,✔️,0.0,False,GPU +37,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),-4511.0,+0/-0,31.796,0.0,38.0,38.0,6.84,0.09,✔️,0.0,False,GPU +38,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),-5304.0,+0/-0,31.813,0.0,39.0,39.0,3.34,0.028,✔️,0.0,False,CPU +39,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),-6029.0,+0/-0,34.136,0.0,40.0,40.0,66.06,1.783,➖,0.0,False,GPU +40,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),-6760.0,+0/-0,38.819,0.0,41.0,41.0,6.51,0.162,✔️,0.0,False,GPU +41,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),-7521.0,+0/-0,41.321,0.0,42.0,42.0,553.48,0.174,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),-8338.0,+0/-0,42.976,0.0,43.0,43.0,553.48,1.636,✔️,0.0,False,CPU +43,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),-9130.0,+0/-0,50.562,0.0,44.0,44.0,0.08,0.017,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),-9878.0,+0/-0,52.296,0.0,45.0,45.0,2.73,0.083,✔️,0.0,False,CPU +45,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),-10636.0,+0/-0,52.871,0.0,46.0,46.0,30.19,0.134,✔️,0.0,False,CPU +46,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),-11503.0,+0/-0,55.965,0.0,47.0,47.0,30.19,0.017,✔️,0.0,False,CPU +47,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),-12250.0,+0/-0,56.26,0.0,48.0,48.0,0.13,0.015,✔️,0.0,False,CPU +48,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),-13425.0,+0/-0,57.211,0.0,49.0,49.0,76.41,2.853,✔️,0.0,False,CPU +49,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),-14476.0,+0/-0,57.769,0.0,50.0,50.0,3.14,0.921,✔️,0.0,False,GPU +50,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),-16303.0,+0/-0,57.822,0.0,51.0,51.0,0.11,0.023,➖,0.0,False,GPU +51,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),-17910.0,+0/-0,70.703,0.0,52.0,52.0,76.41,0.258,✔️,0.0,False,CPU +52,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-19221.0,+0/-0,83.729,0.0,53.0,53.0,0.14,0.123,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/winrate_matrix.png.zip index a743b27742076b214468da0b918a64db6e626287..b706401afd56356367eb31c5e1fb43b7394accf6 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e3ad521dfab263587542afa8704e9433787dfa03aa2cd2dba62173808bf3c2a8 +oid sha256:d96450ca89ffc04b8498ec7d98857fb904961de1611b0d6e5de17de8d237ca08 size 1057411 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index a06bf256e58e8e0b859cd87422d4b7110f6f2273..118639deedc524385647c65b198cc300bc631099 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:82a851656225549bb63e59517a3c1806a7caed200d4ff955d0c90a4a523ccc0d +oid sha256:5ef2246f5f6815f60519a7e53de59bcd848f41cbd41bdd5e81dd5eeec9054530 size 456356 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip index e7c1e6c074cea47584507d90696f91223a1890b1..e374ce11c33babde741329caf088aadc52d179e2 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fc4220f2419fb9ed82a5d2820d1b8068b27c4e85ec87f8845783f9fd6d244669 -size 1022007 +oid sha256:c017602f05184744047042707352b4576c95f3e423b260076db517fe4a6fec60 +size 993438 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip index 1d6eba8878692769975533c916cf329b2d0a0cc9..4f6369e3758f2afe3082f21d8bf0fe3229523432 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8523dea13361b5c5e2f1a86c00d162188f15794b2be61928ec934ccb9291571d +oid sha256:599e9ccc99003612a37a33b2a05e7b0337df2597990fa0689e5016b2af4c276f size 168480 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/website_leaderboard.csv index cc8df56a8ff15d9cb8b60133646d09b4ad9d671a..ad9448e0369114b4df4e22d38d152e72a8590650 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1738.0,+465/-200,0.767,6.71,1.83,2.066,2837.96,9.546,0.0,False,GPU -1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1688.0,+223/-89,0.691,8.29,6.0,12.349,1060.41,10.928,0.0,False,GPU -2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1681.0,+348/-165,0.644,8.5,4.68,10.895,2837.96,1.134,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1584.0,+371/-164,0.542,12.36,5.08,9.756,10.97,0.856,0.0,False,GPU -4,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1583.0,+198/-94,0.521,12.43,7.3,10.653,6.19,1.301,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1521.0,+248/-121,0.458,15.43,5.06,14.183,4655.15,21.791,0.0,False,GPU -6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1509.0,+372/-201,0.511,16.07,5.99,17.32,2969.66,17.821,14.29,True,GPU -7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1485.0,+187/-130,0.405,17.43,14.17,14.644,4655.15,4.031,0.0,False,GPU -8,🧠🔁,Neural Network,[Mitra (default) [14.29% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1481.0,+229/-184,0.439,17.64,12.22,18.18,202.6,4.622,14.29,True,GPU -9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1474.0,+269/-114,0.432,18.0,5.08,15.834,6318.92,3.195,0.0,False,CPU -10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1457.0,+221/-107,0.329,19.0,15.22,15.543,3375.11,1.208,0.0,False,CPU -11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1447.0,+150/-114,0.357,19.57,13.01,18.233,5729.39,1.817,0.0,False,GPU -12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1440.0,+174/-63,0.302,20.0,16.67,16.169,3375.11,0.132,0.0,False,CPU -13,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1433.0,+218/-186,0.404,20.43,10.55,19.367,10182.26,399.569,0.0,False,GPU -14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1433.0,+163/-119,0.354,20.43,11.64,18.608,5729.39,0.373,0.0,False,GPU -15,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1428.0,+171/-184,0.339,20.71,8.46,17.962,2149.3,2.928,0.0,False,GPU -16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1421.0,+193/-93,0.273,21.14,15.22,17.093,1394.84,3.578,0.0,False,CPU -17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1411.0,+214/-96,0.278,21.71,14.33,17.176,1394.84,1.087,0.0,False,CPU -18,🧠⚡,Foundation Model,[TabPFNv2 (default) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1398.0,+221/-189,0.332,22.5,10.58,17.374,6.0,0.388,14.29,True,GPU -19,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1394.0,+259/-194,0.32,22.79,12.78,19.05,2969.66,0.477,14.29,True,GPU -20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1363.0,+102/-64,0.169,24.71,23.44,17.736,2342.58,5.131,0.0,False,CPU -21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1363.0,+102/-82,0.187,24.71,22.61,17.42,2342.58,1.065,0.0,False,CPU -22,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1359.0,+245/-202,0.345,25.0,5.55,20.345,10182.26,40.124,0.0,False,GPU -23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1357.0,+188/-136,0.26,25.14,14.57,20.76,19.13,0.146,0.0,False,GPU -24,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1347.0,+159/-119,0.188,25.79,21.43,18.825,6994.17,9.554,0.0,False,GPU -25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1339.0,+191/-199,0.267,26.29,5.46,19.587,2149.3,0.147,0.0,False,GPU -26,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1326.0,+160/-143,0.145,27.07,21.69,19.122,6994.17,0.583,0.0,False,GPU -27,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1325.0,+160/-127,0.214,27.14,20.85,20.198,326.23,4.282,0.0,False,GPU -28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1308.0,+189/-237,0.227,28.21,13.38,22.319,772.2,2.495,0.0,False,CPU -29,🧠⚡,Foundation Model,[TabICL (default) [14.29% IMPUTED]](https://arxiv.org/abs/2502.05564),✔️,1308.0,+256/-194,0.254,28.21,7.89,24.353,11.7,2.132,14.29,True,GPU -30,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1289.0,+122/-76,0.104,29.43,27.14,19.353,57.99,0.317,0.0,False,CPU -31,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1281.0,+180/-226,0.212,29.93,13.4,22.113,772.2,0.373,0.0,False,CPU -32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1276.0,+135/-105,0.122,30.29,23.95,21.656,44054.5,0.843,0.0,False,CPU -33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1273.0,+139/-119,0.135,30.43,27.11,21.802,3833.57,2.294,0.0,False,CPU -34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1264.0,+194/-273,0.215,31.0,17.71,22.276,731.01,2.021,0.0,False,CPU -35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1262.0,+252/-221,0.192,31.14,20.2,23.176,2750.22,13.569,0.0,False,CPU -36,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1230.0,+249/-225,0.244,33.14,9.78,24.042,51.52,40.219,0.0,False,GPU -37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1228.0,+182/-165,0.125,33.29,21.05,23.947,3833.57,0.22,0.0,False,CPU -38,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1228.0,+182/-225,0.148,33.29,24.78,22.488,5.15,0.526,0.0,False,CPU -39,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1223.0,+115/-138,0.094,33.57,30.68,23.403,14.99,0.113,0.0,False,CPU -40,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1221.0,+124/-126,0.111,33.71,29.37,23.217,44054.5,0.068,0.0,False,CPU -41,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1183.0,+134/-218,0.132,36.0,24.57,23.958,731.01,0.334,0.0,False,CPU -42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1170.0,+54/-72,0.0,36.71,35.84,21.807,5.38,0.624,0.0,False,CPU -43,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1170.0,+140/-177,0.076,36.71,32.22,23.313,20.63,7.532,0.0,False,GPU -44,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1163.0,+150/-236,0.067,37.14,32.08,24.905,9.4,1.444,0.0,False,GPU -45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1148.0,+220/-223,0.097,38.0,29.08,25.292,2750.22,1.239,0.0,False,CPU -46,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1106.0,+117/-141,0.008,40.29,38.08,28.196,19.64,0.478,0.0,False,GPU -47,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,1084.0,+250/-449,0.146,41.43,25.62,35.799,180.38,1.24,0.0,False,CPU -48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1008.0,+129/-210,0.0,45.0,43.49,30.387,14.87,0.391,0.0,False,CPU -49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+105/-290,0.0,45.36,43.81,37.213,0.78,0.155,0.0,False,CPU -50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,981.0,+192/-316,0.028,46.14,42.07,31.994,8.78,0.688,0.0,False,CPU -51,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,961.0,+74/-238,0.0,46.93,45.95,39.676,913.1,0.738,0.0,False,CPU -52,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,946.0,+82/-283,0.0,47.5,46.37,40.327,913.1,0.13,0.0,False,CPU -53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,929.0,+180/-311,0.004,48.14,44.97,39.305,0.77,0.156,0.0,False,CPU -54,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,853.0,+284/-1300,0.081,50.57,41.4,47.681,180.38,0.136,0.0,False,CPU -55,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,832.0,+80/-228,0.0,51.14,50.85,39.077,5.48,0.91,0.0,False,GPU -56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,736.0,+100/-302,0.0,53.29,52.96,46.939,4.65,0.14,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-464.0,+388/-687,0.0,58.0,58.0,69.099,0.58,0.071,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1738.0,+465/-200,2.066,0.767,6.71,1.83,2837.96,9.546,✔️,0.0,False,GPU +1,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1688.0,+223/-89,12.349,0.691,8.29,6.0,1060.41,10.928,✔️,0.0,False,GPU +2,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1681.0,+348/-165,10.895,0.644,8.5,4.68,2837.96,1.134,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1584.0,+371/-164,9.756,0.542,12.36,5.08,10.97,0.856,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1583.0,+198/-94,10.653,0.521,12.43,7.3,6.19,1.301,➖,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1521.0,+248/-121,14.183,0.458,15.43,5.06,4655.15,21.791,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1509.0,+372/-201,17.32,0.511,16.07,5.99,2969.66,17.821,✔️,14.29,True,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1485.0,+187/-130,14.644,0.405,17.43,14.17,4655.15,4.031,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[Mitra (default) [14.29% IMPUTED]](https://arxiv.org/abs/2510.21204),1481.0,+229/-184,18.18,0.439,17.64,12.22,202.6,4.622,✔️,14.29,True,GPU +9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1474.0,+269/-114,15.834,0.432,18.0,5.08,6318.92,3.195,✔️,0.0,False,CPU +10,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1457.0,+221/-107,15.543,0.329,19.0,15.22,3375.11,1.208,✔️,0.0,False,CPU +11,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1447.0,+150/-114,18.233,0.357,19.57,13.01,5729.39,1.817,✔️,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1440.0,+174/-63,16.169,0.302,20.0,16.67,3375.11,0.132,✔️,0.0,False,CPU +13,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1433.0,+218/-186,19.367,0.404,20.43,10.55,10182.26,399.569,✔️,0.0,False,GPU +14,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1433.0,+163/-119,18.608,0.354,20.43,11.64,5729.39,0.373,✔️,0.0,False,GPU +15,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1428.0,+171/-184,17.962,0.339,20.71,8.46,2149.3,2.928,✔️,0.0,False,GPU +16,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1421.0,+193/-93,17.093,0.273,21.14,15.22,1394.84,3.578,✔️,0.0,False,CPU +17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1411.0,+214/-96,17.176,0.278,21.71,14.33,1394.84,1.087,✔️,0.0,False,CPU +18,🧠⚡,Foundation Model,[TabPFNv2 (default) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1398.0,+221/-189,17.374,0.332,22.5,10.58,6.0,0.388,✔️,14.29,True,GPU +19,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [14.29% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1394.0,+259/-194,19.05,0.32,22.79,12.78,2969.66,0.477,✔️,14.29,True,GPU +20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1363.0,+102/-64,17.736,0.169,24.71,23.44,2342.58,5.131,✔️,0.0,False,CPU +21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1363.0,+102/-82,17.42,0.187,24.71,22.61,2342.58,1.065,✔️,0.0,False,CPU +22,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1359.0,+245/-202,20.345,0.345,25.0,5.55,10182.26,40.124,✔️,0.0,False,GPU +23,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1357.0,+188/-136,20.76,0.26,25.14,14.57,19.13,0.146,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1347.0,+159/-119,18.825,0.188,25.79,21.43,6994.17,9.554,✔️,0.0,False,GPU +25,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1339.0,+191/-199,19.587,0.267,26.29,5.46,2149.3,0.147,✔️,0.0,False,GPU +26,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1326.0,+160/-143,19.122,0.145,27.07,21.69,6994.17,0.583,✔️,0.0,False,GPU +27,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1325.0,+160/-127,20.198,0.214,27.14,20.85,326.23,4.282,➖,0.0,False,GPU +28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1308.0,+189/-237,22.319,0.227,28.21,13.38,772.2,2.495,✔️,0.0,False,CPU +29,🧠⚡,Foundation Model,[TabICL (default) [14.29% IMPUTED]](https://arxiv.org/abs/2502.05564),1308.0,+256/-194,24.353,0.254,28.21,7.89,11.7,2.132,✔️,14.29,True,GPU +30,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1289.0,+122/-76,19.353,0.104,29.43,27.14,57.99,0.317,✔️,0.0,False,CPU +31,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1281.0,+180/-226,22.113,0.212,29.93,13.4,772.2,0.373,✔️,0.0,False,CPU +32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1276.0,+135/-105,21.656,0.122,30.29,23.95,44054.5,0.843,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1273.0,+139/-119,21.802,0.135,30.43,27.11,3833.57,2.294,✔️,0.0,False,CPU +34,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1264.0,+194/-273,22.276,0.215,31.0,17.71,731.01,2.021,✔️,0.0,False,CPU +35,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1262.0,+252/-221,23.176,0.192,31.14,20.2,2750.22,13.569,✔️,0.0,False,CPU +36,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1230.0,+249/-225,24.042,0.244,33.14,9.78,51.52,40.219,✔️,0.0,False,GPU +37,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1228.0,+182/-165,23.947,0.125,33.29,21.05,3833.57,0.22,✔️,0.0,False,CPU +38,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1228.0,+182/-225,22.488,0.148,33.29,24.78,5.15,0.526,✔️,0.0,False,CPU +39,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1223.0,+115/-138,23.403,0.094,33.57,30.68,14.99,0.113,✔️,0.0,False,CPU +40,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1221.0,+124/-126,23.217,0.111,33.71,29.37,44054.5,0.068,✔️,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1183.0,+134/-218,23.958,0.132,36.0,24.57,731.01,0.334,✔️,0.0,False,CPU +42,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1170.0,+54/-72,21.807,0.0,36.71,35.84,5.38,0.624,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1170.0,+140/-177,23.313,0.076,36.71,32.22,20.63,7.532,✔️,0.0,False,GPU +44,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1163.0,+150/-236,24.905,0.067,37.14,32.08,9.4,1.444,✔️,0.0,False,GPU +45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1148.0,+220/-223,25.292,0.097,38.0,29.08,2750.22,1.239,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1106.0,+117/-141,28.196,0.008,40.29,38.08,19.64,0.478,✔️,0.0,False,GPU +47,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),1084.0,+250/-449,35.799,0.146,41.43,25.62,180.38,1.24,✔️,0.0,False,CPU +48,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1008.0,+129/-210,30.387,0.0,45.0,43.49,14.87,0.391,✔️,0.0,False,CPU +49,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+105/-290,37.213,0.0,45.36,43.81,0.78,0.155,✔️,0.0,False,CPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),981.0,+192/-316,31.994,0.028,46.14,42.07,8.78,0.688,✔️,0.0,False,CPU +51,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),961.0,+74/-238,39.676,0.0,46.93,45.95,913.1,0.738,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),946.0,+82/-283,40.327,0.0,47.5,46.37,913.1,0.13,✔️,0.0,False,CPU +53,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),929.0,+180/-311,39.305,0.004,48.14,44.97,0.77,0.156,✔️,0.0,False,CPU +54,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),853.0,+284/-1300,47.681,0.081,50.57,41.4,180.38,0.136,✔️,0.0,False,CPU +55,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),832.0,+80/-228,39.077,0.0,51.14,50.85,5.48,0.91,➖,0.0,False,GPU +56,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),736.0,+100/-302,46.939,0.0,53.29,52.96,4.65,0.14,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-464.0,+388/-687,69.099,0.0,58.0,58.0,0.58,0.071,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/winrate_matrix.png.zip index 445d629d4ff752840a9859c740c4195647fca088..37ef65eac2a3383690bfe1d0b4c340d40a9210df 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4e4bc54e85e022a6bb6c5ce09ee4a3f2aca1756bb98f4a49b0560677075644c3 +oid sha256:9b14f973849d0610508336e3cf37a5b7baba3b5749a01df77b92523541cce68a size 1680410 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index dd29dbebbf56a3c1cfa9ca5eac382bb9efa18b74..f284860070fde89fe42c530436ff00de1185ce45 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7cb50ec84ff7772a71f417b1307f323b19aa82231e98600d2c63c8ae6e8b875e +oid sha256:0f7ae8bbbc3e75c7d49b8d2e4e66accc5c121fffc70cf9c4a09870837940041f size 456644 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip index ad260ae7b7a0c0d685e9e2efef2eccb1cf20e884..ecabc30996850927bb9172d488865868af89d1cf 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:05ef32025b5c1a53b6b4d023ae5608490df3acb167848b47e48fbcc33d754b3f -size 1036337 +oid sha256:62c9926537293f74a3c1f74893e269fe8d879ad4a61581db4a1e4cd5392c5a86 +size 975573 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip index 43c8294c6fd51e053ebddd9f0e9aac92551cc52f..cb59ecdaf10f09e6b00bb0bec2055fc6bca66797 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:82cb5a176ada11a344c6b12f22b2bf17a6a42d4fb9ab369ee4726d3e819e75bc +oid sha256:184bd71f14228e7409c6749e7953b06e4ee609f934c3b1e43df6acf04a02c4ac size 125861 diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv index 4f05fb5c88ee5a61d5225392ab49c2f9a4cf43ae..a4c2ec6e6121067ab7b4a9e628750c998dfa2ed1 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/website_leaderboard.csv @@ -1,59 +1,59 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1997.0,+809/-100,0.875,3.17,1.58,2.191,2793.65,8.93,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1857.0,+337/-82,0.729,5.42,4.12,12.495,2793.65,1.02,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1746.0,+356/-135,0.689,8.17,5.69,14.332,1295.16,14.354,0.0,False,GPU -3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1716.0,+553/-144,0.632,9.08,4.46,11.083,9.88,0.825,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1688.0,+378/-116,0.596,10.0,5.22,14.698,3561.2,34.413,0.0,False,GPU -5,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1634.0,+358/-88,0.506,12.0,6.72,12.332,5.93,0.989,0.0,False,GPU -6,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1626.0,+204/-38,0.513,12.33,10.85,15.748,206.59,4.778,0.0,False,GPU -7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1622.0,+272/-96,0.534,12.5,4.43,16.24,4334.8,21.159,0.0,False,GPU -8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1583.0,+233/-63,0.472,14.17,12.85,16.59,4334.8,3.529,0.0,False,GPU -9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1522.0,+224/-109,0.384,17.0,14.03,17.857,3307.58,1.182,0.0,False,CPU -10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1506.0,+327/-108,0.373,17.83,11.35,16.716,3561.2,0.612,0.0,False,GPU -11,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1502.0,+371/-116,0.388,18.0,9.36,14.808,5.48,0.434,0.0,False,GPU -12,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1489.0,+378/-152,0.401,18.67,4.6,18.377,6700.69,3.268,0.0,False,CPU -13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1476.0,+201/-74,0.307,19.33,15.87,18.68,3307.58,0.108,0.0,False,CPU -14,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1445.0,+344/-241,0.382,21.0,9.92,22.477,9411.86,433.416,0.0,False,GPU -15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1433.0,+268/-147,0.283,21.67,15.18,21.223,5568.31,1.781,0.0,False,GPU -16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1418.0,+175/-44,0.219,22.5,21.18,19.816,2029.77,1.073,0.0,False,CPU -17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1412.0,+142/-39,0.197,22.83,22.15,20.228,2029.77,6.631,0.0,False,CPU -18,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1412.0,+262/-84,0.213,22.83,16.26,19.85,1280.01,5.479,0.0,False,CPU -19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1409.0,+244/-141,0.272,23.0,14.96,21.671,5568.31,0.372,0.0,False,GPU -20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1403.0,+285/-92,0.219,23.33,14.81,19.946,1280.01,1.143,0.0,False,CPU -21,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1398.0,+205/-127,0.219,23.58,19.93,21.447,6829.34,7.818,0.0,False,GPU -22,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1394.0,+268/-198,0.246,23.83,18.35,20.929,2059.45,3.769,0.0,False,GPU -23,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),✔️,1379.0,+437/-228,0.297,24.67,6.92,22.95,13.87,2.152,0.0,False,GPU -24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1375.0,+264/-171,0.169,24.92,20.15,21.793,6829.34,0.53,0.0,False,GPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1356.0,+255/-176,0.229,26.0,13.94,24.08,17.09,0.145,0.0,False,GPU -26,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1353.0,+340/-277,0.32,26.17,4.97,23.608,9411.86,42.924,0.0,False,GPU -27,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),➖,1319.0,+250/-164,0.191,28.17,20.83,23.403,410.08,4.104,0.0,False,GPU -28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1302.0,+373/-354,0.223,29.17,18.56,26.238,2721.87,14.595,0.0,False,CPU -29,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1296.0,+161/-125,0.109,29.5,26.85,22.348,155.57,0.254,0.0,False,CPU -30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1278.0,+292/-226,0.145,30.5,21.21,22.852,2059.45,0.127,0.0,False,GPU -31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1270.0,+202/-155,0.115,31.0,23.64,25.058,27108.48,0.742,0.0,False,CPU -32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1264.0,+222/-200,0.119,31.33,27.49,25.243,4370.77,2.162,0.0,False,CPU -33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1246.0,+253/-279,0.12,32.33,25.27,26.005,728.32,2.548,0.0,False,CPU -34,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1235.0,+397/-370,0.284,33.0,8.74,27.727,42.75,45.46,0.0,False,GPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1217.0,+342/-353,0.148,34.0,18.85,25.893,729.17,2.154,0.0,False,CPU -36,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1211.0,+276/-254,0.102,34.33,25.35,25.765,728.32,0.356,0.0,False,CPU -37,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1190.0,+175/-206,0.075,35.5,30.79,26.917,27108.48,0.066,0.0,False,CPU -38,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1190.0,+176/-212,0.088,35.5,30.85,25.981,19.2,6.922,0.0,False,GPU -39,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1181.0,+89/-95,0.0,36.0,35.1,24.783,4.93,0.671,0.0,False,CPU -40,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1181.0,+170/-206,0.035,36.0,34.18,27.166,11.78,0.099,0.0,False,CPU -41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1159.0,+197/-173,0.06,37.17,32.88,26.155,5.12,0.444,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1156.0,+276/-281,0.113,37.33,27.67,28.736,2721.87,1.334,0.0,False,CPU -43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1153.0,+228/-281,0.078,37.5,31.64,28.727,8.99,1.498,0.0,False,GPU -44,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1146.0,+149/-137,0.006,37.83,36.17,27.899,4370.77,0.22,0.0,False,CPU -45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1137.0,+257/-308,0.097,38.33,25.06,27.785,729.17,0.337,0.0,False,CPU -46,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1120.0,+154/-209,0.01,39.17,36.91,29.623,17.24,0.42,0.0,False,GPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+143/-432,0.0,44.67,42.99,37.954,0.74,0.148,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,979.0,+86/-362,0.0,45.5,44.67,38.521,909.67,0.893,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,962.0,+100/-303,0.0,46.17,45.13,39.28,909.67,0.107,0.0,False,CPU -50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,962.0,+232/-508,0.032,46.17,41.47,34.491,8.37,0.685,0.0,False,CPU -51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,939.0,+300/-564,0.037,47.0,40.48,41.716,155.52,1.033,0.0,False,CPU -52,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,934.0,+277/-430,0.004,47.17,43.75,39.387,0.76,0.153,0.0,False,CPU -53,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,924.0,+96/-253,0.0,47.5,47.01,35.184,14.57,0.363,0.0,False,CPU -54,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),➖,748.0,+71/-303,0.0,52.17,51.98,44.164,4.67,0.855,0.0,False,GPU -55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,723.0,+130/-462,0.0,52.67,52.34,44.545,4.62,0.135,0.0,False,CPU -56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,372.0,+132/-1205,0.0,56.33,56.29,55.52,155.52,0.12,0.0,False,CPU -57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,-548.0,+343/-1220,0.0,58.0,58.0,70.032,0.52,0.058,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1997.0,+809/-100,2.191,0.875,3.17,1.58,2793.65,8.93,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1857.0,+337/-82,12.495,0.729,5.42,4.12,2793.65,1.02,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1746.0,+356/-135,14.332,0.689,8.17,5.69,1295.16,14.354,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1716.0,+553/-144,11.083,0.632,9.08,4.46,9.88,0.825,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1688.0,+378/-116,14.698,0.596,10.0,5.22,3561.2,34.413,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1634.0,+358/-88,12.332,0.506,12.0,6.72,5.93,0.989,➖,0.0,False,GPU +6,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1626.0,+204/-38,15.748,0.513,12.33,10.85,206.59,4.778,✔️,0.0,False,GPU +7,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1622.0,+272/-96,16.24,0.534,12.5,4.43,4334.8,21.159,✔️,0.0,False,GPU +8,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1583.0,+233/-63,16.59,0.472,14.17,12.85,4334.8,3.529,✔️,0.0,False,GPU +9,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1522.0,+224/-109,17.857,0.384,17.0,14.03,3307.58,1.182,✔️,0.0,False,CPU +10,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1506.0,+327/-108,16.716,0.373,17.83,11.35,3561.2,0.612,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1502.0,+371/-116,14.808,0.388,18.0,9.36,5.48,0.434,✔️,0.0,False,GPU +12,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1489.0,+378/-152,18.377,0.401,18.67,4.6,6700.69,3.268,✔️,0.0,False,CPU +13,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1476.0,+201/-74,18.68,0.307,19.33,15.87,3307.58,0.108,✔️,0.0,False,CPU +14,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1445.0,+344/-241,22.477,0.382,21.0,9.92,9411.86,433.416,✔️,0.0,False,GPU +15,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1433.0,+268/-147,21.223,0.283,21.67,15.18,5568.31,1.781,✔️,0.0,False,GPU +16,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1418.0,+175/-44,19.816,0.219,22.5,21.18,2029.77,1.073,✔️,0.0,False,CPU +17,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1412.0,+142/-39,20.228,0.197,22.83,22.15,2029.77,6.631,✔️,0.0,False,CPU +18,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1412.0,+262/-84,19.85,0.213,22.83,16.26,1280.01,5.479,✔️,0.0,False,CPU +19,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1409.0,+244/-141,21.671,0.272,23.0,14.96,5568.31,0.372,✔️,0.0,False,GPU +20,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1403.0,+285/-92,19.946,0.219,23.33,14.81,1280.01,1.143,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1398.0,+205/-127,21.447,0.219,23.58,19.93,6829.34,7.818,✔️,0.0,False,GPU +22,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1394.0,+268/-198,20.929,0.246,23.83,18.35,2059.45,3.769,✔️,0.0,False,GPU +23,🧠⚡,Foundation Model,[TabICL (default)](https://arxiv.org/abs/2502.05564),1379.0,+437/-228,22.95,0.297,24.67,6.92,13.87,2.152,✔️,0.0,False,GPU +24,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1375.0,+264/-171,21.793,0.169,24.92,20.15,6829.34,0.53,✔️,0.0,False,GPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1356.0,+255/-176,24.08,0.229,26.0,13.94,17.09,0.145,✔️,0.0,False,GPU +26,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1353.0,+340/-277,23.608,0.32,26.17,4.97,9411.86,42.924,✔️,0.0,False,GPU +27,🧠⚡,Foundation Model,[BetaTabPFN (default)](https://arxiv.org/abs/2502.02527),1319.0,+250/-164,23.403,0.191,28.17,20.83,410.08,4.104,➖,0.0,False,GPU +28,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1302.0,+373/-354,26.238,0.223,29.17,18.56,2721.87,14.595,✔️,0.0,False,CPU +29,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1296.0,+161/-125,22.348,0.109,29.5,26.85,155.57,0.254,✔️,0.0,False,CPU +30,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1278.0,+292/-226,22.852,0.145,30.5,21.21,2059.45,0.127,✔️,0.0,False,GPU +31,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1270.0,+202/-155,25.058,0.115,31.0,23.64,27108.48,0.742,✔️,0.0,False,CPU +32,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1264.0,+222/-200,25.243,0.119,31.33,27.49,4370.77,2.162,✔️,0.0,False,CPU +33,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1246.0,+253/-279,26.005,0.12,32.33,25.27,728.32,2.548,✔️,0.0,False,CPU +34,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1235.0,+397/-370,27.727,0.284,33.0,8.74,42.75,45.46,✔️,0.0,False,GPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1217.0,+342/-353,25.893,0.148,34.0,18.85,729.17,2.154,✔️,0.0,False,CPU +36,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1211.0,+276/-254,25.765,0.102,34.33,25.35,728.32,0.356,✔️,0.0,False,CPU +37,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1190.0,+175/-206,26.917,0.075,35.5,30.79,27108.48,0.066,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1190.0,+176/-212,25.981,0.088,35.5,30.85,19.2,6.922,✔️,0.0,False,GPU +39,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1181.0,+89/-95,24.783,0.0,36.0,35.1,4.93,0.671,✔️,0.0,False,CPU +40,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1181.0,+170/-206,27.166,0.035,36.0,34.18,11.78,0.099,✔️,0.0,False,CPU +41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1159.0,+197/-173,26.155,0.06,37.17,32.88,5.12,0.444,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1156.0,+276/-281,28.736,0.113,37.33,27.67,2721.87,1.334,✔️,0.0,False,CPU +43,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1153.0,+228/-281,28.727,0.078,37.5,31.64,8.99,1.498,✔️,0.0,False,GPU +44,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1146.0,+149/-137,27.899,0.006,37.83,36.17,4370.77,0.22,✔️,0.0,False,CPU +45,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1137.0,+257/-308,27.785,0.097,38.33,25.06,729.17,0.337,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1120.0,+154/-209,29.623,0.01,39.17,36.91,17.24,0.42,✔️,0.0,False,GPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+143/-432,37.954,0.0,44.67,42.99,0.74,0.148,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),979.0,+86/-362,38.521,0.0,45.5,44.67,909.67,0.893,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),962.0,+100/-303,39.28,0.0,46.17,45.13,909.67,0.107,✔️,0.0,False,CPU +50,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),962.0,+232/-508,34.491,0.032,46.17,41.47,8.37,0.685,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),939.0,+300/-564,41.716,0.037,47.0,40.48,155.52,1.033,✔️,0.0,False,CPU +52,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),934.0,+277/-430,39.387,0.004,47.17,43.75,0.76,0.153,✔️,0.0,False,CPU +53,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),924.0,+96/-253,35.184,0.0,47.5,47.01,14.57,0.363,✔️,0.0,False,CPU +54,🧠⚡,Foundation Model,[TabFlex (default)](https://arxiv.org/abs/2506.05584),748.0,+71/-303,44.164,0.0,52.17,51.98,4.67,0.855,➖,0.0,False,GPU +55,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),723.0,+130/-462,44.545,0.0,52.67,52.34,4.62,0.135,✔️,0.0,False,CPU +56,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),372.0,+132/-1205,55.52,0.0,56.33,56.29,155.52,0.12,✔️,0.0,False,CPU +57,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),-548.0,+343/-1220,70.032,0.0,58.0,58.0,0.52,0.058,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip index e914d639b577e9d0dcb98b4a1950066961a8c575..9af854fe4ceb891a3a8ef250157b196f1a219df3 100644 --- a/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_multiclass/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:dfb2066c14637bcb9500b96764cc10c89af12f9bdf93c1a3c9ac51a3f7548333 +oid sha256:76fb2bc92d4d8abf3c42704a0af86d4692dd925e1bdbe4ff037276396e4297d8 size 1681180 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip index fd2d2f1ef586b78ae3c9c8c739f85b7251f6748f..f4e454eb2f688088e86471df8f04b143af190d0d 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b50ac480dcca750fbb098580041f64724a47a36f90d82e379b690813b343a643 +oid sha256:58947d7e8350c9eaa0f743fcff74de42d93efc9a6f4a3f18fefd1f2645d1cf37 size 442857 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip index ad8ba1d965b6e66b444ddaa5520489e126f3e7b3..5f55cb416b45055fe81a3af016c4a13424cde45e 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2d9bfb5f60e3e3973b0240dfd2851a66ac6b7eb86eef49947ee06f704de16399 -size 1008866 +oid sha256:7d9458d0b7dbd62842671c733157b291fdc86b830fba5a0563af2635466baaa3 +size 999231 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/tuning-impact-elo.png.zip index d4e7923dc02d09682e2ec84ee323abeff328e9c4..33c9f88cd146e2a665527f3e03ac2b4b1571ddfa 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6ac6848f4b0d7d0089cdcbd0dad246e47bf30b6035421265cd5aef579c978520 +oid sha256:57850ccb00e4965ad63f6c192e1dc8a67fc6c98114c4ba4758e878b2a280cfc4 size 147318 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/website_leaderboard.csv index e227eaaabdd5c649aa1df699c196a864faebaefa..7b6ff2f7933900472b48320c2cee1eb29dc6fdf4 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/website_leaderboard.csv @@ -1,56 +1,56 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1878.0,+277/-162,0.798,5.46,2.46,1.248,2043.44,8.369,0.0,False,GPU -1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1798.0,+204/-85,0.729,7.46,2.38,2.558,7125.58,336.907,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1798.0,+115/-60,0.652,7.46,4.37,2.649,616.43,4.041,0.0,False,GPU -3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1761.0,+145/-59,0.677,8.54,4.62,3.061,7125.58,38.573,0.0,False,GPU -4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1751.0,+194/-142,0.672,8.85,3.38,2.186,2043.44,0.781,0.0,False,GPU -5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1695.0,+96/-70,0.535,10.77,5.86,4.784,1489.94,5.79,0.0,False,CPU -6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1692.0,+106/-54,0.573,10.85,8.82,4.256,4057.12,10.478,0.0,False,GPU -7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1642.0,+178/-151,0.535,12.77,7.33,3.66,6.94,0.519,0.0,False,GPU -8,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1642.0,+150/-73,0.542,12.77,6.52,4.24,44.65,38.298,0.0,False,GPU -9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1608.0,+154/-138,0.501,14.15,5.57,6.753,3811.43,7.581,0.0,False,GPU -10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1566.0,+126/-126,0.43,16.0,11.47,5.413,4057.12,0.337,0.0,False,GPU -11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1560.0,+92/-63,0.332,16.31,13.64,7.383,686.46,5.477,0.0,False,CPU -12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1535.0,+82/-66,0.293,17.46,13.48,7.168,2895.38,1.317,0.0,False,CPU -13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1516.0,+109/-75,0.362,18.38,14.25,6.559,708.95,1.973,0.0,False,GPU -14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1506.0,+135/-95,0.314,18.85,15.04,5.831,4228.53,1.188,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1498.0,+109/-88,0.272,19.23,10.75,7.292,2895.38,0.073,0.0,False,CPU -16,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1481.0,+208/-182,0.407,20.08,5.06,13.045,5.49,0.754,0.0,False,GPU -17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1475.0,+108/-87,0.243,20.38,16.02,8.18,686.46,0.743,0.0,False,CPU -18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1464.0,+95/-64,0.268,20.92,17.27,7.123,708.95,0.069,0.0,False,GPU -19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1439.0,+58/-51,0.18,22.15,20.62,8.102,848.99,0.475,0.0,False,CPU -20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1437.0,+53/-46,0.181,22.23,21.12,8.118,848.99,2.378,0.0,False,CPU -21,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1427.0,+106/-89,0.221,22.77,19.49,9.404,16.07,0.29,0.0,False,GPU -22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1412.0,+113/-82,0.201,23.54,20.78,6.664,4228.53,0.125,0.0,False,GPU -23,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1404.0,+126/-138,0.265,23.92,14.23,9.279,3811.43,0.452,0.0,False,GPU -24,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1403.0,+211/-142,0.322,23.96,5.75,6.396,3805.62,10.413,46.15,True,GPU -25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1398.0,+116/-125,0.188,24.23,19.58,9.254,8.35,0.087,0.0,False,CPU -26,🧠🔁,Neural Network,[Mitra (default) [46.15% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1350.0,+167/-112,0.214,26.69,12.3,8.579,70.97,1.59,46.15,True,GPU -27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1336.0,+123/-115,0.142,27.46,24.43,8.326,13.9,0.123,0.0,False,GPU -28,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1308.0,+180/-154,0.218,28.88,16.23,7.904,3805.62,0.262,46.15,True,GPU -29,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1289.0,+126/-104,0.085,29.85,26.13,10.165,9.17,1.416,0.0,False,GPU -30,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1281.0,+113/-112,0.103,30.23,25.29,12.346,161.73,0.784,0.0,False,CPU -31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1277.0,+89/-103,0.071,30.46,27.55,9.496,4452.11,0.852,0.0,False,CPU -32,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1261.0,+114/-110,0.116,31.23,25.13,12.658,161.73,0.12,0.0,False,CPU -33,🧠⚡,Foundation Model,[TabPFNv2 (default) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1256.0,+184/-178,0.19,31.46,19.12,9.06,2.78,0.315,46.15,True,GPU -34,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1234.0,+173/-152,0.084,32.54,14.85,12.992,2.47,0.755,0.0,False,GPU -35,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1221.0,+114/-126,0.054,33.15,29.53,10.114,4452.11,0.089,0.0,False,CPU -36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1220.0,+72/-95,0.031,33.23,31.93,10.993,2.03,0.296,0.0,False,CPU -37,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1191.0,+94/-114,0.062,34.62,31.76,11.709,2.15,0.18,0.0,False,CPU -38,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1186.0,+155/-179,0.118,34.85,23.77,13.499,3017.28,0.248,0.0,False,CPU -39,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1179.0,+69/-87,0.016,35.15,33.41,13.32,526.17,0.77,0.0,False,CPU -40,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1144.0,+148/-191,0.118,36.77,19.83,14.059,3017.28,0.027,0.0,False,CPU -41,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1134.0,+88/-107,0.015,37.19,35.35,14.071,526.17,0.119,0.0,False,CPU -42,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1113.0,+101/-107,0.023,38.08,35.85,14.254,0.42,0.061,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1069.0,+83/-106,0.004,39.85,38.7,14.361,527.21,2.831,0.0,False,CPU -44,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1042.0,+93/-142,0.0,40.85,39.11,14.199,20.5,0.081,0.0,False,CPU -45,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1030.0,+109/-172,0.05,41.31,38.59,15.156,10.04,0.03,0.0,False,CPU -46,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1026.0,+102/-118,0.0,41.46,40.27,14.845,527.21,0.311,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+66/-87,0.0,42.35,41.83,15.385,0.63,0.064,0.0,False,CPU -48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,881.0,+101/-164,0.0,45.85,45.08,18.896,3.08,0.293,0.0,False,CPU -49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,859.0,+134/-176,0.0,46.38,45.51,20.364,90.87,0.906,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,705.0,+107/-204,0.0,49.38,49.19,22.511,90.87,0.055,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,571.0,+216/-360,0.0,51.15,50.64,29.184,0.25,0.044,0.0,False,CPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,425.0,+111/-304,0.0,52.58,52.52,36.25,177.04,0.14,0.0,False,CPU -53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,338.0,+142/-400,0.0,53.27,53.2,36.435,177.04,0.059,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,187.0,+134/-333,0.0,54.23,54.22,38.891,0.88,0.109,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1878.0,+277/-162,1.248,0.798,5.46,2.46,2043.44,8.369,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1798.0,+204/-85,2.558,0.729,7.46,2.38,7125.58,336.907,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1798.0,+115/-60,2.649,0.652,7.46,4.37,616.43,4.041,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1761.0,+145/-59,3.061,0.677,8.54,4.62,7125.58,38.573,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1751.0,+194/-142,2.186,0.672,8.85,3.38,2043.44,0.781,✔️,0.0,False,GPU +5,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1695.0,+96/-70,4.784,0.535,10.77,5.86,1489.94,5.79,✔️,0.0,False,CPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1692.0,+106/-54,4.256,0.573,10.85,8.82,4057.12,10.478,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1642.0,+178/-151,3.66,0.535,12.77,7.33,6.94,0.519,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1642.0,+150/-73,4.24,0.542,12.77,6.52,44.65,38.298,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1608.0,+154/-138,6.753,0.501,14.15,5.57,3811.43,7.581,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1566.0,+126/-126,5.413,0.43,16.0,11.47,4057.12,0.337,✔️,0.0,False,GPU +11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1560.0,+92/-63,7.383,0.332,16.31,13.64,686.46,5.477,✔️,0.0,False,CPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1535.0,+82/-66,7.168,0.293,17.46,13.48,2895.38,1.317,✔️,0.0,False,CPU +13,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1516.0,+109/-75,6.559,0.362,18.38,14.25,708.95,1.973,✔️,0.0,False,GPU +14,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1506.0,+135/-95,5.831,0.314,18.85,15.04,4228.53,1.188,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1498.0,+109/-88,7.292,0.272,19.23,10.75,2895.38,0.073,✔️,0.0,False,CPU +16,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1481.0,+208/-182,13.045,0.407,20.08,5.06,5.49,0.754,➖,0.0,False,GPU +17,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1475.0,+108/-87,8.18,0.243,20.38,16.02,686.46,0.743,✔️,0.0,False,CPU +18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1464.0,+95/-64,7.123,0.268,20.92,17.27,708.95,0.069,✔️,0.0,False,GPU +19,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1439.0,+58/-51,8.102,0.18,22.15,20.62,848.99,0.475,✔️,0.0,False,CPU +20,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1437.0,+53/-46,8.118,0.181,22.23,21.12,848.99,2.378,✔️,0.0,False,CPU +21,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1427.0,+106/-89,9.404,0.221,22.77,19.49,16.07,0.29,✔️,0.0,False,GPU +22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1412.0,+113/-82,6.664,0.201,23.54,20.78,4228.53,0.125,✔️,0.0,False,GPU +23,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1404.0,+126/-138,9.279,0.265,23.92,14.23,3811.43,0.452,✔️,0.0,False,GPU +24,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1403.0,+211/-142,6.396,0.322,23.96,5.75,3805.62,10.413,✔️,46.15,True,GPU +25,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1398.0,+116/-125,9.254,0.188,24.23,19.58,8.35,0.087,✔️,0.0,False,CPU +26,🧠⚡,Foundation Model,[Mitra (default) [46.15% IMPUTED]](https://arxiv.org/abs/2510.21204),1350.0,+167/-112,8.579,0.214,26.69,12.3,70.97,1.59,✔️,46.15,True,GPU +27,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1336.0,+123/-115,8.326,0.142,27.46,24.43,13.9,0.123,✔️,0.0,False,GPU +28,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1308.0,+180/-154,7.904,0.218,28.88,16.23,3805.62,0.262,✔️,46.15,True,GPU +29,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1289.0,+126/-104,10.165,0.085,29.85,26.13,9.17,1.416,✔️,0.0,False,GPU +30,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1281.0,+113/-112,12.346,0.103,30.23,25.29,161.73,0.784,✔️,0.0,False,CPU +31,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1277.0,+89/-103,9.496,0.071,30.46,27.55,4452.11,0.852,✔️,0.0,False,CPU +32,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1261.0,+114/-110,12.658,0.116,31.23,25.13,161.73,0.12,✔️,0.0,False,CPU +33,🧠⚡,Foundation Model,[TabPFNv2 (default) [46.15% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1256.0,+184/-178,9.06,0.19,31.46,19.12,2.78,0.315,✔️,46.15,True,GPU +34,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1234.0,+173/-152,12.992,0.084,32.54,14.85,2.47,0.755,✔️,0.0,False,GPU +35,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1221.0,+114/-126,10.114,0.054,33.15,29.53,4452.11,0.089,✔️,0.0,False,CPU +36,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1220.0,+72/-95,10.993,0.031,33.23,31.93,2.03,0.296,✔️,0.0,False,CPU +37,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1191.0,+94/-114,11.709,0.062,34.62,31.76,2.15,0.18,✔️,0.0,False,CPU +38,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1186.0,+155/-179,13.499,0.118,34.85,23.77,3017.28,0.248,✔️,0.0,False,CPU +39,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1179.0,+69/-87,13.32,0.016,35.15,33.41,526.17,0.77,✔️,0.0,False,CPU +40,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1144.0,+148/-191,14.059,0.118,36.77,19.83,3017.28,0.027,✔️,0.0,False,CPU +41,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1134.0,+88/-107,14.071,0.015,37.19,35.35,526.17,0.119,✔️,0.0,False,CPU +42,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1113.0,+101/-107,14.254,0.023,38.08,35.85,0.42,0.061,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1069.0,+83/-106,14.361,0.004,39.85,38.7,527.21,2.831,✔️,0.0,False,CPU +44,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1042.0,+93/-142,14.199,0.0,40.85,39.11,20.5,0.081,✔️,0.0,False,CPU +45,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1030.0,+109/-172,15.156,0.05,41.31,38.59,10.04,0.03,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1026.0,+102/-118,14.845,0.0,41.46,40.27,527.21,0.311,✔️,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+66/-87,15.385,0.0,42.35,41.83,0.63,0.064,✔️,0.0,False,CPU +48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),881.0,+101/-164,18.896,0.0,45.85,45.08,3.08,0.293,✔️,0.0,False,CPU +49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),859.0,+134/-176,20.364,0.0,46.38,45.51,90.87,0.906,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),705.0,+107/-204,22.511,0.0,49.38,49.19,90.87,0.055,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),571.0,+216/-360,29.184,0.0,51.15,50.64,0.25,0.044,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),425.0,+111/-304,36.25,0.0,52.58,52.52,177.04,0.14,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),338.0,+142/-400,36.435,0.0,53.27,53.2,177.04,0.059,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),187.0,+134/-333,38.891,0.0,54.23,54.22,0.88,0.109,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/winrate_matrix.png.zip index 53ef079a19e785f8aa87b4989c6509cd75fc99c9..314a855f897cde724e35293cd027fc800a091b96 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_all/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_all/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2a82e1d19971463577b44de7c5e892de47540ab8eb410c021ee45e38cf509158 +oid sha256:d827b9fd6dc3715e44eccd4345416eeb09dae7aa6d65061db10a527e9017d1d7 size 1701640 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip index 34e3865de5058b187da287b6ef6a3623fca0f691..721961c74828795949b74a1db9199f9e133ec4f6 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bc183c073380caee2a8d78e894b780b0397e83f82bb3124534c03daf683071bd +oid sha256:eca5e40122891bdaebd55352bab16cac3a713dbf3dbcea3a35e62ba124e87033 size 430652 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip index d3186d6a05168a73e7ac8c0a1edfbdf17d244e9e..8c78f812b874b0965ccd16761fd7a6189caf5b28 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6617321e0cfec776451c064538587cc84c0a4b31724e9fd6f8f4abd53574ccc4 -size 984135 +oid sha256:79e9ab80ab665190263dcfce02068ccad00f1173f418eb09277c70384228b1f8 +size 917540 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/tuning-impact-elo.png.zip index 3596f8595f447410176b1deabb1a3c30a70848ab..371a533611da751d5b0376b202773135094e8a50 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c35f0fd3382e45b0a83b6ff9b9295b5be17648c447a527baafcb236f76eaae58 +oid sha256:2352a4f60b7396cae96db8c068b1cf67e91ee92354640f18cf713f1d8e25af2c size 119314 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/website_leaderboard.csv index dbc54b0b2b4f8c3dd4bc8feaa1c2728697a1133a..46a7e63e483c8172dd9737f1097129e0df9c843a 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/website_leaderboard.csv @@ -1,52 +1,52 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,2061.0,+1014/-71,0.832,4.4,3.24,1.163,739.1,11.65,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,2027.0,+1086/-121,0.835,5.0,2.74,1.262,739.1,0.689,0.0,False,GPU -2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1950.0,+850/-36,0.699,6.6,5.71,2.284,2552.28,4.522,0.0,False,GPU -3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1916.0,+658/-8,0.655,7.4,3.4,3.209,474.94,11.837,0.0,False,CPU -4,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1885.0,+615/-13,0.612,8.2,6.09,3.513,400.37,2.16,0.0,False,GPU -5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1870.0,+930/-101,0.637,8.6,6.91,2.851,2068.17,1.355,0.0,False,GPU -6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1822.0,+1355/-177,0.687,10.0,2.25,2.083,4529.88,357.209,0.0,False,GPU -7,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1790.0,+1289/-174,0.65,11.0,4.14,2.39,4529.88,39.544,0.0,False,GPU -8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1783.0,+1058/-258,0.548,11.2,6.42,2.625,1.59,0.506,0.0,False,GPU -9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1688.0,+928/-198,0.485,14.4,10.08,4.112,2068.17,0.068,0.0,False,GPU -10,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1677.0,+780/-243,0.471,14.8,11.42,4.058,2552.28,0.201,0.0,False,GPU -11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1677.0,+503/-139,0.377,14.8,11.88,5.836,223.05,9.707,0.0,False,CPU -12,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1671.0,+1088/-218,0.526,15.0,6.19,3.203,41.5,38.298,0.0,False,GPU -13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1660.0,+548/-97,0.349,15.4,12.03,6.007,223.05,0.793,0.0,False,CPU -14,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1649.0,+524/-91,0.418,15.8,13.49,4.481,438.55,0.52,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1644.0,+510/-102,0.3,16.0,10.62,6.235,1696.82,0.538,0.0,False,CPU -16,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1617.0,+754/-226,0.336,17.0,12.43,5.187,3456.17,0.611,0.0,False,GPU -17,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1590.0,+500/-169,0.28,18.0,7.25,6.475,1696.82,0.055,0.0,False,CPU -18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1559.0,+428/-97,0.253,19.2,17.73,5.477,438.55,0.035,0.0,False,GPU -19,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1553.0,+728/-163,0.287,19.4,16.95,5.3,6.87,0.098,0.0,False,GPU -20,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1537.0,+750/-208,0.306,20.0,4.21,4.7,2.07,0.754,0.0,False,GPU -21,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1521.0,+561/-275,0.27,20.6,16.66,7.061,4.99,0.047,0.0,False,CPU -22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1505.0,+528/-187,0.183,21.2,19.63,6.188,3456.17,0.037,0.0,False,GPU -23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1494.0,+356/-58,0.189,21.6,20.89,6.853,289.35,1.798,0.0,False,CPU -24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1484.0,+386/-70,0.181,22.0,21.09,6.921,289.35,0.36,0.0,False,CPU -25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1435.0,+648/-216,0.129,23.8,21.42,7.402,5.37,0.025,0.0,False,GPU -26,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1353.0,+354/-228,0.137,26.8,24.63,8.025,0.57,0.074,0.0,False,CPU -27,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1302.0,+357/-139,0.002,28.6,27.72,8.909,2.32,0.093,0.0,False,GPU -28,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1297.0,+375/-186,0.018,28.8,28.08,8.225,0.66,0.213,0.0,False,CPU -29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1274.0,+225/-245,0.064,29.6,27.65,8.161,4108.24,0.479,0.0,False,CPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1250.0,+508/-284,0.028,30.4,28.51,9.603,6.54,0.284,0.0,False,GPU -31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1221.0,+204/-296,0.029,31.4,29.47,9.253,4108.24,0.037,0.0,False,CPU -32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1172.0,+353/-451,0.023,33.0,31.22,10.288,35.47,0.502,0.0,False,CPU -33,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1147.0,+417/-890,0.144,33.8,25.8,13.625,1820.58,0.133,0.0,False,CPU -34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1140.0,+314/-411,0.006,34.0,32.83,10.454,35.47,0.054,0.0,False,CPU -35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1134.0,+107/-401,0.0,34.2,33.96,10.053,79.96,0.571,0.0,False,CPU -36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1124.0,+128/-390,0.0,34.5,34.09,10.217,79.96,0.053,0.0,False,CPU -37,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1061.0,+165/-350,0.0,36.4,35.28,13.469,14.05,0.029,0.0,False,CPU -38,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1033.0,+387/-674,0.006,37.2,35.12,12.12,0.18,0.054,0.0,False,CPU -39,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1026.0,+316/-953,0.1,37.4,34.8,14.242,1820.58,0.012,0.0,False,CPU -40,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+78/-318,0.0,38.1,37.96,11.35,0.42,0.053,0.0,False,CPU -41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,996.0,+363/-1057,0.12,38.2,34.58,14.681,3.47,0.01,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,871.0,+51/-752,0.0,41.2,40.95,14.269,317.48,0.719,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,842.0,+43/-769,0.0,41.8,41.54,14.827,317.48,0.127,0.0,False,CPU -44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,800.0,+94/-889,0.0,42.6,42.42,18.666,142.18,1.027,0.0,False,CPU -45,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,662.0,+74/-902,0.0,44.8,44.73,21.083,142.18,0.057,0.0,False,CPU -46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,662.0,+107/-831,0.0,44.8,44.56,18.199,1.33,0.126,0.0,False,CPU -47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,361.0,+132/-1452,0.0,47.6,47.49,28.294,0.06,0.036,0.0,False,CPU -48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,145.0,+92/-1782,0.0,48.8,48.8,39.482,48.35,0.054,0.0,False,CPU -49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-119.0,+197/-2129,0.0,50.0,49.99,39.829,48.35,0.013,0.0,False,CPU -50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,-276.0,+185/-2881,0.0,50.6,50.6,42.789,0.13,0.02,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),2061.0,+1014/-71,1.163,0.832,4.4,3.24,739.1,11.65,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),2027.0,+1086/-121,1.262,0.835,5.0,2.74,739.1,0.689,✔️,0.0,False,GPU +2,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1950.0,+850/-36,2.284,0.699,6.6,5.71,2552.28,4.522,✔️,0.0,False,GPU +3,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1916.0,+658/-8,3.209,0.655,7.4,3.4,474.94,11.837,✔️,0.0,False,CPU +4,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1885.0,+615/-13,3.513,0.612,8.2,6.09,400.37,2.16,✔️,0.0,False,GPU +5,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1870.0,+930/-101,2.851,0.637,8.6,6.91,2068.17,1.355,✔️,0.0,False,GPU +6,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1822.0,+1355/-177,2.083,0.687,10.0,2.25,4529.88,357.209,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1790.0,+1289/-174,2.39,0.65,11.0,4.14,4529.88,39.544,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1783.0,+1058/-258,2.625,0.548,11.2,6.42,1.59,0.506,✔️,0.0,False,GPU +9,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1688.0,+928/-198,4.112,0.485,14.4,10.08,2068.17,0.068,✔️,0.0,False,GPU +10,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1677.0,+780/-243,4.058,0.471,14.8,11.42,2552.28,0.201,✔️,0.0,False,GPU +11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1677.0,+503/-139,5.836,0.377,14.8,11.88,223.05,9.707,✔️,0.0,False,CPU +12,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1671.0,+1088/-218,3.203,0.526,15.0,6.19,41.5,38.298,✔️,0.0,False,GPU +13,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1660.0,+548/-97,6.007,0.349,15.4,12.03,223.05,0.793,✔️,0.0,False,CPU +14,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1649.0,+524/-91,4.481,0.418,15.8,13.49,438.55,0.52,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1644.0,+510/-102,6.235,0.3,16.0,10.62,1696.82,0.538,✔️,0.0,False,CPU +16,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1617.0,+754/-226,5.187,0.336,17.0,12.43,3456.17,0.611,✔️,0.0,False,GPU +17,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1590.0,+500/-169,6.475,0.28,18.0,7.25,1696.82,0.055,✔️,0.0,False,CPU +18,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1559.0,+428/-97,5.477,0.253,19.2,17.73,438.55,0.035,✔️,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1553.0,+728/-163,5.3,0.287,19.4,16.95,6.87,0.098,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1537.0,+750/-208,4.7,0.306,20.0,4.21,2.07,0.754,➖,0.0,False,GPU +21,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1521.0,+561/-275,7.061,0.27,20.6,16.66,4.99,0.047,✔️,0.0,False,CPU +22,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1505.0,+528/-187,6.188,0.183,21.2,19.63,3456.17,0.037,✔️,0.0,False,GPU +23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1494.0,+356/-58,6.853,0.189,21.6,20.89,289.35,1.798,✔️,0.0,False,CPU +24,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1484.0,+386/-70,6.921,0.181,22.0,21.09,289.35,0.36,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1435.0,+648/-216,7.402,0.129,23.8,21.42,5.37,0.025,✔️,0.0,False,GPU +26,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1353.0,+354/-228,8.025,0.137,26.8,24.63,0.57,0.074,✔️,0.0,False,CPU +27,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1302.0,+357/-139,8.909,0.002,28.6,27.72,2.32,0.093,✔️,0.0,False,GPU +28,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1297.0,+375/-186,8.225,0.018,28.8,28.08,0.66,0.213,✔️,0.0,False,CPU +29,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1274.0,+225/-245,8.161,0.064,29.6,27.65,4108.24,0.479,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1250.0,+508/-284,9.603,0.028,30.4,28.51,6.54,0.284,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1221.0,+204/-296,9.253,0.029,31.4,29.47,4108.24,0.037,✔️,0.0,False,CPU +32,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1172.0,+353/-451,10.288,0.023,33.0,31.22,35.47,0.502,✔️,0.0,False,CPU +33,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1147.0,+417/-890,13.625,0.144,33.8,25.8,1820.58,0.133,✔️,0.0,False,CPU +34,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1140.0,+314/-411,10.454,0.006,34.0,32.83,35.47,0.054,✔️,0.0,False,CPU +35,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1134.0,+107/-401,10.053,0.0,34.2,33.96,79.96,0.571,✔️,0.0,False,CPU +36,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1124.0,+128/-390,10.217,0.0,34.5,34.09,79.96,0.053,✔️,0.0,False,CPU +37,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1061.0,+165/-350,13.469,0.0,36.4,35.28,14.05,0.029,✔️,0.0,False,CPU +38,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1033.0,+387/-674,12.12,0.006,37.2,35.12,0.18,0.054,✔️,0.0,False,CPU +39,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1026.0,+316/-953,14.242,0.1,37.4,34.8,1820.58,0.012,✔️,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+78/-318,11.35,0.0,38.1,37.96,0.42,0.053,✔️,0.0,False,CPU +41,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),996.0,+363/-1057,14.681,0.12,38.2,34.58,3.47,0.01,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),871.0,+51/-752,14.269,0.0,41.2,40.95,317.48,0.719,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),842.0,+43/-769,14.827,0.0,41.8,41.54,317.48,0.127,✔️,0.0,False,CPU +44,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),800.0,+94/-889,18.666,0.0,42.6,42.42,142.18,1.027,✔️,0.0,False,CPU +45,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),662.0,+74/-902,21.083,0.0,44.8,44.73,142.18,0.057,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),662.0,+107/-831,18.199,0.0,44.8,44.56,1.33,0.126,✔️,0.0,False,CPU +47,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),361.0,+132/-1452,28.294,0.0,47.6,47.49,0.06,0.036,✔️,0.0,False,CPU +48,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),145.0,+92/-1782,39.482,0.0,48.8,48.8,48.35,0.054,✔️,0.0,False,CPU +49,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),-119.0,+197/-2129,39.829,0.0,50.0,49.99,48.35,0.013,✔️,0.0,False,CPU +50,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),-276.0,+185/-2881,42.789,0.0,50.6,50.6,0.13,0.02,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/winrate_matrix.png.zip index 4592db7088a316aea22b530972c07d31c944a8e8..c9ee70930761bab92116f38e9bb15d5da129797a 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_medium/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:16545df5ba8fac5ff84297fb83742815f52924c8c210dbf293fec1eec1c6aee9 +oid sha256:8b24bec781fb0ff276e1e91e681472c7916ffdd71ab904ecd6bfe910f0030436 size 1244143 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip index 42fe7bbf0990bb2a2b2976973c024cb37278e120..a66a426a1ea2d2d43dbf52c8d58e39010aa8f693 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c2329eab6fd3f94694a7e4df84c1a8da6122a5d25488fde802f0b9b242c76d96 +oid sha256:91438b089230ed8d01b23828976c184e416bd7cdc717743f5ac54356f4f1ba29 size 449846 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip index ad1754fc0f1cf3973d8427499dd79cd546aa433e..961a5cdae84636659065451f3fd4094d71aa43d3 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8cb2d1519d325164a2e9eefa8edacf5116dfc39f2aac7087b5bc2ea77d78fe26 -size 1058098 +oid sha256:1b3f59902166219b8d17fc77a944a58e13574015e6db771f42e34d90d6736fa2 +size 1051427 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/tuning-impact-elo.png.zip index ab8f67fe5a5f536ec2430ae26f09373f84408d54..77275335e1cb3573d6eb14b725254c870aaf735c 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b3ae294b248d5d62805ac74a762fac39a3b398eb8f9c3e09ecfab5dae16af7f7 +oid sha256:c40e5cd50d050b4fd0512f1b5ee53ddf9fd865cbceda45700df563f7976487d2 size 159497 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/website_leaderboard.csv index c5f3cc5175bd0031c5bdfab6b5cd209538c58bb8..84c89880db752e30929ad764896b0f5f71dd6e7e 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/website_leaderboard.csv @@ -1,56 +1,56 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1891.0,+265/-104,0.738,5.88,2.47,2.855,9830.47,232.266,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1880.0,+464/-211,0.767,6.12,2.14,1.301,2355.62,7.55,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1846.0,+282/-80,0.656,7.0,3.72,2.109,716.17,4.181,0.0,False,GPU -3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1846.0,+180/-36,0.674,7.0,4.99,3.48,9830.47,22.221,0.0,False,GPU -4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1713.0,+345/-190,0.561,11.25,3.97,2.764,2355.62,0.879,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1710.0,+158/-52,0.533,11.38,6.74,4.888,51.27,43.085,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1687.0,+130/-51,0.512,12.25,10.65,5.134,4918.07,35.607,0.0,False,GPU -7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1672.0,+142/-65,0.442,12.88,10.65,5.768,4919.72,5.372,0.0,False,CPU -8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1651.0,+309/-190,0.51,13.75,8.04,4.306,8.24,0.532,0.0,False,GPU -9,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1644.0,+544/-239,0.523,14.06,3.75,3.301,6437.34,56.116,12.5,True,GPU -10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1577.0,+165/-113,0.381,17.0,12.55,6.226,4918.07,0.949,0.0,False,GPU -11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1572.0,+116/-77,0.272,17.25,15.04,8.35,1206.7,4.573,0.0,False,CPU -12,🧠🔁,Neural Network,[Mitra (default) [12.50% IMPUTED]](https://arxiv.org/abs/2510.21204),✔️,1552.0,+246/-167,0.348,18.19,8.58,6.847,94.13,2.176,12.5,True,GPU -13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1548.0,+124/-62,0.252,18.38,16.21,7.752,3495.99,1.462,0.0,False,CPU -14,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1538.0,+206/-133,0.358,18.88,5.49,9.546,9006.35,8.016,0.0,False,GPU -15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1515.0,+188/-91,0.233,20.0,15.41,7.803,3495.99,0.148,0.0,False,CPU -16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1515.0,+141/-90,0.291,20.0,14.78,7.857,1672.82,2.447,0.0,False,GPU -17,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1515.0,+153/-109,0.277,20.0,17.31,6.233,4322.16,1.598,0.0,False,GPU -18,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1513.0,+408/-258,0.453,20.12,5.79,18.26,5.86,0.71,0.0,False,GPU -19,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1476.0,+150/-74,0.231,22.0,16.98,8.152,1672.82,0.097,0.0,False,GPU -20,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1475.0,+382/-238,0.355,22.06,11.85,5.751,6437.34,0.557,12.5,True,GPU -21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1472.0,+94/-57,0.136,22.25,20.33,8.84,1277.27,0.648,0.0,False,CPU -22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1465.0,+89/-51,0.133,22.62,21.26,8.909,1277.27,3.746,0.0,False,CPU -23,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1448.0,+125/-102,0.144,23.5,20.2,9.538,1206.7,0.689,0.0,False,CPU -24,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1421.0,+141/-132,0.154,24.88,21.51,11.969,25.13,0.294,0.0,False,GPU -25,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1419.0,+162/-120,0.179,25.0,21.57,6.961,4322.16,0.198,0.0,False,GPU -26,🧠⚡,Foundation Model,[TabPFNv2 (default) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),✔️,1400.0,+328/-281,0.308,25.94,14.4,7.63,8.68,0.726,12.5,True,GPU -27,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1389.0,+147/-121,0.102,26.5,21.98,10.625,10.03,0.115,0.0,False,CPU -28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1360.0,+148/-186,0.138,28.0,22.48,13.632,644.11,0.953,0.0,False,CPU -29,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1340.0,+168/-199,0.169,29.0,21.79,14.036,644.11,0.187,0.0,False,CPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1340.0,+150/-102,0.113,29.0,24.67,10.515,22.41,7.82,0.0,False,GPU -31,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1328.0,+188/-190,0.12,29.62,16.81,12.542,9006.35,0.527,0.0,False,GPU -32,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1326.0,+164/-137,0.12,29.75,26.79,8.903,25.53,0.154,0.0,False,GPU -33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1311.0,+132/-150,0.067,30.5,27.26,10.331,5194.73,1.75,0.0,False,CPU -34,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1257.0,+190/-194,0.066,33.25,29.07,10.652,5194.73,0.121,0.0,False,CPU -35,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1241.0,+217/-212,0.097,34.0,22.26,13.421,3991.28,0.402,0.0,False,CPU -36,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1222.0,+233/-228,0.122,34.88,15.45,13.945,3991.28,0.034,0.0,False,CPU -37,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1220.0,+214/-280,0.121,35.0,11.51,15.544,7.0,1.362,0.0,False,GPU -38,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1214.0,+100/-178,0.026,35.25,32.69,15.363,657.99,1.135,0.0,False,CPU -39,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1198.0,+92/-125,0.015,36.0,34.93,12.723,4.54,0.374,0.0,False,CPU -40,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1175.0,+76/-124,0.006,37.0,35.99,14.418,2150.84,7.995,0.0,False,CPU -41,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1160.0,+97/-168,0.018,37.62,35.75,15.588,0.46,0.117,0.0,False,CPU -42,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1148.0,+114/-214,0.025,38.12,35.51,16.48,657.99,0.147,0.0,False,CPU -43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1121.0,+107/-158,0.0,39.25,38.02,14.856,2150.84,0.566,0.0,False,CPU -44,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1114.0,+86/-123,0.0,39.5,38.78,14.012,4.08,0.255,0.0,False,CPU -45,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1055.0,+114/-185,0.0,41.75,40.45,15.454,12.71,0.039,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1030.0,+146/-217,0.0,42.62,41.09,14.656,22.25,0.131,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+92/-204,0.0,43.62,42.96,17.907,0.69,0.125,0.0,False,CPU -48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,988.0,+106/-216,0.0,44.0,43.14,19.332,6.77,0.511,0.0,False,CPU -49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,907.0,+147/-333,0.0,46.25,44.95,21.426,88.79,0.572,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,734.0,+153/-330,0.0,49.75,49.48,23.404,88.79,0.052,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,658.0,+224/-609,0.0,50.88,50.11,29.74,0.43,0.047,0.0,False,CPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,523.0,+100/-474,0.0,52.44,52.35,34.23,742.99,0.364,0.0,False,CPU -53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,484.0,+130/-555,0.0,52.81,52.72,34.314,742.99,0.097,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,334.0,+99/-505,0.0,54.0,53.99,36.454,3.98,0.12,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1891.0,+265/-104,2.855,0.738,5.88,2.47,9830.47,232.266,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1880.0,+464/-211,1.301,0.767,6.12,2.14,2355.62,7.55,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1846.0,+282/-80,2.109,0.656,7.0,3.72,716.17,4.181,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1846.0,+180/-36,3.48,0.674,7.0,4.99,9830.47,22.221,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1713.0,+345/-190,2.764,0.561,11.25,3.97,2355.62,0.879,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1710.0,+158/-52,4.888,0.533,11.38,6.74,51.27,43.085,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1687.0,+130/-51,5.134,0.512,12.25,10.65,4918.07,35.607,✔️,0.0,False,GPU +7,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1672.0,+142/-65,5.768,0.442,12.88,10.65,4919.72,5.372,✔️,0.0,False,CPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1651.0,+309/-190,4.306,0.51,13.75,8.04,8.24,0.532,✔️,0.0,False,GPU +9,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1644.0,+544/-239,3.301,0.523,14.06,3.75,6437.34,56.116,✔️,12.5,True,GPU +10,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1577.0,+165/-113,6.226,0.381,17.0,12.55,4918.07,0.949,✔️,0.0,False,GPU +11,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1572.0,+116/-77,8.35,0.272,17.25,15.04,1206.7,4.573,✔️,0.0,False,CPU +12,🧠⚡,Foundation Model,[Mitra (default) [12.50% IMPUTED]](https://arxiv.org/abs/2510.21204),1552.0,+246/-167,6.847,0.348,18.19,8.58,94.13,2.176,✔️,12.5,True,GPU +13,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1548.0,+124/-62,7.752,0.252,18.38,16.21,3495.99,1.462,✔️,0.0,False,CPU +14,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1538.0,+206/-133,9.546,0.358,18.88,5.49,9006.35,8.016,✔️,0.0,False,GPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1515.0,+188/-91,7.803,0.233,20.0,15.41,3495.99,0.148,✔️,0.0,False,CPU +16,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1515.0,+141/-90,7.857,0.291,20.0,14.78,1672.82,2.447,✔️,0.0,False,GPU +17,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1515.0,+153/-109,6.233,0.277,20.0,17.31,4322.16,1.598,✔️,0.0,False,GPU +18,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1513.0,+408/-258,18.26,0.453,20.12,5.79,5.86,0.71,➖,0.0,False,GPU +19,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1476.0,+150/-74,8.152,0.231,22.0,16.98,1672.82,0.097,✔️,0.0,False,GPU +20,🧠⚡,Foundation Model,[TabPFNv2 (tuned) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1475.0,+382/-238,5.751,0.355,22.06,11.85,6437.34,0.557,✔️,12.5,True,GPU +21,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1472.0,+94/-57,8.84,0.136,22.25,20.33,1277.27,0.648,✔️,0.0,False,CPU +22,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1465.0,+89/-51,8.909,0.133,22.62,21.26,1277.27,3.746,✔️,0.0,False,CPU +23,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1448.0,+125/-102,9.538,0.144,23.5,20.2,1206.7,0.689,✔️,0.0,False,CPU +24,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1421.0,+141/-132,11.969,0.154,24.88,21.51,25.13,0.294,✔️,0.0,False,GPU +25,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1419.0,+162/-120,6.961,0.179,25.0,21.57,4322.16,0.198,✔️,0.0,False,GPU +26,🧠⚡,Foundation Model,[TabPFNv2 (default) [12.50% IMPUTED]](https://www.nature.com/articles/s41586-024-08328-6),1400.0,+328/-281,7.63,0.308,25.94,14.4,8.68,0.726,✔️,12.5,True,GPU +27,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1389.0,+147/-121,10.625,0.102,26.5,21.98,10.03,0.115,✔️,0.0,False,CPU +28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1360.0,+148/-186,13.632,0.138,28.0,22.48,644.11,0.953,✔️,0.0,False,CPU +29,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1340.0,+168/-199,14.036,0.169,29.0,21.79,644.11,0.187,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1340.0,+150/-102,10.515,0.113,29.0,24.67,22.41,7.82,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1328.0,+188/-190,12.542,0.12,29.62,16.81,9006.35,0.527,✔️,0.0,False,GPU +32,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1326.0,+164/-137,8.903,0.12,29.75,26.79,25.53,0.154,✔️,0.0,False,GPU +33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1311.0,+132/-150,10.331,0.067,30.5,27.26,5194.73,1.75,✔️,0.0,False,CPU +34,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1257.0,+190/-194,10.652,0.066,33.25,29.07,5194.73,0.121,✔️,0.0,False,CPU +35,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1241.0,+217/-212,13.421,0.097,34.0,22.26,3991.28,0.402,✔️,0.0,False,CPU +36,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1222.0,+233/-228,13.945,0.122,34.88,15.45,3991.28,0.034,✔️,0.0,False,CPU +37,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1220.0,+214/-280,15.544,0.121,35.0,11.51,7.0,1.362,✔️,0.0,False,GPU +38,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1214.0,+100/-178,15.363,0.026,35.25,32.69,657.99,1.135,✔️,0.0,False,CPU +39,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1198.0,+92/-125,12.723,0.015,36.0,34.93,4.54,0.374,✔️,0.0,False,CPU +40,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1175.0,+76/-124,14.418,0.006,37.0,35.99,2150.84,7.995,✔️,0.0,False,CPU +41,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1160.0,+97/-168,15.588,0.018,37.62,35.75,0.46,0.117,✔️,0.0,False,CPU +42,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1148.0,+114/-214,16.48,0.025,38.12,35.51,657.99,0.147,✔️,0.0,False,CPU +43,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1121.0,+107/-158,14.856,0.0,39.25,38.02,2150.84,0.566,✔️,0.0,False,CPU +44,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1114.0,+86/-123,14.012,0.0,39.5,38.78,4.08,0.255,✔️,0.0,False,CPU +45,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1055.0,+114/-185,15.454,0.0,41.75,40.45,12.71,0.039,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1030.0,+146/-217,14.656,0.0,42.62,41.09,22.25,0.131,✔️,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+92/-204,17.907,0.0,43.62,42.96,0.69,0.125,✔️,0.0,False,CPU +48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),988.0,+106/-216,19.332,0.0,44.0,43.14,6.77,0.511,✔️,0.0,False,CPU +49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),907.0,+147/-333,21.426,0.0,46.25,44.95,88.79,0.572,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),734.0,+153/-330,23.404,0.0,49.75,49.48,88.79,0.052,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),658.0,+224/-609,29.74,0.0,50.88,50.11,0.43,0.047,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),523.0,+100/-474,34.23,0.0,52.44,52.35,742.99,0.364,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),484.0,+130/-555,34.314,0.0,52.81,52.72,742.99,0.097,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),334.0,+99/-505,36.454,0.0,54.0,53.99,3.98,0.12,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/winrate_matrix.png.zip index 951af4ef779cf3c320d7a75c78b1705435fb2d36..8a4d17083ed3f6a051d8403730395a4a8560224f 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_small/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_small/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a88aa7db882b5775021c82fa4b41628af6c471fd32523be36727dffeca3e7038 +oid sha256:df72aa9435a6b7a363028bbccda17403a16e33722f0dec460b93afad13a684e6 size 1493505 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip index 9770d8ced84ec9f8078f354435f1a647dbd8ee6d..0b9b58587d1aeb0f4c0d6d4ce36dfe7646eb658a 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/pareto_front_improvability_vs_time_infer.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e4e9d30d6fe968d6fcfa038753f9a4f6c6620a372f1f0cafd6eadd9382db811b +oid sha256:580ad436f857ed94dc52d20da597435a03e58278947a397ab620c1017a0fa4f8 size 451328 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip index 957bf041d72e21a94cae09122e8faa18484ae1fd..60217de8a022dd6ec77109eefd621ebe0bc8cd8b 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/pareto_n_configs_imp.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:16909d75cbd781c4b5082e06c67a8f8944286346175c7ae0d2269f001ab6720d -size 1044019 +oid sha256:ea8d34ceef272c73d66d5dc117c66624fc6a3bb9fba13bebbb1e2ff8d15bbd77 +size 1042022 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip index 19129600e032cbfe31c52d2d66186d7a33791222..79b0e9881b6ebd09bbc9ad472868c2d907baa04c 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/tuning-impact-elo.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6f25e9c69fd6e3ccea2977599e19b42743c4969436e0810430cabc9180dc7ab9 +oid sha256:0b6dc2dcebc9972bf40387ee971a1b6b95e9d115f0819c86647081d67993eec3 size 120889 diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/website_leaderboard.csv b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/website_leaderboard.csv index a69a092ddc81da31abce4a88c97bc436b3de2008..601be121811987b8fb57ec8c68320a67aca39762 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/website_leaderboard.csv +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/website_leaderboard.csv @@ -1,56 +1,56 @@ -#,Type,TypeName,Model,Verified,Elo [⬆️],Elo 95% CI,Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Improvability (%) [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Imputed (%) [⬇️],Imputed,Hardware -0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),✔️,1906.0,+304/-113,0.726,6.43,2.55,3.095,10054.02,249.652,0.0,False,GPU -1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),✔️,1900.0,+513/-229,0.772,6.57,2.05,1.229,2279.61,6.732,0.0,False,GPU -2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1884.0,+400/-86,0.683,7.0,3.48,1.908,669.82,4.041,0.0,False,GPU -3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),✔️,1873.0,+180/-50,0.671,7.29,4.98,3.69,10054.02,28.583,0.0,False,GPU -4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1780.0,+499/-189,0.597,10.14,3.32,2.584,6601.21,73.561,0.0,False,GPU -5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),✔️,1772.0,+224/-44,0.565,10.43,6.19,4.944,47.81,37.998,0.0,False,GPU -6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),✔️,1756.0,+146/-30,0.551,11.0,9.95,5.167,4840.44,33.436,0.0,False,GPU -7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),✔️,1729.0,+403/-218,0.545,12.0,3.79,2.847,2279.61,0.781,0.0,False,GPU -8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),✔️,1714.0,+373/-213,0.554,12.57,7.37,4.191,7.82,0.519,0.0,False,GPU -9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",✔️,1693.0,+192/-49,0.444,13.43,10.94,6.056,5390.56,5.054,0.0,False,CPU -10,🧠🔁,Neural Network,[Mitra (default)](https://arxiv.org/abs/2510.21204),✔️,1658.0,+220/-118,0.398,14.86,7.71,6.637,105.28,2.433,0.0,False,GPU -11,🧠🔁,Neural Network,[LimiX (default)](https://arxiv.org/abs/2509.03505),➖,1624.0,+410/-299,0.518,16.29,5.15,19.429,5.73,0.638,0.0,False,GPU -12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),✔️,1605.0,+159/-48,0.287,17.14,15.34,7.956,3301.07,1.317,0.0,False,CPU -13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),✔️,1605.0,+257/-127,0.39,17.14,12.18,6.481,4840.44,0.706,0.0,False,GPU -14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1574.0,+114/-57,0.246,18.57,17.2,9.035,997.12,3.787,0.0,False,CPU -15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),✔️,1568.0,+226/-106,0.266,18.86,14.48,8.008,3301.07,0.125,0.0,False,CPU -16,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1558.0,+418/-227,0.406,19.29,10.76,5.384,6601.21,0.588,0.0,False,GPU -17,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),✔️,1544.0,+165/-105,0.295,20.0,14.25,8.299,1709.03,2.724,0.0,False,GPU -18,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),✔️,1529.0,+188/-118,0.267,20.71,17.7,6.513,4228.53,1.447,0.0,False,GPU -19,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),✔️,1515.0,+180/-138,0.267,21.43,15.3,10.909,9360.97,7.658,0.0,False,GPU -20,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),✔️,1512.0,+142/-77,0.251,21.57,16.24,8.482,1709.03,0.097,0.0,False,GPU -21,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),✔️,1469.0,+405/-336,0.352,23.71,13.17,7.531,8.97,0.837,0.0,False,GPU -22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),✔️,1466.0,+83/-44,0.098,23.86,23.13,9.543,911.22,0.527,0.0,False,CPU -23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),✔️,1463.0,+78/-38,0.1,24.0,23.39,9.588,911.22,3.532,0.0,False,CPU -24,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1434.0,+126/-68,0.105,25.43,23.65,10.356,997.12,0.636,0.0,False,CPU -25,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),✔️,1425.0,+196/-137,0.16,25.86,21.99,7.312,4228.53,0.173,0.0,False,GPU -26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),✔️,1422.0,+220/-139,0.117,26.0,21.17,11.174,9.39,0.105,0.0,False,CPU -27,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),✔️,1407.0,+139/-152,0.121,26.71,24.25,13.106,27.26,0.29,0.0,False,GPU -28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1392.0,+209/-236,0.158,27.43,21.57,14.555,613.95,0.952,0.0,False,CPU -29,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1384.0,+236/-250,0.193,27.86,20.58,14.881,613.95,0.135,0.0,False,CPU -30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),✔️,1366.0,+155/-135,0.129,28.71,23.97,11.015,22.96,8.006,0.0,False,GPU -31,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),✔️,1325.0,+195/-187,0.114,30.71,27.44,9.406,17.97,0.145,0.0,False,GPU -32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1290.0,+275/-208,0.111,32.43,20.76,14.054,3778.28,0.323,0.0,False,CPU -33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1282.0,+124/-160,0.026,32.86,31.52,11.205,5937.36,1.406,0.0,False,CPU -34,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),✔️,1276.0,+260/-286,0.138,33.14,10.38,16.185,7.12,1.138,0.0,False,GPU -35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1272.0,+275/-228,0.14,33.29,14.11,14.636,3778.28,0.028,0.0,False,CPU -36,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),✔️,1272.0,+146/-174,0.037,33.29,30.98,14.048,9360.97,0.522,0.0,False,GPU -37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1229.0,+155/-260,0.03,35.29,32.39,16.471,628.51,0.812,0.0,False,CPU -38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1222.0,+185/-280,0.031,35.57,32.35,11.535,5937.36,0.119,0.0,False,CPU -39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),✔️,1175.0,+138/-220,0.02,37.57,35.46,16.642,0.43,0.117,0.0,False,CPU -40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1173.0,+176/-289,0.029,37.64,34.8,17.646,628.51,0.149,0.0,False,CPU -41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),✔️,1172.0,+59/-100,0.0,37.71,37.36,13.734,4.41,0.363,0.0,False,CPU -42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),✔️,1150.0,+55/-106,0.0,38.57,38.08,15.611,2163.39,7.622,0.0,False,CPU -43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),✔️,1114.0,+109/-169,0.0,40.0,39.21,14.895,3.97,0.24,0.0,False,CPU -44,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),✔️,1095.0,+89/-158,0.0,40.71,39.47,15.956,10.54,0.036,0.0,False,CPU -45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),✔️,1095.0,+106/-175,0.0,40.71,39.78,16.062,2163.39,0.713,0.0,False,CPU -46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),✔️,1002.0,+135/-301,0.0,43.86,42.36,15.699,20.5,0.13,0.0,False,CPU -47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),✔️,1000.0,+131/-217,0.0,43.93,43.18,19.276,0.67,0.118,0.0,False,CPU -48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),✔️,943.0,+77/-213,0.0,45.57,45.12,21.065,6.88,0.452,0.0,False,CPU -49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,937.0,+178/-381,0.0,45.71,44.31,22.177,86.71,0.53,0.0,False,CPU -50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,757.0,+168/-358,0.0,49.57,49.27,24.152,86.71,0.05,0.0,False,CPU -51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),✔️,712.0,+245/-524,0.0,50.29,49.48,30.287,0.4,0.044,0.0,False,CPU -52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,533.0,+85/-893,0.0,52.5,52.4,36.2,753.48,0.227,0.0,False,CPU -53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,504.0,+117/-984,0.0,52.79,52.68,36.249,753.48,0.091,0.0,False,CPU -54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),✔️,352.0,+97/-674,0.0,54.0,53.99,37.99,4.35,0.119,0.0,False,CPU +#,Type,TypeName,Model,Elo [⬆️],Elo 95% CI,Improvability (%) [⬇️],Score [⬆️],Rank [⬇️],Harmonic Rank [⬇️],Median Train Time (s/1K) [⬇️],Median Predict Time (s/1K) [⬇️],Verified,Imputed (%) [⬇️],Imputed,Hardware +0,🧠⚡,Foundation Model,[TabDPT (tuned + ensembled)](https://arxiv.org/abs/2410.18164),1906.0,+304/-113,3.095,0.726,6.43,2.55,10054.02,249.652,✔️,0.0,False,GPU +1,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned + ensembled)](https://arxiv.org/abs/2511.08667),1900.0,+513/-229,1.229,0.772,6.57,2.05,2279.61,6.732,✔️,0.0,False,GPU +2,📊,Reference Pipeline,"[AutoGluon 1.4 (extreme, 4h)](https://arxiv.org/abs/2003.06505)",1884.0,+400/-86,1.908,0.683,7.0,3.48,669.82,4.041,✔️,0.0,False,GPU +3,🧠⚡,Foundation Model,[TabDPT (tuned)](https://arxiv.org/abs/2410.18164),1873.0,+180/-50,3.69,0.671,7.29,4.98,10054.02,28.583,✔️,0.0,False,GPU +4,🧠⚡,Foundation Model,[TabPFNv2 (tuned + ensembled)](https://www.nature.com/articles/s41586-024-08328-6),1780.0,+499/-189,2.584,0.597,10.14,3.32,6601.21,73.561,✔️,0.0,False,GPU +5,🧠⚡,Foundation Model,[TabDPT (default)](https://arxiv.org/abs/2410.18164),1772.0,+224/-44,4.944,0.565,10.43,6.19,47.81,37.998,✔️,0.0,False,GPU +6,🧠🔁,Neural Network,[RealMLP (tuned + ensembled)](https://arxiv.org/abs/2407.04491),1756.0,+146/-30,5.167,0.551,11.0,9.95,4840.44,33.436,✔️,0.0,False,GPU +7,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (tuned)](https://arxiv.org/abs/2511.08667),1729.0,+403/-218,2.847,0.545,12.0,3.79,2279.61,0.781,✔️,0.0,False,GPU +8,🧠⚡,Foundation Model,[RealTabPFN-v2.5 (default)](https://arxiv.org/abs/2511.08667),1714.0,+373/-213,4.191,0.554,12.57,7.37,7.82,0.519,✔️,0.0,False,GPU +9,📊,Reference Pipeline,"[AutoGluon 1.4 (best, 4h)](https://arxiv.org/abs/2003.06505)",1693.0,+192/-49,6.056,0.444,13.43,10.94,5390.56,5.054,✔️,0.0,False,CPU +10,🧠⚡,Foundation Model,[Mitra (default)](https://arxiv.org/abs/2510.21204),1658.0,+220/-118,6.637,0.398,14.86,7.71,105.28,2.433,✔️,0.0,False,GPU +11,🧠⚡,Foundation Model,[LimiX (default)](https://arxiv.org/abs/2509.03505),1624.0,+410/-299,19.429,0.518,16.29,5.15,5.73,0.638,➖,0.0,False,GPU +12,🌳,Tree-based,[CatBoost (tuned + ensembled)](https://arxiv.org/abs/1706.09516),1605.0,+159/-48,7.956,0.287,17.14,15.34,3301.07,1.317,✔️,0.0,False,CPU +13,🧠🔁,Neural Network,[RealMLP (tuned)](https://arxiv.org/abs/2407.04491),1605.0,+257/-127,6.481,0.39,17.14,12.18,4840.44,0.706,✔️,0.0,False,GPU +14,🌳,Tree-based,[LightGBM (tuned + ensembled)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1574.0,+114/-57,9.035,0.246,18.57,17.2,997.12,3.787,✔️,0.0,False,CPU +15,🌳,Tree-based,[CatBoost (tuned)](https://arxiv.org/abs/1706.09516),1568.0,+226/-106,8.008,0.266,18.86,14.48,3301.07,0.125,✔️,0.0,False,CPU +16,🧠⚡,Foundation Model,[TabPFNv2 (tuned)](https://www.nature.com/articles/s41586-024-08328-6),1558.0,+418/-227,5.384,0.406,19.29,10.76,6601.21,0.588,✔️,0.0,False,GPU +17,❓,Other,[xRFM (tuned + ensembled)](https://arxiv.org/abs/2508.10053),1544.0,+165/-105,8.299,0.295,20.0,14.25,1709.03,2.724,✔️,0.0,False,GPU +18,🧠🔁,Neural Network,[TabM (tuned + ensembled)](https://arxiv.org/abs/2410.24210),1529.0,+188/-118,6.513,0.267,20.71,17.7,4228.53,1.447,✔️,0.0,False,GPU +19,🧠🔁,Neural Network,[ModernNCA (tuned + ensembled)](https://arxiv.org/abs/2407.03257),1515.0,+180/-138,10.909,0.267,21.43,15.3,9360.97,7.658,✔️,0.0,False,GPU +20,❓,Other,[xRFM (tuned)](https://arxiv.org/abs/2508.10053),1512.0,+142/-77,8.482,0.251,21.57,16.24,1709.03,0.097,✔️,0.0,False,GPU +21,🧠⚡,Foundation Model,[TabPFNv2 (default)](https://www.nature.com/articles/s41586-024-08328-6),1469.0,+405/-336,7.531,0.352,23.71,13.17,8.97,0.837,✔️,0.0,False,GPU +22,🌳,Tree-based,[XGBoost (tuned)](https://arxiv.org/abs/1603.02754),1466.0,+83/-44,9.543,0.098,23.86,23.13,911.22,0.527,✔️,0.0,False,CPU +23,🌳,Tree-based,[XGBoost (tuned + ensembled)](https://arxiv.org/abs/1603.02754),1463.0,+78/-38,9.588,0.1,24.0,23.39,911.22,3.532,✔️,0.0,False,CPU +24,🌳,Tree-based,[LightGBM (tuned)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1434.0,+126/-68,10.356,0.105,25.43,23.65,997.12,0.636,✔️,0.0,False,CPU +25,🧠🔁,Neural Network,[TabM (tuned)](https://arxiv.org/abs/2410.24210),1425.0,+196/-137,7.312,0.16,25.86,21.99,4228.53,0.173,✔️,0.0,False,GPU +26,🌳,Tree-based,[CatBoost (default)](https://arxiv.org/abs/1706.09516),1422.0,+220/-139,11.174,0.117,26.0,21.17,9.39,0.105,✔️,0.0,False,CPU +27,🧠🔁,Neural Network,[ModernNCA (default)](https://arxiv.org/abs/2407.03257),1407.0,+139/-152,13.106,0.121,26.71,24.25,27.26,0.29,✔️,0.0,False,GPU +28,🌳,Tree-based,[ExtraTrees (tuned + ensembled)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1392.0,+209/-236,14.555,0.158,27.43,21.57,613.95,0.952,✔️,0.0,False,CPU +29,🌳,Tree-based,[ExtraTrees (tuned)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1384.0,+236/-250,14.881,0.193,27.86,20.58,613.95,0.135,✔️,0.0,False,CPU +30,🧠🔁,Neural Network,[RealMLP (default)](https://arxiv.org/abs/2407.04491),1366.0,+155/-135,11.015,0.129,28.71,23.97,22.96,8.006,✔️,0.0,False,GPU +31,🧠🔁,Neural Network,[TabM (default)](https://arxiv.org/abs/2410.24210),1325.0,+195/-187,9.406,0.114,30.71,27.44,17.97,0.145,✔️,0.0,False,GPU +32,🌳,Tree-based,[EBM (tuned + ensembled)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1290.0,+275/-208,14.054,0.111,32.43,20.76,3778.28,0.323,✔️,0.0,False,CPU +33,🧠🔁,Neural Network,[TorchMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1282.0,+124/-160,11.205,0.026,32.86,31.52,5937.36,1.406,✔️,0.0,False,CPU +34,❓,Other,[xRFM (default)](https://arxiv.org/abs/2508.10053),1276.0,+260/-286,16.185,0.138,33.14,10.38,7.12,1.138,✔️,0.0,False,GPU +35,🌳,Tree-based,[EBM (tuned)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1272.0,+275/-228,14.636,0.14,33.29,14.11,3778.28,0.028,✔️,0.0,False,CPU +36,🧠🔁,Neural Network,[ModernNCA (tuned)](https://arxiv.org/abs/2407.03257),1272.0,+146/-174,14.048,0.037,33.29,30.98,9360.97,0.522,✔️,0.0,False,GPU +37,🌳,Tree-based,[RandomForest (tuned + ensembled)](https://link.springer.com/article/10.1023/A:1010933404324),1229.0,+155/-260,16.471,0.03,35.29,32.39,628.51,0.812,✔️,0.0,False,CPU +38,🧠🔁,Neural Network,[TorchMLP (tuned)](https://arxiv.org/abs/2003.06505),1222.0,+185/-280,11.535,0.031,35.57,32.35,5937.36,0.119,✔️,0.0,False,CPU +39,🌳,Tree-based,[ExtraTrees (default)](https://link.springer.com/article/10.1007/s10994-006-6226-1),1175.0,+138/-220,16.642,0.02,37.57,35.46,0.43,0.117,✔️,0.0,False,CPU +40,🌳,Tree-based,[RandomForest (tuned)](https://link.springer.com/article/10.1023/A:1010933404324),1173.0,+176/-289,17.646,0.029,37.64,34.8,628.51,0.149,✔️,0.0,False,CPU +41,🌳,Tree-based,[LightGBM (default)](https://papers.nips.cc/paper_files/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html),1172.0,+59/-100,13.734,0.0,37.71,37.36,4.41,0.363,✔️,0.0,False,CPU +42,🧠🔁,Neural Network,[FastaiMLP (tuned + ensembled)](https://arxiv.org/abs/2003.06505),1150.0,+55/-106,15.611,0.0,38.57,38.08,2163.39,7.622,✔️,0.0,False,CPU +43,🌳,Tree-based,[XGBoost (default)](https://arxiv.org/abs/1603.02754),1114.0,+109/-169,14.895,0.0,40.0,39.21,3.97,0.24,✔️,0.0,False,CPU +44,🌳,Tree-based,[EBM (default)](https://www.cs.cornell.edu/~yinlou/papers/lou-kdd13.pdf),1095.0,+89/-158,15.956,0.0,40.71,39.47,10.54,0.036,✔️,0.0,False,CPU +45,🧠🔁,Neural Network,[FastaiMLP (tuned)](https://arxiv.org/abs/2003.06505),1095.0,+106/-175,16.062,0.0,40.71,39.78,2163.39,0.713,✔️,0.0,False,CPU +46,🧠🔁,Neural Network,[TorchMLP (default)](https://arxiv.org/abs/2003.06505),1002.0,+135/-301,15.699,0.0,43.86,42.36,20.5,0.13,✔️,0.0,False,CPU +47,🌳,Tree-based,[RandomForest (default)](https://link.springer.com/article/10.1023/A:1010933404324),1000.0,+131/-217,19.276,0.0,43.93,43.18,0.67,0.118,✔️,0.0,False,CPU +48,🧠🔁,Neural Network,[FastaiMLP (default)](https://arxiv.org/abs/2003.06505),943.0,+77/-213,21.065,0.0,45.57,45.12,6.88,0.452,✔️,0.0,False,CPU +49,📏,Baseline,[KNN (tuned + ensembled)](https://scikit-learn.org/stable/modules/neighbors.html),937.0,+178/-381,22.177,0.0,45.71,44.31,86.71,0.53,✔️,0.0,False,CPU +50,📏,Baseline,[KNN (tuned)](https://scikit-learn.org/stable/modules/neighbors.html),757.0,+168/-358,24.152,0.0,49.57,49.27,86.71,0.05,✔️,0.0,False,CPU +51,📏,Baseline,[KNN (default)](https://scikit-learn.org/stable/modules/neighbors.html),712.0,+245/-524,30.287,0.0,50.29,49.48,0.4,0.044,✔️,0.0,False,CPU +52,📏,Baseline,[Linear (tuned + ensembled)](https://scikit-learn.org/stable/modules/linear_model.html),533.0,+85/-893,36.2,0.0,52.5,52.4,753.48,0.227,✔️,0.0,False,CPU +53,📏,Baseline,[Linear (tuned)](https://scikit-learn.org/stable/modules/linear_model.html),504.0,+117/-984,36.249,0.0,52.79,52.68,753.48,0.091,✔️,0.0,False,CPU +54,📏,Baseline,[Linear (default)](https://scikit-learn.org/stable/modules/linear_model.html),352.0,+97/-674,37.99,0.0,54.0,53.99,4.35,0.119,✔️,0.0,False,CPU diff --git a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip index 6c7afdc0fc2d3c05d69d554bc5e1e985175cfcb6..9ccfcb7bee0cca2aabd77f85e287b03fcb5e8e95 100644 --- a/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip +++ b/data/imputation_yes/splits_lite/tasks_regression/datasets_tabpfn/winrate_matrix.png.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ad885aa3f6b8056563f89a79a12b7e4ed824df9776bacd8a1a556f5fd353b318 +oid sha256:5e24ab0d5b1957d46f486d7858888173c4738f1556e1d664359b611e980849cb size 1396925