Spaces:
Running
Running
Deploy ff0c9c8: header logos, two modalities (#228)
Browse files- data/encodings.csv +18 -90
- data/leaderboards.csv +0 -0
- data/per_dataset.csv +0 -0
- index.html +7 -41
data/encodings.csv
CHANGED
|
@@ -1,127 +1,55 @@
|
|
| 1 |
tier,task,modality,encoding,elo,elo_low,elo_high,datasets,models,win_rate,best_on
|
| 2 |
core,classification,text,TAR,1238.2504,1216.2728,1264.1539,10,23,0.7976,1072
|
| 3 |
core,classification,text,Frozen,1000.0,974.0964,1021.9776,10,23,0.2024,272
|
| 4 |
-
core,classification,image,TAR,
|
| 5 |
-
core,classification,image,Frozen,1000.0,
|
| 6 |
-
core,classification,image-text,Frozen,1000.0,973.9107,1029.5978,4,23,0.6957,384
|
| 7 |
-
core,classification,image-text,TAR,856.3912,826.7935,882.4805,4,23,0.3043,168
|
| 8 |
-
core,classification,text+image,TAR,1157.5895,1143.5111,1175.5488,18,23,0.7124,1744
|
| 9 |
-
core,classification,text+image,Frozen,1000.0,982.0408,1014.0784,18,23,0.2876,704
|
| 10 |
-
core,classification,text+image-text,TAR,1110.6071,1093.6591,1127.556,14,23,0.654,1240
|
| 11 |
-
core,classification,text+image-text,Frozen,1000.0,983.0512,1016.948,14,23,0.346,656
|
| 12 |
-
core,classification,image+image-text,TAR,1005.0357,988.2189,1022.7553,12,23,0.5072,840
|
| 13 |
-
core,classification,image+image-text,Frozen,1000.0,982.2803,1016.8167,12,23,0.4928,816
|
| 14 |
core,classification,all,TAR,1097.9436,1084.8994,1110.0947,22,23,0.6373,1912
|
| 15 |
core,classification,all,Frozen,1000.0,987.8489,1013.0442,22,23,0.3627,1088
|
| 16 |
core,regression,text,TAR,1104.778,1089.447,1120.9468,10,23,0.6464,892
|
| 17 |
core,regression,text,Frozen,1000.0,983.8312,1015.331,10,23,0.3536,488
|
| 18 |
-
core,regression,image,TAR,
|
| 19 |
-
core,regression,image,Frozen,1000.0,
|
| 20 |
-
core,regression,image-text,TAR,1070.4365,1049.3355,1092.621,5,23,0.6,414
|
| 21 |
-
core,regression,image-text,Frozen,1000.0,977.8155,1021.101,5,23,0.4,276
|
| 22 |
-
core,regression,text+image,TAR,1132.3161,1118.8438,1145.8817,13,23,0.6817,1223
|
| 23 |
-
core,regression,text+image,Frozen,1000.0,986.4345,1013.4724,13,23,0.3183,571
|
| 24 |
-
core,regression,text+image-text,TAR,1093.1399,1080.7361,1105.4622,15,23,0.6309,1306
|
| 25 |
-
core,regression,text+image-text,Frozen,1000.0,987.6777,1012.4038,15,23,0.3691,764
|
| 26 |
-
core,regression,image+image-text,TAR,1126.8247,1110.8083,1144.7149,8,23,0.6748,745
|
| 27 |
-
core,regression,image+image-text,Frozen,1000.0,982.1098,1016.0164,8,23,0.3252,359
|
| 28 |
core,regression,all,TAR,1114.4661,1103.1729,1126.0841,18,23,0.659,1637
|
| 29 |
core,regression,all,Frozen,1000.0,988.382,1011.2932,18,23,0.341,847
|
| 30 |
core,all,text,TAR,1164.9312,1151.7306,1178.2345,20,23,0.721,1964
|
| 31 |
core,all,text,Frozen,1000.0,986.6966,1013.2005,20,23,0.279,760
|
| 32 |
-
core,all,image,TAR,
|
| 33 |
-
core,all,image,Frozen,1000.0,
|
| 34 |
-
core,all,image-text,Frozen,1000.0,982.6319,1017.2953,9,23,0.5314,660
|
| 35 |
-
core,all,image-text,TAR,978.1516,960.8563,995.5198,9,23,0.4686,582
|
| 36 |
-
core,all,text+image,TAR,1146.7229,1136.5284,1157.1112,31,23,0.6994,2967
|
| 37 |
-
core,all,text+image,Frozen,1000.0,989.6118,1010.1945,31,23,0.3006,1275
|
| 38 |
-
core,all,text+image-text,TAR,1101.428,1091.0141,1112.7994,29,23,0.642,2546
|
| 39 |
-
core,all,text+image-text,Frozen,1000.0,988.6287,1010.4139,29,23,0.358,1420
|
| 40 |
-
core,all,image+image-text,TAR,1051.9966,1038.8617,1063.9792,20,23,0.5743,1585
|
| 41 |
-
core,all,image+image-text,Frozen,1000.0,988.0174,1013.1348,20,23,0.4257,1175
|
| 42 |
core,all,all,TAR,1105.37,1096.6799,1115.1837,40,23,0.6472,3549
|
| 43 |
core,all,all,Frozen,1000.0,990.1863,1008.6901,40,23,0.3528,1935
|
| 44 |
extended,classification,text,Frozen,1000.0,984.3648,1016.2041,10,23,0.6448,886
|
| 45 |
extended,classification,text,TAR,896.3944,880.1903,912.0297,10,23,0.3552,488
|
| 46 |
-
extended,classification,image,
|
| 47 |
-
extended,classification,image,
|
| 48 |
-
extended,classification,image-text,Frozen,1000.0,948.6631,1069.2054,2,23,0.8721,225
|
| 49 |
-
extended,classification,image-text,TAR,666.5326,597.3272,717.8695,2,23,0.1279,33
|
| 50 |
-
extended,classification,text+image,Frozen,1000.0,988.0701,1011.6417,16,23,0.5888,1297
|
| 51 |
-
extended,classification,text+image,TAR,937.6464,926.0047,949.5763,16,23,0.4112,906
|
| 52 |
-
extended,classification,text+image-text,Frozen,1000.0,985.9365,1015.7284,12,23,0.6808,1111
|
| 53 |
-
extended,classification,text+image-text,TAR,868.4495,852.7211,882.513,12,23,0.3192,521
|
| 54 |
-
extended,classification,image+image-text,Frozen,1000.0,983.9014,1015.2543,8,23,0.5852,636
|
| 55 |
-
extended,classification,image+image-text,TAR,940.2317,924.9774,956.3303,8,23,0.4148,451
|
| 56 |
extended,classification,all,Frozen,1000.0,988.0522,1012.1898,18,23,0.6185,1522
|
| 57 |
extended,classification,all,TAR,916.0649,903.8751,928.0127,18,23,0.3815,939
|
| 58 |
extended,regression,text,Frozen,1000.0,986.4961,1014.5933,10,23,0.5703,787
|
| 59 |
extended,regression,text,TAR,950.832,936.2387,964.3359,10,23,0.4297,593
|
| 60 |
-
extended,regression,image,Frozen,1000.0,
|
| 61 |
-
extended,regression,image,TAR,
|
| 62 |
-
extended,regression,image-text,Frozen,1000.0,985.2903,1014.3994,11,23,0.7668,1164
|
| 63 |
-
extended,regression,image-text,TAR,793.2201,778.8207,807.9298,11,23,0.2332,354
|
| 64 |
-
extended,regression,text+image,Frozen,1000.0,986.0839,1015.1461,11,23,0.6041,917
|
| 65 |
-
extended,regression,text+image,TAR,926.6021,911.456,940.5182,11,23,0.3959,601
|
| 66 |
-
extended,regression,text+image-text,Frozen,1000.0,989.0573,1010.0197,21,23,0.6732,1951
|
| 67 |
-
extended,regression,text+image-text,TAR,874.4371,864.4174,885.3798,21,23,0.3268,947
|
| 68 |
-
extended,regression,image+image-text,Frozen,1000.0,985.6204,1015.5194,12,23,0.7814,1294
|
| 69 |
-
extended,regression,image+image-text,TAR,778.7097,763.1903,793.0894,12,23,0.2186,362
|
| 70 |
extended,regression,all,Frozen,1000.0,989.6044,1010.3185,22,23,0.6854,2081
|
| 71 |
extended,regression,all,TAR,864.6925,854.374,875.0881,22,23,0.3146,955
|
| 72 |
extended,all,text,Frozen,1000.0,990.1897,1010.1907,20,23,0.6075,1673
|
| 73 |
extended,all,text,TAR,924.1319,913.9412,933.9422,20,23,0.3925,1081
|
| 74 |
-
extended,all,image,Frozen,1000.0,
|
| 75 |
-
extended,all,image,TAR,
|
| 76 |
-
extended,all,image-text,Frozen,1000.0,986.7896,1014.7702,13,23,0.7821,1389
|
| 77 |
-
extended,all,image-text,TAR,778.0035,763.2333,791.2138,13,23,0.2179,387
|
| 78 |
-
extended,all,text+image,Frozen,1000.0,991.164,1008.7145,27,23,0.595,2214
|
| 79 |
-
extended,all,text+image,TAR,933.1558,924.4414,941.9918,27,23,0.405,1507
|
| 80 |
-
extended,all,text+image-text,Frozen,1000.0,991.5572,1008.4695,33,23,0.6759,3062
|
| 81 |
-
extended,all,text+image-text,TAR,872.2883,863.8188,880.7311,33,23,0.3241,1468
|
| 82 |
-
extended,all,image+image-text,Frozen,1000.0,989.1752,1010.6802,20,23,0.7037,1930
|
| 83 |
-
extended,all,image+image-text,TAR,849.7514,839.0712,860.5763,20,23,0.2963,813
|
| 84 |
extended,all,all,Frozen,1000.0,992.7771,1007.7776,40,23,0.6555,3603
|
| 85 |
extended,all,all,TAR,888.2645,880.4869,895.4874,40,23,0.3445,1894
|
| 86 |
all,classification,text,TAR,1051.7664,1039.7049,1064.608,20,23,0.574,1560
|
| 87 |
all,classification,text,Frozen,1000.0,987.1584,1012.0615,20,23,0.426,1158
|
| 88 |
-
all,classification,image,
|
| 89 |
-
all,classification,image,
|
| 90 |
-
all,classification,image-text,Frozen,1000.0,973.2634,1027.5185,6,23,0.7519,609
|
| 91 |
-
all,classification,image-text,TAR,807.4315,779.913,834.1681,6,23,0.2481,201
|
| 92 |
-
all,classification,text+image,TAR,1048.8101,1039.2331,1058.7709,34,23,0.5698,2650
|
| 93 |
-
all,classification,text+image,Frozen,1000.0,990.0392,1009.577,34,23,0.4302,2001
|
| 94 |
-
all,classification,text+image-text,Frozen,1000.0,988.2595,1011.4489,26,23,0.5009,1767
|
| 95 |
-
all,classification,text+image-text,TAR,999.4091,987.9603,1011.1496,26,23,0.4991,1761
|
| 96 |
-
all,classification,image+image-text,Frozen,1000.0,988.6478,1012.3464,20,23,0.5294,1452
|
| 97 |
-
all,classification,image+image-text,TAR,979.5764,967.23,990.9286,20,23,0.4706,1291
|
| 98 |
all,classification,all,TAR,1015.3455,1005.5401,1023.6602,40,23,0.5221,2851
|
| 99 |
all,classification,all,Frozen,1000.0,991.6853,1009.8054,40,23,0.4779,2610
|
| 100 |
all,regression,text,TAR,1026.4865,1016.2584,1036.2945,20,23,0.538,1485
|
| 101 |
all,regression,text,Frozen,1000.0,990.192,1010.2281,20,23,0.462,1275
|
| 102 |
-
all,regression,image,
|
| 103 |
-
all,regression,image,
|
| 104 |
-
all,regression,image-text,Frozen,1000.0,987.09,1011.7019,16,23,0.6522,1440
|
| 105 |
-
all,regression,image-text,TAR,890.7995,879.0976,903.7095,16,23,0.3478,768
|
| 106 |
-
all,regression,text+image,TAR,1035.3688,1026.0831,1045.6195,24,23,0.5507,1824
|
| 107 |
-
all,regression,text+image,Frozen,1000.0,989.7493,1009.2857,24,23,0.4493,1488
|
| 108 |
-
all,regression,text+image-text,Frozen,1000.0,991.9849,1008.939,36,23,0.5465,2715
|
| 109 |
-
all,regression,text+image-text,TAR,967.5965,958.6575,975.6116,36,23,0.4535,2253
|
| 110 |
-
all,regression,image+image-text,Frozen,1000.0,988.5978,1011.1665,20,23,0.5989,1653
|
| 111 |
-
all,regression,image+image-text,TAR,930.3499,919.1834,941.7521,20,23,0.4011,1107
|
| 112 |
all,regression,all,Frozen,1000.0,992.2422,1008.4369,40,23,0.5304,2928
|
| 113 |
all,regression,all,TAR,978.8256,970.3886,986.5833,40,23,0.4696,2592
|
| 114 |
all,all,text,TAR,1038.9781,1030.6002,1047.9682,40,23,0.5559,3045
|
| 115 |
all,all,text,Frozen,1000.0,991.0099,1008.3779,40,23,0.4441,2433
|
| 116 |
-
all,all,image,
|
| 117 |
-
all,all,image,
|
| 118 |
-
all,all,image-text,Frozen,1000.0,989.14,1012.2007,22,23,0.6789,2049
|
| 119 |
-
all,all,image-text,TAR,869.9127,857.712,880.7727,22,23,0.3211,969
|
| 120 |
-
all,all,text+image,TAR,1043.2035,1036.4069,1049.6174,58,23,0.5619,4474
|
| 121 |
-
all,all,text+image,Frozen,1000.0,993.586,1006.7966,58,23,0.4381,3489
|
| 122 |
-
all,all,text+image-text,Frozen,1000.0,992.9127,1006.4148,62,23,0.5275,4482
|
| 123 |
-
all,all,text+image-text,TAR,980.8422,974.4274,987.9295,62,23,0.4725,4014
|
| 124 |
-
all,all,image+image-text,Frozen,1000.0,991.6654,1008.5373,40,23,0.5642,3105
|
| 125 |
-
all,all,image+image-text,TAR,955.1068,946.5695,963.4414,40,23,0.4358,2398
|
| 126 |
all,all,all,Frozen,1000.0,994.0332,1005.9858,80,23,0.5043,5538
|
| 127 |
all,all,all,TAR,996.9939,991.008,1002.9607,80,23,0.4957,5443
|
|
|
|
| 1 |
tier,task,modality,encoding,elo,elo_low,elo_high,datasets,models,win_rate,best_on
|
| 2 |
core,classification,text,TAR,1238.2504,1216.2728,1264.1539,10,23,0.7976,1072
|
| 3 |
core,classification,text,Frozen,1000.0,974.0964,1021.9776,10,23,0.2024,272
|
| 4 |
+
core,classification,image,TAR,1005.0357,988.2189,1022.7553,12,23,0.5072,840
|
| 5 |
+
core,classification,image,Frozen,1000.0,982.2803,1016.8167,12,23,0.4928,816
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
core,classification,all,TAR,1097.9436,1084.8994,1110.0947,22,23,0.6373,1912
|
| 7 |
core,classification,all,Frozen,1000.0,987.8489,1013.0442,22,23,0.3627,1088
|
| 8 |
core,regression,text,TAR,1104.778,1089.447,1120.9468,10,23,0.6464,892
|
| 9 |
core,regression,text,Frozen,1000.0,983.8312,1015.331,10,23,0.3536,488
|
| 10 |
+
core,regression,image,TAR,1126.8247,1110.8083,1144.7149,8,23,0.6748,745
|
| 11 |
+
core,regression,image,Frozen,1000.0,982.1098,1016.0164,8,23,0.3252,359
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
core,regression,all,TAR,1114.4661,1103.1729,1126.0841,18,23,0.659,1637
|
| 13 |
core,regression,all,Frozen,1000.0,988.382,1011.2932,18,23,0.341,847
|
| 14 |
core,all,text,TAR,1164.9312,1151.7306,1178.2345,20,23,0.721,1964
|
| 15 |
core,all,text,Frozen,1000.0,986.6966,1013.2005,20,23,0.279,760
|
| 16 |
+
core,all,image,TAR,1051.9966,1038.8617,1063.9792,20,23,0.5743,1585
|
| 17 |
+
core,all,image,Frozen,1000.0,988.0174,1013.1348,20,23,0.4257,1175
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
core,all,all,TAR,1105.37,1096.6799,1115.1837,40,23,0.6472,3549
|
| 19 |
core,all,all,Frozen,1000.0,990.1863,1008.6901,40,23,0.3528,1935
|
| 20 |
extended,classification,text,Frozen,1000.0,984.3648,1016.2041,10,23,0.6448,886
|
| 21 |
extended,classification,text,TAR,896.3944,880.1903,912.0297,10,23,0.3552,488
|
| 22 |
+
extended,classification,image,Frozen,1000.0,983.9014,1015.2543,8,23,0.5852,636
|
| 23 |
+
extended,classification,image,TAR,940.2317,924.9774,956.3303,8,23,0.4148,451
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
extended,classification,all,Frozen,1000.0,988.0522,1012.1898,18,23,0.6185,1522
|
| 25 |
extended,classification,all,TAR,916.0649,903.8751,928.0127,18,23,0.3815,939
|
| 26 |
extended,regression,text,Frozen,1000.0,986.4961,1014.5933,10,23,0.5703,787
|
| 27 |
extended,regression,text,TAR,950.832,936.2387,964.3359,10,23,0.4297,593
|
| 28 |
+
extended,regression,image,Frozen,1000.0,985.6204,1015.5194,12,23,0.7814,1294
|
| 29 |
+
extended,regression,image,TAR,778.7097,763.1903,793.0894,12,23,0.2186,362
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
extended,regression,all,Frozen,1000.0,989.6044,1010.3185,22,23,0.6854,2081
|
| 31 |
extended,regression,all,TAR,864.6925,854.374,875.0881,22,23,0.3146,955
|
| 32 |
extended,all,text,Frozen,1000.0,990.1897,1010.1907,20,23,0.6075,1673
|
| 33 |
extended,all,text,TAR,924.1319,913.9412,933.9422,20,23,0.3925,1081
|
| 34 |
+
extended,all,image,Frozen,1000.0,989.1752,1010.6802,20,23,0.7037,1930
|
| 35 |
+
extended,all,image,TAR,849.7514,839.0712,860.5763,20,23,0.2963,813
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
extended,all,all,Frozen,1000.0,992.7771,1007.7776,40,23,0.6555,3603
|
| 37 |
extended,all,all,TAR,888.2645,880.4869,895.4874,40,23,0.3445,1894
|
| 38 |
all,classification,text,TAR,1051.7664,1039.7049,1064.608,20,23,0.574,1560
|
| 39 |
all,classification,text,Frozen,1000.0,987.1584,1012.0615,20,23,0.426,1158
|
| 40 |
+
all,classification,image,Frozen,1000.0,988.6478,1012.3464,20,23,0.5294,1452
|
| 41 |
+
all,classification,image,TAR,979.5764,967.23,990.9286,20,23,0.4706,1291
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
all,classification,all,TAR,1015.3455,1005.5401,1023.6602,40,23,0.5221,2851
|
| 43 |
all,classification,all,Frozen,1000.0,991.6853,1009.8054,40,23,0.4779,2610
|
| 44 |
all,regression,text,TAR,1026.4865,1016.2584,1036.2945,20,23,0.538,1485
|
| 45 |
all,regression,text,Frozen,1000.0,990.192,1010.2281,20,23,0.462,1275
|
| 46 |
+
all,regression,image,Frozen,1000.0,988.5978,1011.1665,20,23,0.5989,1653
|
| 47 |
+
all,regression,image,TAR,930.3499,919.1834,941.7521,20,23,0.4011,1107
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
all,regression,all,Frozen,1000.0,992.2422,1008.4369,40,23,0.5304,2928
|
| 49 |
all,regression,all,TAR,978.8256,970.3886,986.5833,40,23,0.4696,2592
|
| 50 |
all,all,text,TAR,1038.9781,1030.6002,1047.9682,40,23,0.5559,3045
|
| 51 |
all,all,text,Frozen,1000.0,991.0099,1008.3779,40,23,0.4441,2433
|
| 52 |
+
all,all,image,Frozen,1000.0,991.6654,1008.5373,40,23,0.5642,3105
|
| 53 |
+
all,all,image,TAR,955.1068,946.5695,963.4414,40,23,0.4358,2398
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
all,all,all,Frozen,1000.0,994.0332,1005.9858,80,23,0.5043,5538
|
| 55 |
all,all,all,TAR,996.9939,991.008,1002.9607,80,23,0.4957,5443
|
data/leaderboards.csv
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
data/per_dataset.csv
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
index.html
CHANGED
|
@@ -50,8 +50,6 @@ header { display: grid; gap: 14px; }
|
|
| 50 |
:root:not([data-theme="light"]) .logo .light { display: none; }
|
| 51 |
:root:not([data-theme="light"]) .logo .dark { display: block; }
|
| 52 |
}
|
| 53 |
-
.facts { display: flex; flex-wrap: wrap; gap: 8px 22px; font: 13px/1.4 var(--mono); color: var(--muted); }
|
| 54 |
-
.facts b { color: var(--ink); font-weight: 500; }
|
| 55 |
section { display: grid; gap: 16px; }
|
| 56 |
.head { display: flex; flex-wrap: wrap; align-items: end; justify-content: space-between; gap: 12px; }
|
| 57 |
.tabs { display: flex; flex-wrap: wrap; gap: 6px; }
|
|
@@ -105,7 +103,9 @@ b.enc { color: var(--c); }
|
|
| 105 |
#intro { font-size: 17px; line-height: 1.6; max-width: 84ch; }
|
| 106 |
.filters .label { font: 500 11.5px/1 var(--mono); letter-spacing: 0.05em; text-transform: uppercase; color: var(--muted); margin-right: 4px; }
|
| 107 |
.tabs button .n { font-family: var(--mono); opacity: .6; margin-left: 4px; }
|
| 108 |
-
.links a { color: var(--accent); }
|
|
|
|
|
|
|
| 109 |
code { font-family: var(--mono); font-size: 0.92em; overflow-wrap: anywhere; }
|
| 110 |
|
| 111 |
.topbar { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 12px 20px; }
|
|
@@ -141,8 +141,7 @@ pre.bib { margin: 0; padding: 16px 18px; font: 12.5px/1.5 var(--mono); white-spa
|
|
| 141 |
</div>
|
| 142 |
<div class="version" id="version"></div>
|
| 143 |
<p class="muted" id="intro">MulTaBench is a benchmark for multimodal tabular learning with text and images. Its purpose is to drive smarter encoding methods, towards our vision of truly Multimodal Tabular Foundation Models. Its 80 datasets balance image and text modalities, classification and regression, and how hard the text or images are to encode, with the <b>Core</b> slice focusing on the 40 datasets where frozen embeddings are not enough.</p>
|
| 144 |
-
<p class="links"><a href="https://github.com/alanarazi7/MulTaBench">Code</a> 路 <a href="https://huggingface.co/multabench">Datasets</a> 路 <a href="https://arxiv.org/abs/2605.10616">Paper</a> 路 <a href="https://github.com/alanarazi7/MulTaBench/blob/master/docs/protocol.md">Protocol</a>
|
| 145 |
-
<div class="facts" id="facts"></div>
|
| 146 |
</header>
|
| 147 |
|
| 148 |
<div class="sticky">
|
|
@@ -209,7 +208,6 @@ pre.bib { margin: 0; padding: 16px 18px; font: 12.5px/1.5 var(--mono); white-spa
|
|
| 209 |
<h2>Using the leaderboard in your paper</h2>
|
| 210 |
<p class="muted" id="dlNote"></p>
|
| 211 |
<div class="panel dl">
|
| 212 |
-
<div><span class="label">Elo vs cost plot</span><button class="btn" data-dl="svg">SVG</button><button class="btn" data-dl="png">PNG</button></div>
|
| 213 |
<div><span class="label">Ranking table</span><button class="btn" data-dl="csv">CSV</button><button class="btn" data-dl="json">JSON</button><a class="btn" href="data/leaderboards.csv" download>CSV 路 every filter</a></div>
|
| 214 |
<div><span class="label">Per dataset and fold</span><a class="btn" href="data/per_dataset.csv" download>Per dataset CSV</a><a class="btn" href="https://github.com/alanarazi7/MulTaBench/tree/master/multabench/leaderboard/models">Every run, per model (GitHub)</a></div>
|
| 215 |
</div>
|
|
@@ -273,13 +271,7 @@ async function load() {
|
|
| 273 |
const info = Object.fromEntries((await get('models')).map(r => [r.model, r]));
|
| 274 |
const entries = boards['all|all|all|all'].map(r => r.entry).sort();
|
| 275 |
const encodingOf = Object.fromEntries(boards['all|all|all|all'].map(r => [r.entry, r.encoding]));
|
| 276 |
-
|
| 277 |
-
const have = new Set(per_dataset.map(r => r.entry + '|' + r.dataset));
|
| 278 |
-
const errors = [];
|
| 279 |
-
entries.forEach(m => datasets.forEach(d => { if (!have.has(m + '|' + d)) errors.push({ entry: m, dataset: d }); }));
|
| 280 |
-
const runs = per_dataset.reduce((n, r) => n + r.folds, 0);
|
| 281 |
-
return { boards, encodingBoards, per_dataset, cost, updated, info, errors, encodingOf,
|
| 282 |
-
meta: { entries, models: Object.keys(info).length, datasets: datasets.length, runs } };
|
| 283 |
}
|
| 284 |
load().then(start).catch(err => {
|
| 285 |
document.querySelector('.wrap').insertAdjacentHTML('afterbegin', `<p class="flag">Could not load the leaderboard data: ${err.message}</p>`);
|
|
@@ -293,7 +285,7 @@ const encColor = e => `var(--${e === 'end-to-end' ? 'e2e' : e})`;
|
|
| 293 |
const encodingCell = e => `<td><span class="chip ${e}" title="${ENC_TITLE[e]}">${ENC_LABEL[e]}</span></td>`;
|
| 294 |
const TIERS = [['core', 'Core'], ['extended', 'Extended']];
|
| 295 |
const TASKS = [['classification', 'Classification'], ['regression', 'Regression']];
|
| 296 |
-
const MODS = [['text', 'Text'], ['image', 'Image']
|
| 297 |
const ENCS = [['frozen', 'Frozen'], ['tar', 'TAR']];
|
| 298 |
const taskOf = d => d.startsWith('REG_') ? 'regression' : 'classification';
|
| 299 |
const store = {
|
|
@@ -414,18 +406,6 @@ function drawPareto(board, front) {
|
|
| 414 |
}
|
| 415 |
|
| 416 |
let current = { board: [], front: new Set(), label: '' };
|
| 417 |
-
function resolvedSVG() {
|
| 418 |
-
const css = getComputedStyle(document.documentElement);
|
| 419 |
-
const svg = document.querySelector('#paretoChart svg').cloneNode(true);
|
| 420 |
-
svg.setAttribute('font-family', css.getPropertyValue('--mono'));
|
| 421 |
-
svg.setAttribute('font-size', '11');
|
| 422 |
-
const [, , vw, vh] = svg.getAttribute('viewBox').split(' ');
|
| 423 |
-
svg.setAttribute('width', vw);
|
| 424 |
-
svg.setAttribute('height', vh);
|
| 425 |
-
svg.insertAdjacentHTML('afterbegin', `<rect width="100%" height="100%" fill="var(--surface)"/>`);
|
| 426 |
-
svg.querySelectorAll('text').forEach(t => { if (!t.getAttribute('fill')) t.setAttribute('fill', t.classList.contains('name') ? 'var(--ink)' : 'var(--muted)'); });
|
| 427 |
-
return new XMLSerializer().serializeToString(svg).replace(/var\((--[a-z-]+)\)/g, (_, v) => css.getPropertyValue(v).trim());
|
| 428 |
-
}
|
| 429 |
function save(blob, name) {
|
| 430 |
const a = el('a', { href: URL.createObjectURL(blob), download: name });
|
| 431 |
a.click();
|
|
@@ -438,20 +418,7 @@ document.querySelectorAll('[data-dl]').forEach(b => b.onclick = () => {
|
|
| 438 |
normalized_error: r.norm_error, best_on: r.champions, s_per_1k: timeOf[r.entry], pareto: current.front.has(r.entry) }));
|
| 439 |
if (kind === 'csv') save(new Blob([[Object.keys(rows[0]).join(','), ...rows.map(r => Object.values(r).join(','))].join('\n')], { type: 'text/csv' }), base + '.csv');
|
| 440 |
if (kind === 'json') save(new Blob([JSON.stringify({ version: LATEST.version, date: LATEST.date, filter: current.label, rows }, null, 2)], { type: 'application/json' }), base + '.json');
|
| 441 |
-
if (kind === 'svg') save(new Blob([resolvedSVG()], { type: 'image/svg+xml' }), base + '-pareto.svg');
|
| 442 |
-
if (kind === 'png') {
|
| 443 |
-
const img = new Image();
|
| 444 |
-
img.onload = () => {
|
| 445 |
-
const c = el('canvas', { width: img.width * 2, height: img.height * 2 });
|
| 446 |
-
c.getContext('2d').drawImage(img, 0, 0, c.width, c.height);
|
| 447 |
-
c.toBlob(blob => save(blob, base + '-pareto.png'));
|
| 448 |
-
};
|
| 449 |
-
img.src = 'data:image/svg+xml;charset=utf-8,' + encodeURIComponent(resolvedSVG());
|
| 450 |
-
}
|
| 451 |
});
|
| 452 |
-
document.getElementById('facts').innerHTML = [
|
| 453 |
-
['datasets', D.meta.datasets], ['models', D.meta.models], ['entries', D.meta.entries.length], ['runs', fmt(D.meta.runs)]
|
| 454 |
-
].map(([k, v]) => `<span>${k} <b>${v}</b></span>`).join('');
|
| 455 |
|
| 456 |
|
| 457 |
const dsInfo = {};
|
|
@@ -551,14 +518,13 @@ function drawBoard(board, front) {
|
|
| 551 |
t.appendChild(tb);
|
| 552 |
}
|
| 553 |
|
| 554 |
-
const MOD_NAMES = { text: 'text', image: 'image', 'image-text': 'image + text' };
|
| 555 |
function renderBoard() {
|
| 556 |
const filterKey = keyOf(sel.tier, TIERS) + '|' + keyOf(sel.task, TASKS) + '|' + keyOf(sel.modality, MODS);
|
| 557 |
const board = D.boards[filterKey + '|' + encKey()];
|
| 558 |
const names = Object.keys(dsInfo).filter(inFilter);
|
| 559 |
const count = (field, key) => names.filter(d => dsInfo[d][field] === key).length;
|
| 560 |
const parts = [...TIERS.map(([k]) => [count('tier', k), k]), ...TASKS.map(([k]) => [count('task', k), k]),
|
| 561 |
-
...MODS.map(([k]) => [count('modality', k),
|
| 562 |
document.getElementById('boardNote').textContent = `${names.length} datasets: ${parts.map(([n, k]) => `${n} ${k}`).join(', ')}.` + (names.length < 10 ? ' Few datasets, so wide intervals.' : '');
|
| 563 |
const front = paretoFront(board);
|
| 564 |
current = { board, front, label: [...[sel.tier, sel.task, sel.modality].map((v, i) => keyOf(v, [TIERS, TASKS, MODS][i])), encKey()].join('_') };
|
|
|
|
| 50 |
:root:not([data-theme="light"]) .logo .light { display: none; }
|
| 51 |
:root:not([data-theme="light"]) .logo .dark { display: block; }
|
| 52 |
}
|
|
|
|
|
|
|
| 53 |
section { display: grid; gap: 16px; }
|
| 54 |
.head { display: flex; flex-wrap: wrap; align-items: end; justify-content: space-between; gap: 12px; }
|
| 55 |
.tabs { display: flex; flex-wrap: wrap; gap: 6px; }
|
|
|
|
| 103 |
#intro { font-size: 17px; line-height: 1.6; max-width: 84ch; }
|
| 104 |
.filters .label { font: 500 11.5px/1 var(--mono); letter-spacing: 0.05em; text-transform: uppercase; color: var(--muted); margin-right: 4px; }
|
| 105 |
.tabs button .n { font-family: var(--mono); opacity: .6; margin-left: 4px; }
|
| 106 |
+
.links a { color: var(--accent); white-space: nowrap; text-decoration: none; font-weight: 500; }
|
| 107 |
+
.links a:hover { text-decoration: underline; }
|
| 108 |
+
.links .ico { display: inline-block; width: 1.05em; height: 1.05em; margin-right: 5px; vertical-align: -0.15em; line-height: 1; }
|
| 109 |
code { font-family: var(--mono); font-size: 0.92em; overflow-wrap: anywhere; }
|
| 110 |
|
| 111 |
.topbar { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 12px 20px; }
|
|
|
|
| 141 |
</div>
|
| 142 |
<div class="version" id="version"></div>
|
| 143 |
<p class="muted" id="intro">MulTaBench is a benchmark for multimodal tabular learning with text and images. Its purpose is to drive smarter encoding methods, towards our vision of truly Multimodal Tabular Foundation Models. Its 80 datasets balance image and text modalities, classification and regression, and how hard the text or images are to encode, with the <b>Core</b> slice focusing on the 40 datasets where frozen embeddings are not enough.</p>
|
| 144 |
+
<p class="links"><a href="https://github.com/alanarazi7/MulTaBench"><svg class="ico" viewBox="0 0 16 16" aria-hidden="true"><path fill="currentColor" d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"/></svg>Code</a> 路 <a href="https://huggingface.co/multabench"><span class="ico">馃</span>Datasets</a> 路 <a href="https://arxiv.org/abs/2605.10616"><svg class="ico" viewBox="0 0 24 24" aria-hidden="true"><path fill="#B31B1B" d="M3.8423 0a1.0037 1.0037 0 0 0-.922.6078c-.1536.3687-.0438.6275.2938 1.1113l6.9185 8.3597-1.0223 1.1058a1.0393 1.0393 0 0 0 .003 1.4229l1.2292 1.3135-5.4391 6.4444c-.2803.299-.4538.823-.2971 1.1986a1.0253 1.0253 0 0 0 .9585.635.9133.9133 0 0 0 .6891-.3405l5.783-6.126 7.4902 8.0051a.8527.8527 0 0 0 .6835.2597.9575.9575 0 0 0 .8777-.6138c.1577-.377-.017-.7502-.306-1.1407l-7.0518-8.3418 1.0632-1.13a.9626.9626 0 0 0 .0089-1.3165L4.6336.4639s-.3733-.4535-.768-.463zm0 .272h.0166c.2179.0052.4874.2715.5644.3639l.005.006.0052.0055 10.169 10.9905a.6915.6915 0 0 1-.0072.945l-1.0666 1.133-1.4982-1.7724-8.5994-10.39c-.3286-.472-.352-.6183-.2592-.841a.7307.7307 0 0 1 .6704-.4401Zm14.341 1.5701a.877.877 0 0 0-.6554.2418l-5.6962 6.1584 1.6944 1.8319 5.3089-6.5138c.3251-.4335.479-.6603.3247-1.0292a1.1205 1.1205 0 0 0-.9763-.689zm-7.6557 12.2823 1.3186 1.4135-5.7864 6.1295a.6494.6494 0 0 1-.4959.26.7516.7516 0 0 1-.706-.4669c-.1119-.2682.0359-.6864.2442-.9083l.0051-.0055.0047-.0055z"/></svg>Paper</a> 路 <a href="https://github.com/alanarazi7/MulTaBench/blob/master/docs/protocol.md">Protocol</a></p>
|
|
|
|
| 145 |
</header>
|
| 146 |
|
| 147 |
<div class="sticky">
|
|
|
|
| 208 |
<h2>Using the leaderboard in your paper</h2>
|
| 209 |
<p class="muted" id="dlNote"></p>
|
| 210 |
<div class="panel dl">
|
|
|
|
| 211 |
<div><span class="label">Ranking table</span><button class="btn" data-dl="csv">CSV</button><button class="btn" data-dl="json">JSON</button><a class="btn" href="data/leaderboards.csv" download>CSV 路 every filter</a></div>
|
| 212 |
<div><span class="label">Per dataset and fold</span><a class="btn" href="data/per_dataset.csv" download>Per dataset CSV</a><a class="btn" href="https://github.com/alanarazi7/MulTaBench/tree/master/multabench/leaderboard/models">Every run, per model (GitHub)</a></div>
|
| 213 |
</div>
|
|
|
|
| 271 |
const info = Object.fromEntries((await get('models')).map(r => [r.model, r]));
|
| 272 |
const entries = boards['all|all|all|all'].map(r => r.entry).sort();
|
| 273 |
const encodingOf = Object.fromEntries(boards['all|all|all|all'].map(r => [r.entry, r.encoding]));
|
| 274 |
+
return { boards, encodingBoards, per_dataset, cost, updated, info, encodingOf, meta: { entries } };
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 275 |
}
|
| 276 |
load().then(start).catch(err => {
|
| 277 |
document.querySelector('.wrap').insertAdjacentHTML('afterbegin', `<p class="flag">Could not load the leaderboard data: ${err.message}</p>`);
|
|
|
|
| 285 |
const encodingCell = e => `<td><span class="chip ${e}" title="${ENC_TITLE[e]}">${ENC_LABEL[e]}</span></td>`;
|
| 286 |
const TIERS = [['core', 'Core'], ['extended', 'Extended']];
|
| 287 |
const TASKS = [['classification', 'Classification'], ['regression', 'Regression']];
|
| 288 |
+
const MODS = [['text', 'Text'], ['image', 'Image']];
|
| 289 |
const ENCS = [['frozen', 'Frozen'], ['tar', 'TAR']];
|
| 290 |
const taskOf = d => d.startsWith('REG_') ? 'regression' : 'classification';
|
| 291 |
const store = {
|
|
|
|
| 406 |
}
|
| 407 |
|
| 408 |
let current = { board: [], front: new Set(), label: '' };
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 409 |
function save(blob, name) {
|
| 410 |
const a = el('a', { href: URL.createObjectURL(blob), download: name });
|
| 411 |
a.click();
|
|
|
|
| 418 |
normalized_error: r.norm_error, best_on: r.champions, s_per_1k: timeOf[r.entry], pareto: current.front.has(r.entry) }));
|
| 419 |
if (kind === 'csv') save(new Blob([[Object.keys(rows[0]).join(','), ...rows.map(r => Object.values(r).join(','))].join('\n')], { type: 'text/csv' }), base + '.csv');
|
| 420 |
if (kind === 'json') save(new Blob([JSON.stringify({ version: LATEST.version, date: LATEST.date, filter: current.label, rows }, null, 2)], { type: 'application/json' }), base + '.json');
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 421 |
});
|
|
|
|
|
|
|
|
|
|
| 422 |
|
| 423 |
|
| 424 |
const dsInfo = {};
|
|
|
|
| 518 |
t.appendChild(tb);
|
| 519 |
}
|
| 520 |
|
|
|
|
| 521 |
function renderBoard() {
|
| 522 |
const filterKey = keyOf(sel.tier, TIERS) + '|' + keyOf(sel.task, TASKS) + '|' + keyOf(sel.modality, MODS);
|
| 523 |
const board = D.boards[filterKey + '|' + encKey()];
|
| 524 |
const names = Object.keys(dsInfo).filter(inFilter);
|
| 525 |
const count = (field, key) => names.filter(d => dsInfo[d][field] === key).length;
|
| 526 |
const parts = [...TIERS.map(([k]) => [count('tier', k), k]), ...TASKS.map(([k]) => [count('task', k), k]),
|
| 527 |
+
...MODS.map(([k]) => [count('modality', k), k])].filter(([n]) => n);
|
| 528 |
document.getElementById('boardNote').textContent = `${names.length} datasets: ${parts.map(([n, k]) => `${n} ${k}`).join(', ')}.` + (names.length < 10 ? ' Few datasets, so wide intervals.' : '');
|
| 529 |
const front = paretoFront(board);
|
| 530 |
current = { board, front, label: [...[sel.tier, sel.task, sel.modality].map((v, i) => keyOf(v, [TIERS, TASKS, MODS][i])), encKey()].join('_') };
|