Spaces:
Running
Running
exclude RT-J
Browse files- index.html +1 -1
index.html
CHANGED
|
@@ -13,4 +13,4 @@ td.m,th.m{text-align:left} td.best{font-weight:700}
|
|
| 13 |
td.rg,th.rg{text-align:left;color:#555;font-size:.8rem}
|
| 14 |
tr:hover td{background:#f6f8fb}
|
| 15 |
a{color:#1a5fc4}
|
| 16 |
-
</style></head><body><h1>RelBench Leaderboard</h1><p class=banner>We are redesigning the RelBench leaderboard to better reflect progress on Relational Foundation Models (RFMs) and Relational Deep Learning (RDL), expected to be live soon. Stay tuned!</p><h2>Classification</h2><p class=metric>Metric: AUROC on the official test split (higher is better).</p><div class=tablewrap><table><tr><th>#</th><th class=m>Method</th><th class=rg>Regime</th><th>Mean</th><th>rel-amazon<br>user-churn</th><th>rel-amazon<br>item-churn</th><th>rel-avito<br>user-visits</th><th>rel-avito<br>user-clicks</th><th>rel-event<br>user-repeat</th><th>rel-event<br>user-ignore</th><th>rel-f1<br>driver-dnf</th><th>rel-f1<br>driver-top3</th><th>rel-hm<br>user-churn</th><th>rel-stack<br>user-engagement</th><th>rel-stack<br>user-badge</th><th>rel-trial<br>study-outcome</th></tr><tr><td>1</td><td class=m>KumoRFM (fine-tuned)</td><td class=rg>task-specific</td><td class=best>81.1</td><td>70.5</td><td>82.8</td><td class=best>78.3</td><td>66.8</td><td>80.6</td><td>89.4</td><td>82.6</td><td class=best>99.6</td><td class=best>71.2</td><td>90.7</td><td>89.9</td><td>71.2</td></tr><tr><td>2</td><td class=m>PluRel (pretrained + fine-tuned)</td><td class=rg>task-specific</td><td>79.7</td><td>63.2</td><td>82.8</td><td>60.1</td><td>58.6</td><td>83.0</td><td class=best>91.2</td><td>80.1</td><td>89.3</td><td>63.8</td><td class=best>95.6</td><td class=best>94.3</td><td class=best>94.6</td></tr><tr><td>3</td><td class=m>KumoRFM-2 (in-context)</td><td class=rg>zero-shot</td><td>79.6</td><td>69.1</td><td>82.2</td><td>69.4</td><td>67.4</td><td>81.7</td><td>90.8</td><td class=best>84.6</td><td>92.2</td><td>69.3</td><td>89.4</td><td>87.2</td><td>72.0</td></tr><tr><td>4</td><td class=m>RT (pretrained + fine-tuned)</td><td class=rg>task-specific</td><td>78.9</td><td>70.8</td><td class=best>83.4</td><td>66.6</td><td>65.8</td><td>77.4</td><td>87.1</td><td>84.2</td><td>92.1</td><td>70.5</td><td>90.2</td><td>88.7</td><td>70.2</td></tr><tr><td>5</td><td class=m>GelGT</td><td class=rg>task-specific</td><td>78.7</td><td>70.5</td><td>83.0</td><td>67.0</td><td>68.4</td><td class=best>83.6</td><td>87.8</td><td>76.1</td><td>84.1</td><td>70.0</td><td>90.9</td><td>90.4</td><td>72.5</td></tr><tr><td>6</td><td class=m>RelAgent (GPT-5.2 agent)</td><td class=rg>task-specific</td><td>78.4</td><td>70.8</td><td>82.8</td><td>67.8</td><td>68.4</td><td>78.2</td><td>87.2</td><td>78.3</td><td>85.2</td><td>71.1</td><td>90.4</td><td>88.4</td><td>71.9</td></tr><tr><td>7</td><td class=m>RGP</td><td class=rg>task-specific</td><td>78.2</td><td>70.9</td><td>82.6</td><td>66.6</td><td class=best>69.4</td><td>78.9</td><td>84.4</td><td>78.4</td><td>87.9</td><td>70.2</td><td>90.5</td><td>88.7</td><td>70.3</td></tr><tr><td>8</td><td class=m>RelGNN</td><td class=rg>task-specific</td><td>78.1</td><td>71.0</td><td>82.6</td><td>66.2</td><td>68.2</td><td>79.6</td><td>86.2</td><td>75.3</td><td>85.7</td><td>70.9</td><td>90.8</td><td>89.0</td><td>71.2</td></tr><tr><td>9</td><td class=m>Rel-LLM (Llama-3.2-1B + GNN soft prompts, fine-tuned)</td><td class=rg>task-specific</td><td>77.8</td><td class=best>71.9</td><td>83.4</td><td>67.0</td><td>66.7</td><td>79.3</td><td>83.7</td><td>77.1</td><td>82.2</td><td>70.5</td><td>91.2</td><td>89.6</td><td>71.0</td></tr><tr><td>10</td><td class=m>RT (from scratch)</td><td class=rg>task-specific</td><td>77.1</td><td>70.5</td><td>83.2</td><td>65.0</td><td>63.6</td><td>79.7</td><td>85.1</td><td>78.7</td><td>82.7</td><td>69.9</td><td>90.0</td><td>88.5</td><td>68.6</td></tr><tr><td>11</td><td class=m>KumoRFM (in-context)</td><td class=rg>zero-shot</td><td>76.7</td><td>67.3</td><td>79.9</td><td>64.8</td><td>64.1</td><td>76.1</td><td>89.2</td><td>82.4</td><td>91.1</td><td>67.7</td><td>87.1</td><td>80.0</td><td>70.8</td></tr><tr><td>12</td><td class=m>RelGT</td><td class=rg>task-specific</td><td>76.6</td><td>70.4</td><td>82.5</td><td>66.8</td><td>68.3</td><td>76.1</td><td>81.6</td><td>75.9</td><td>83.5</td><td>69.3</td><td>90.5</td><td>86.3</td><td>68.6</td></tr><tr><td>13</td><td class=m>RDL (GraphSAGE)</td><td class=rg>task-specific</td><td>75.8</td><td>70.4</td><td>82.8</td><td>66.2</td><td>65.9</td><td>76.9</td><td>81.6</td><td>72.6</td><td>75.5</td><td>69.9</td><td>90.6</td><td>88.9</td><td>68.6</td></tr><tr><td>14</td><td class=m>GIN</td><td class=rg>task-specific</td><td>75.2</td><td>70.5</td><td>82.7</td><td>66.0</td><td>66.0</td><td>74.4</td><td>79.5</td><td>71.8</td><td>73.6</td><td>69.9</td><td>90.5</td><td>88.7</td><td>68.4</td></tr><tr><td>15</td><td class=m>RT-J (tuned + ensembled context)</td><td class=rg>zero-shot</td><td>74.7</td><td>68.7</td><td>79.0</td><td>60.1</td><td>59.1</td><td>76.8</td><td>85.6</td><td>82.5</td><td>91.7</td><td>67.1</td><td>83.9</td><td>75.2</td><td>66.5</td></tr><tr><td>16</td><td class=m>RDB-PFN (fine-tuned)</td><td class=rg>task-specific</td><td>73.7</td><td>65.8</td><td>80.5</td><td>66.0</td><td>64.6</td><td>74.6</td><td>82.8</td><td>72.3</td><td>73.6</td><td>67.4</td><td>88.3</td><td>84.5</td><td>64.3</td></tr><tr><td>17</td><td class=m>RDB-PFN (ICL, 1,024-example context)</td><td class=rg>zero-shot</td><td>73.2</td><td>64.8</td><td>78.2</td><td>65.5</td><td>62.7</td><td>75.3</td><td>82.7</td><td>71.9</td><td>81.2</td><td>66.5</td><td>86.6</td><td>81.3</td><td>61.6</td></tr><tr><td>18</td><td class=m>RT-J (default context)</td><td class=rg>zero-shot</td><td>73.1</td><td>66.9</td><td>77.6</td><td>57.1</td><td>58.8</td><td>74.1</td><td>84.5</td><td>80.9</td><td>91.4</td><td>64.4</td><td>83.7</td><td>74.4</td><td>62.9</td></tr><tr><td>19</td><td class=m>TabPFN-2.5 + DFS (ICL, 1,024-example context)</td><td class=rg>zero-shot</td><td>72.9</td><td>64.5</td><td>79.6</td><td>61.7</td><td>63.3</td><td>73.1</td><td>83.2</td><td>71.7</td><td>80.4</td><td>66.8</td><td>85.3</td><td>82.1</td><td>62.6</td></tr><tr><td>20</td><td class=m>RELATE (RelGNN backbone)</td><td class=rg>task-specific</td><td>72.8</td><td>68.9</td><td>81.2</td><td>66.2</td><td>66.1</td><td>67.1</td><td>81.1</td><td>68.9</td><td>69.0</td><td>69.4</td><td>90.1</td><td>86.6</td><td>58.4</td></tr><tr><td>21</td><td class=m>TabICL v1.1 + DFS (ICL, 1,024-example context)</td><td class=rg>zero-shot</td><td>72.4</td><td>64.8</td><td>78.9</td><td>64.4</td><td>61.8</td><td>70.0</td><td>80.8</td><td>71.7</td><td>80.6</td><td>66.6</td><td>85.4</td><td>83.0</td><td>60.8</td></tr><tr><td>22</td><td class=m>HGT+PE (Laplacian positional encodings)</td><td class=rg>task-specific</td><td>72.2</td><td>66.2</td><td>78.0</td><td>65.0</td><td>64.6</td><td>65.4</td><td>81.6</td><td>71.2</td><td>76.3</td><td>65.7</td><td>88.2</td><td>85.7</td><td>59.2</td></tr><tr><td>23</td><td class=m>HGT</td><td class=rg>task-specific</td><td>71.8</td><td>66.4</td><td>78.0</td><td>64.3</td><td>63.8</td><td>65.0</td><td>82.5</td><td>70.8</td><td>70.8</td><td>67.0</td><td>88.5</td><td>86.1</td><td>58.4</td></tr><tr><td>24</td><td class=m>PluRel (synthetic + real)</td><td class=rg>zero-shot</td><td>71.8</td><td>65.0</td><td>72.5</td><td>63.4</td><td>47.9</td><td>76.0</td><td>81.0</td><td>81.0</td><td>88.4</td><td>66.0</td><td>86.2</td><td>82.0</td><td>51.8</td></tr><tr><td>25</td><td class=m>RT (zero-shot, leave-one-DB-out)</td><td class=rg>zero-shot</td><td>71.1</td><td>64.0</td><td>70.9</td><td>61.8</td><td>59.5</td><td>72.6</td><td>83.6</td><td>81.2</td><td>89.3</td><td>62.8</td><td>75.7</td><td>80.1</td><td>51.8</td></tr><tr><td>26</td><td class=m>GAT</td><td class=rg>task-specific</td><td>70.8</td><td>63.2</td><td>70.0</td><td>64.8</td><td>65.8</td><td>68.2</td><td>82.0</td><td>70.3</td><td>60.0</td><td>64.7</td><td>89.6</td><td>84.5</td><td>66.2</td></tr><tr><td>27</td><td class=m>RELATE (HGT+PE backbone)</td><td class=rg>task-specific</td><td>69.6</td><td>65.5</td><td>75.1</td><td>62.6</td><td>64.3</td><td>72.3</td><td>85.1</td><td>66.5</td><td>47.8</td><td>65.2</td><td>88.0</td><td>82.3</td><td>59.8</td></tr><tr><td>28</td><td class=m>PluRel (synthetic only)</td><td class=rg>zero-shot</td><td>68.2</td><td>64.4</td><td>71.0</td><td>63.5</td><td>45.9</td><td>53.1</td><td>80.1</td><td>76.7</td><td>82.6</td><td>63.7</td><td>82.4</td><td>81.4</td><td>53.8</td></tr><tr><td>29</td><td class=m>LightGBM (raw entity features)</td><td class=rg>task-specific</td><td>63.7</td><td>52.2</td><td>62.5</td><td>53.0</td><td>53.6</td><td>68.0</td><td>79.9</td><td>68.6</td><td>73.9</td><td>55.2</td><td>63.4</td><td>63.4</td><td>70.1</td></tr></table></div><h2>Regression</h2><p class=metric>Metric: NMAE = MAE / train-split std, on the official test split (lower is better).</p><div class=tablewrap><table><tr><th>#</th><th class=m>Method</th><th class=rg>Regime</th><th>Mean</th><th>rel-amazon<br>user-ltv</th><th>rel-amazon<br>item-ltv</th><th>rel-avito<br>ad-ctr</th><th>rel-event<br>user-attendance</th><th>rel-f1<br>driver-position</th><th>rel-hm<br>item-sales</th><th>rel-stack<br>post-votes</th><th>rel-trial<br>study-adverse</th><th>rel-trial<br>site-success</th></tr><tr><td>1</td><td class=m>RT (pretrained + fine-tuned)</td><td class=rg>task-specific</td><td class=best>0.2328</td><td>0.2569</td><td>0.0804</td><td>0.4319</td><td class=best>0.0303</td><td>0.3757</td><td>0.0948</td><td>0.1455</td><td>0.1275</td><td>0.5519</td></tr><tr><td>2</td><td class=m>PluRel (pretrained + fine-tuned)</td><td class=rg>task-specific</td><td>0.2370</td><td>0.2672</td><td>0.0840</td><td>0.3923</td><td>0.0708</td><td class=best>0.3745</td><td>0.0966</td><td>0.1472</td><td>0.1240</td><td>0.5766</td></tr><tr><td>3</td><td class=m>KumoRFM (fine-tuned)</td><td class=rg>task-specific</td><td>0.2604</td><td>0.2474</td><td>0.0824</td><td>0.3554</td><td>0.3110</td><td>0.3887</td><td class=best>0.0686</td><td>0.1273</td><td>0.1304</td><td>0.6325</td></tr><tr><td>4</td><td class=m>RT-J (tuned + ensembled context)</td><td class=rg>zero-shot</td><td>0.2612</td><td>0.3018</td><td>0.1026</td><td>0.4425</td><td>0.5531</td><td>0.4118</td><td>0.1173</td><td class=best>0.1145</td><td>0.1518</td><td class=best>0.1553</td></tr><tr><td>5</td><td class=m>RT-J (default context)</td><td class=rg>zero-shot</td><td>0.2677</td><td>0.2876</td><td>0.0871</td><td>0.4603</td><td>0.5534</td><td>0.4123</td><td>0.1035</td><td>0.1411</td><td>0.1547</td><td>0.2092</td></tr><tr><td>6</td><td class=m>RelGNN</td><td class=rg>task-specific</td><td>0.2854</td><td>0.2475</td><td>0.0825</td><td>0.3867</td><td>0.3110</td><td>0.5406</td><td>0.1090</td><td>0.1273</td><td>0.1311</td><td>0.6325</td></tr><tr><td>7</td><td class=m>PluRel (synthetic + real)</td><td class=rg>zero-shot</td><td>0.2898</td><td>0.2852</td><td>0.1041</td><td>0.4182</td><td>0.0878</td><td>0.4835</td><td>0.1555</td><td>0.1654</td><td>0.1731</td><td>0.7350</td></tr><tr><td>8</td><td class=m>KumoRFM-2 (in-context)</td><td class=rg>zero-shot</td><td>0.2913</td><td class=best>0.2421</td><td>0.0795</td><td>0.3554</td><td>0.3071</td><td>0.4062</td><td class=best>0.0686</td><td>0.1254</td><td>0.1277</td><td>0.9099</td></tr><tr><td>9</td><td class=m>RelGT</td><td class=rg>task-specific</td><td>0.2920</td><td>0.2481</td><td>0.0828</td><td>0.3606</td><td>0.3270</td><td>0.5575</td><td>0.1082</td><td>0.1281</td><td>0.1297</td><td>0.6857</td></tr><tr><td>10</td><td class=m>GelGT</td><td class=rg>task-specific</td><td>0.2951</td><td>0.2479</td><td>0.0833</td><td>0.3784</td><td>0.3167</td><td>0.5315</td><td>0.1131</td><td>0.1270</td><td>0.1255</td><td>0.7324</td></tr><tr><td>11</td><td class=m>RelAgent (GPT-5.2 agent)</td><td class=rg>task-specific</td><td>0.2958</td><td>0.2426</td><td>0.0707</td><td class=best>0.3449</td><td>0.3150</td><td>0.5720</td><td>0.0707</td><td>0.1254</td><td class=best>0.1097</td><td>0.8112</td></tr><tr><td>12</td><td class=m>KumoRFM (in-context)</td><td class=rg>zero-shot</td><td>0.3036</td><td>0.2810</td><td>0.0935</td><td>0.3658</td><td>0.3450</td><td>0.3910</td><td>0.0808</td><td>0.1273</td><td>0.1717</td><td>0.8763</td></tr><tr><td>13</td><td class=m>Rel-LLM (Llama-3.2-1B + GNN soft prompts, fine-tuned)</td><td class=rg>task-specific</td><td>0.3106</td><td>0.2450</td><td>0.0816</td><td>0.3867</td><td>0.3280</td><td>0.5646</td><td>0.1050</td><td>0.1215</td><td>0.1288</td><td>0.8343</td></tr><tr><td>14</td><td class=m>PluRel (synthetic only)</td><td class=rg>zero-shot</td><td>0.3110</td><td>0.3388</td><td>0.1154</td><td>0.4252</td><td>0.0878</td><td>0.5426</td><td>0.1749</td><td>0.1800</td><td>0.1889</td><td>0.7457</td></tr><tr><td>15</td><td class=m>RT (from scratch)</td><td class=rg>task-specific</td><td>0.3159</td><td>0.2590</td><td>0.0845</td><td>0.4064</td><td>0.5040</td><td>0.4775</td><td>0.1001</td><td>0.1471</td><td>0.1306</td><td>0.7341</td></tr><tr><td>16</td><td class=m>Data Scientist + LightGBM</td><td class=rg>task-specific</td><td>0.3202</td><td>0.2422</td><td class=best>0.0696</td><td>0.4599</td><td>0.3712</td><td>0.5641</td><td>0.0727</td><td>0.1273</td><td>0.1197</td><td>0.8553</td></tr><tr><td>17</td><td class=m>RDL (GraphSAGE)</td><td class=rg>task-specific</td><td>0.3204</td><td>0.2489</td><td>0.0847</td><td>0.4285</td><td>0.3372</td><td>0.5725</td><td>0.1131</td><td>0.1273</td><td>0.1311</td><td>0.8406</td></tr><tr><td>18</td><td class=m>GIN</td><td class=rg>task-specific</td><td>0.3214</td><td>0.2490</td><td>0.0848</td><td>0.4285</td><td>0.3450</td><td>0.5796</td><td>0.1110</td><td>0.1273</td><td>0.1309</td><td>0.8364</td></tr><tr><td>19</td><td class=m>Data Scientist + AutoGluon</td><td class=rg>task-specific</td><td>0.3325</td><td>0.2504</td><td>0.0768</td><td>0.4703</td><td>0.3346</td><td>0.6051</td><td>0.0868</td><td>0.1332</td><td>0.1318</td><td>0.9036</td></tr><tr><td>20</td><td class=m>GAT</td><td class=rg>task-specific</td><td>0.3382</td><td>0.2891</td><td>0.0997</td><td>0.4494</td><td>0.3437</td><td>0.6075</td><td>0.1595</td><td>0.1332</td><td>0.1357</td><td>0.8259</td></tr><tr><td>21</td><td class=m>LightGBM (raw entity features)</td><td class=rg>task-specific</td><td>0.3412</td><td>0.2919</td><td>0.1025</td><td>0.4285</td><td>0.3450</td><td>0.5935</td><td>0.1534</td><td>0.1332</td><td>0.1298</td><td>0.8931</td></tr><tr><td>22</td><td class=m>RT (zero-shot, leave-one-DB-out)</td><td class=rg>zero-shot</td><td>0.3461</td><td>0.3277</td><td>0.1029</td><td>0.6235</td><td>0.0662</td><td>0.4310</td><td>0.1719</td><td>0.2128</td><td>0.2233</td><td>0.9552</td></tr><tr><td>23</td><td class=m>HGT</td><td class=rg>task-specific</td><td>0.3464</td><td>0.2680</td><td>0.0945</td><td>0.4829</td><td>0.3444</td><td>0.6015</td><td>0.1294</td><td>0.1330</td><td>0.1332</td><td>0.9305</td></tr><tr><td>24</td><td class=m>HGT+PE (Laplacian positional encodings)</td><td class=rg>task-specific</td><td>0.3504</td><td>0.2759</td><td>0.0945</td><td>0.5048</td><td>0.3412</td><td>0.6251</td><td>0.1290</td><td>0.1332</td><td>0.1258</td><td>0.9238</td></tr><tr><td>25</td><td class=m>Griffin (fine-tuned)</td><td class=rg>task-specific</td><td>0.3686</td><td>0.3409</td><td>0.1130</td><td>0.4526</td><td>0.4846</td><td>0.5596</td><td>0.1205</td><td>0.2733</td><td>0.1743</td><td>0.7988</td></tr><tr><td>26</td><td class=m>Entity Median</td><td class=rg>task-specific</td><td>0.4278</td><td>0.3030</td><td>0.1124</td><td>0.4808</td><td>0.3516</td><td>1.2125</td><td>0.1575</td><td>0.1352</td><td>0.1708</td><td>0.9267</td></tr><tr><td>27</td><td class=m>Entity Mean</td><td class=rg>task-specific</td><td>0.4551</td><td>0.3314</td><td>0.1327</td><td>0.4808</td><td>0.3973</td><td>1.2100</td><td>0.2241</td><td>0.2077</td><td>0.1708</td><td>0.9414</td></tr></table></div><h2>Recommendation</h2><p class=metric>Metric: MAP on the official test split (higher is better).</p><div class=tablewrap><table><tr><th>#</th><th class=m>Method</th><th class=rg>Regime</th><th>Mean</th><th>rel-amazon<br>user-item-purchase</th><th>rel-amazon<br>user-item-rate</th><th>rel-amazon<br>user-item-review</th><th>rel-avito<br>user-ad-visit</th><th>rel-f1<br>driver-circuit-compete</th><th>rel-hm<br>user-item-purchase</th><th>rel-stack<br>user-post-comment</th><th>rel-stack<br>post-post-related</th><th>rel-trial<br>condition-sponsor-run</th><th>rel-trial<br>site-sponsor-run</th></tr><tr><td>1</td><td class=m>ID-GNN (4 layers)</td><td class=rg>task-specific</td><td class=best>14.0</td><td>0.1</td><td>0.1</td><td>0.1</td><td class=best>3.9</td><td class=best>76.2</td><td class=best>2.9</td><td class=best>13.8</td><td class=best>12.5</td><td>11.3</td><td class=best>19.0</td></tr><tr><td>2</td><td class=m>ID-GNN (2 layers)</td><td class=rg>task-specific</td><td>12.3</td><td>0.1</td><td>0.1</td><td>0.1</td><td>3.6</td><td>62.3</td><td>2.8</td><td>12.7</td><td>10.7</td><td class=best>11.4</td><td>19.0</td></tr><tr><td>3</td><td class=m>LightGBM (entity features + heuristic ranks)</td><td class=rg>task-specific</td><td>7.3</td><td>0.1</td><td>0.2</td><td>0.1</td><td>0.1</td><td>57.8</td><td>0.4</td><td>0.0</td><td>1.9</td><td>4.5</td><td>8.2</td></tr><tr><td>4</td><td class=m>Global Popularity</td><td class=rg>task-specific</td><td>5.9</td><td>0.2</td><td>0.1</td><td>0.1</td><td>0.0</td><td>50.1</td><td>0.3</td><td>0.0</td><td>1.5</td><td>2.5</td><td>3.8</td></tr><tr><td>5</td><td class=m>Past Visit</td><td class=rg>task-specific</td><td>5.3</td><td>0.1</td><td>0.1</td><td>0.0</td><td>1.9</td><td>20.8</td><td>0.9</td><td>1.4</td><td>1.7</td><td>8.4</td><td>17.3</td></tr><tr><td>6</td><td class=m>GraphSAGE (two-tower) (4 layers)</td><td class=rg>task-specific</td><td>3.4</td><td class=best>0.9</td><td class=best>1.0</td><td class=best>0.6</td><td>0.1</td><td>16.6</td><td>0.7</td><td>0.2</td><td>0.1</td><td>2.7</td><td>11.1</td></tr><tr><td>7</td><td class=m>GraphSAGE (two-tower) (2 layers)</td><td class=rg>task-specific</td><td>2.6</td><td>0.7</td><td>0.8</td><td>0.5</td><td>0.0</td><td>9.7</td><td>0.8</td><td>0.2</td><td>0.0</td><td>3.1</td><td>10.4</td></tr></table></div></body></html>
|
|
|
|
| 13 |
td.rg,th.rg{text-align:left;color:#555;font-size:.8rem}
|
| 14 |
tr:hover td{background:#f6f8fb}
|
| 15 |
a{color:#1a5fc4}
|
| 16 |
+
</style></head><body><h1>RelBench Leaderboard</h1><p class=banner>We are redesigning the RelBench leaderboard to better reflect progress on Relational Foundation Models (RFMs) and Relational Deep Learning (RDL), expected to be live soon. Stay tuned!</p><h2>Classification</h2><p class=metric>Metric: AUROC on the official test split (higher is better).</p><div class=tablewrap><table><tr><th>#</th><th class=m>Method</th><th class=rg>Regime</th><th>Mean</th><th>rel-amazon<br>user-churn</th><th>rel-amazon<br>item-churn</th><th>rel-avito<br>user-visits</th><th>rel-avito<br>user-clicks</th><th>rel-event<br>user-repeat</th><th>rel-event<br>user-ignore</th><th>rel-f1<br>driver-dnf</th><th>rel-f1<br>driver-top3</th><th>rel-hm<br>user-churn</th><th>rel-stack<br>user-engagement</th><th>rel-stack<br>user-badge</th><th>rel-trial<br>study-outcome</th></tr><tr><td>1</td><td class=m>KumoRFM (fine-tuned)</td><td class=rg>task-specific</td><td class=best>81.1</td><td>70.5</td><td>82.8</td><td class=best>78.3</td><td>66.8</td><td>80.6</td><td>89.4</td><td>82.6</td><td class=best>99.6</td><td class=best>71.2</td><td>90.7</td><td>89.9</td><td>71.2</td></tr><tr><td>2</td><td class=m>PluRel (pretrained + fine-tuned)</td><td class=rg>task-specific</td><td>79.7</td><td>63.2</td><td>82.8</td><td>60.1</td><td>58.6</td><td>83.0</td><td class=best>91.2</td><td>80.1</td><td>89.3</td><td>63.8</td><td class=best>95.6</td><td class=best>94.3</td><td class=best>94.6</td></tr><tr><td>3</td><td class=m>KumoRFM-2 (in-context)</td><td class=rg>zero-shot</td><td>79.6</td><td>69.1</td><td>82.2</td><td>69.4</td><td>67.4</td><td>81.7</td><td>90.8</td><td class=best>84.6</td><td>92.2</td><td>69.3</td><td>89.4</td><td>87.2</td><td>72.0</td></tr><tr><td>4</td><td class=m>RT (pretrained + fine-tuned)</td><td class=rg>task-specific</td><td>78.9</td><td>70.8</td><td class=best>83.4</td><td>66.6</td><td>65.8</td><td>77.4</td><td>87.1</td><td>84.2</td><td>92.1</td><td>70.5</td><td>90.2</td><td>88.7</td><td>70.2</td></tr><tr><td>5</td><td class=m>GelGT</td><td class=rg>task-specific</td><td>78.7</td><td>70.5</td><td>83.0</td><td>67.0</td><td>68.4</td><td class=best>83.6</td><td>87.8</td><td>76.1</td><td>84.1</td><td>70.0</td><td>90.9</td><td>90.4</td><td>72.5</td></tr><tr><td>6</td><td class=m>RelAgent (GPT-5.2 agent)</td><td class=rg>task-specific</td><td>78.4</td><td>70.8</td><td>82.8</td><td>67.8</td><td>68.4</td><td>78.2</td><td>87.2</td><td>78.3</td><td>85.2</td><td>71.1</td><td>90.4</td><td>88.4</td><td>71.9</td></tr><tr><td>7</td><td class=m>RGP</td><td class=rg>task-specific</td><td>78.2</td><td>70.9</td><td>82.6</td><td>66.6</td><td class=best>69.4</td><td>78.9</td><td>84.4</td><td>78.4</td><td>87.9</td><td>70.2</td><td>90.5</td><td>88.7</td><td>70.3</td></tr><tr><td>8</td><td class=m>RelGNN</td><td class=rg>task-specific</td><td>78.1</td><td>71.0</td><td>82.6</td><td>66.2</td><td>68.2</td><td>79.6</td><td>86.2</td><td>75.3</td><td>85.7</td><td>70.9</td><td>90.8</td><td>89.0</td><td>71.2</td></tr><tr><td>9</td><td class=m>Rel-LLM (Llama-3.2-1B + GNN soft prompts, fine-tuned)</td><td class=rg>task-specific</td><td>77.8</td><td class=best>71.9</td><td>83.4</td><td>67.0</td><td>66.7</td><td>79.3</td><td>83.7</td><td>77.1</td><td>82.2</td><td>70.5</td><td>91.2</td><td>89.6</td><td>71.0</td></tr><tr><td>10</td><td class=m>RT (from scratch)</td><td class=rg>task-specific</td><td>77.1</td><td>70.5</td><td>83.2</td><td>65.0</td><td>63.6</td><td>79.7</td><td>85.1</td><td>78.7</td><td>82.7</td><td>69.9</td><td>90.0</td><td>88.5</td><td>68.6</td></tr><tr><td>11</td><td class=m>KumoRFM (in-context)</td><td class=rg>zero-shot</td><td>76.7</td><td>67.3</td><td>79.9</td><td>64.8</td><td>64.1</td><td>76.1</td><td>89.2</td><td>82.4</td><td>91.1</td><td>67.7</td><td>87.1</td><td>80.0</td><td>70.8</td></tr><tr><td>12</td><td class=m>RelGT</td><td class=rg>task-specific</td><td>76.6</td><td>70.4</td><td>82.5</td><td>66.8</td><td>68.3</td><td>76.1</td><td>81.6</td><td>75.9</td><td>83.5</td><td>69.3</td><td>90.5</td><td>86.3</td><td>68.6</td></tr><tr><td>13</td><td class=m>RDL (GraphSAGE)</td><td class=rg>task-specific</td><td>75.8</td><td>70.4</td><td>82.8</td><td>66.2</td><td>65.9</td><td>76.9</td><td>81.6</td><td>72.6</td><td>75.5</td><td>69.9</td><td>90.6</td><td>88.9</td><td>68.6</td></tr><tr><td>14</td><td class=m>GIN</td><td class=rg>task-specific</td><td>75.2</td><td>70.5</td><td>82.7</td><td>66.0</td><td>66.0</td><td>74.4</td><td>79.5</td><td>71.8</td><td>73.6</td><td>69.9</td><td>90.5</td><td>88.7</td><td>68.4</td></tr><tr><td>15</td><td class=m>RDB-PFN (fine-tuned)</td><td class=rg>task-specific</td><td>73.7</td><td>65.8</td><td>80.5</td><td>66.0</td><td>64.6</td><td>74.6</td><td>82.8</td><td>72.3</td><td>73.6</td><td>67.4</td><td>88.3</td><td>84.5</td><td>64.3</td></tr><tr><td>16</td><td class=m>RDB-PFN (ICL, 1,024-example context)</td><td class=rg>zero-shot</td><td>73.2</td><td>64.8</td><td>78.2</td><td>65.5</td><td>62.7</td><td>75.3</td><td>82.7</td><td>71.9</td><td>81.2</td><td>66.5</td><td>86.6</td><td>81.3</td><td>61.6</td></tr><tr><td>17</td><td class=m>TabPFN-2.5 + DFS (ICL, 1,024-example context)</td><td class=rg>zero-shot</td><td>72.9</td><td>64.5</td><td>79.6</td><td>61.7</td><td>63.3</td><td>73.1</td><td>83.2</td><td>71.7</td><td>80.4</td><td>66.8</td><td>85.3</td><td>82.1</td><td>62.6</td></tr><tr><td>18</td><td class=m>RELATE (RelGNN backbone)</td><td class=rg>task-specific</td><td>72.8</td><td>68.9</td><td>81.2</td><td>66.2</td><td>66.1</td><td>67.1</td><td>81.1</td><td>68.9</td><td>69.0</td><td>69.4</td><td>90.1</td><td>86.6</td><td>58.4</td></tr><tr><td>19</td><td class=m>TabICL v1.1 + DFS (ICL, 1,024-example context)</td><td class=rg>zero-shot</td><td>72.4</td><td>64.8</td><td>78.9</td><td>64.4</td><td>61.8</td><td>70.0</td><td>80.8</td><td>71.7</td><td>80.6</td><td>66.6</td><td>85.4</td><td>83.0</td><td>60.8</td></tr><tr><td>20</td><td class=m>HGT+PE (Laplacian positional encodings)</td><td class=rg>task-specific</td><td>72.2</td><td>66.2</td><td>78.0</td><td>65.0</td><td>64.6</td><td>65.4</td><td>81.6</td><td>71.2</td><td>76.3</td><td>65.7</td><td>88.2</td><td>85.7</td><td>59.2</td></tr><tr><td>21</td><td class=m>HGT</td><td class=rg>task-specific</td><td>71.8</td><td>66.4</td><td>78.0</td><td>64.3</td><td>63.8</td><td>65.0</td><td>82.5</td><td>70.8</td><td>70.8</td><td>67.0</td><td>88.5</td><td>86.1</td><td>58.4</td></tr><tr><td>22</td><td class=m>PluRel (synthetic + real)</td><td class=rg>zero-shot</td><td>71.8</td><td>65.0</td><td>72.5</td><td>63.4</td><td>47.9</td><td>76.0</td><td>81.0</td><td>81.0</td><td>88.4</td><td>66.0</td><td>86.2</td><td>82.0</td><td>51.8</td></tr><tr><td>23</td><td class=m>RT (zero-shot, leave-one-DB-out)</td><td class=rg>zero-shot</td><td>71.1</td><td>64.0</td><td>70.9</td><td>61.8</td><td>59.5</td><td>72.6</td><td>83.6</td><td>81.2</td><td>89.3</td><td>62.8</td><td>75.7</td><td>80.1</td><td>51.8</td></tr><tr><td>24</td><td class=m>GAT</td><td class=rg>task-specific</td><td>70.8</td><td>63.2</td><td>70.0</td><td>64.8</td><td>65.8</td><td>68.2</td><td>82.0</td><td>70.3</td><td>60.0</td><td>64.7</td><td>89.6</td><td>84.5</td><td>66.2</td></tr><tr><td>25</td><td class=m>RELATE (HGT+PE backbone)</td><td class=rg>task-specific</td><td>69.6</td><td>65.5</td><td>75.1</td><td>62.6</td><td>64.3</td><td>72.3</td><td>85.1</td><td>66.5</td><td>47.8</td><td>65.2</td><td>88.0</td><td>82.3</td><td>59.8</td></tr><tr><td>26</td><td class=m>PluRel (synthetic only)</td><td class=rg>zero-shot</td><td>68.2</td><td>64.4</td><td>71.0</td><td>63.5</td><td>45.9</td><td>53.1</td><td>80.1</td><td>76.7</td><td>82.6</td><td>63.7</td><td>82.4</td><td>81.4</td><td>53.8</td></tr><tr><td>27</td><td class=m>LightGBM (raw entity features)</td><td class=rg>task-specific</td><td>63.7</td><td>52.2</td><td>62.5</td><td>53.0</td><td>53.6</td><td>68.0</td><td>79.9</td><td>68.6</td><td>73.9</td><td>55.2</td><td>63.4</td><td>63.4</td><td>70.1</td></tr></table></div><h2>Regression</h2><p class=metric>Metric: NMAE = MAE / train-split std, on the official test split (lower is better).</p><div class=tablewrap><table><tr><th>#</th><th class=m>Method</th><th class=rg>Regime</th><th>Mean</th><th>rel-amazon<br>user-ltv</th><th>rel-amazon<br>item-ltv</th><th>rel-avito<br>ad-ctr</th><th>rel-event<br>user-attendance</th><th>rel-f1<br>driver-position</th><th>rel-hm<br>item-sales</th><th>rel-stack<br>post-votes</th><th>rel-trial<br>study-adverse</th><th>rel-trial<br>site-success</th></tr><tr><td>1</td><td class=m>RT (pretrained + fine-tuned)</td><td class=rg>task-specific</td><td class=best>0.2328</td><td>0.2569</td><td>0.0804</td><td>0.4319</td><td class=best>0.0303</td><td>0.3757</td><td>0.0948</td><td>0.1455</td><td>0.1275</td><td class=best>0.5519</td></tr><tr><td>2</td><td class=m>PluRel (pretrained + fine-tuned)</td><td class=rg>task-specific</td><td>0.2370</td><td>0.2672</td><td>0.0840</td><td>0.3923</td><td>0.0708</td><td class=best>0.3745</td><td>0.0966</td><td>0.1472</td><td>0.1240</td><td>0.5766</td></tr><tr><td>3</td><td class=m>KumoRFM (fine-tuned)</td><td class=rg>task-specific</td><td>0.2604</td><td>0.2474</td><td>0.0824</td><td>0.3554</td><td>0.3110</td><td>0.3887</td><td class=best>0.0686</td><td>0.1273</td><td>0.1304</td><td>0.6325</td></tr><tr><td>4</td><td class=m>RelGNN</td><td class=rg>task-specific</td><td>0.2854</td><td>0.2475</td><td>0.0825</td><td>0.3867</td><td>0.3110</td><td>0.5406</td><td>0.1090</td><td>0.1273</td><td>0.1311</td><td>0.6325</td></tr><tr><td>5</td><td class=m>PluRel (synthetic + real)</td><td class=rg>zero-shot</td><td>0.2898</td><td>0.2852</td><td>0.1041</td><td>0.4182</td><td>0.0878</td><td>0.4835</td><td>0.1555</td><td>0.1654</td><td>0.1731</td><td>0.7350</td></tr><tr><td>6</td><td class=m>KumoRFM-2 (in-context)</td><td class=rg>zero-shot</td><td>0.2913</td><td class=best>0.2421</td><td>0.0795</td><td>0.3554</td><td>0.3071</td><td>0.4062</td><td class=best>0.0686</td><td>0.1254</td><td>0.1277</td><td>0.9099</td></tr><tr><td>7</td><td class=m>RelGT</td><td class=rg>task-specific</td><td>0.2920</td><td>0.2481</td><td>0.0828</td><td>0.3606</td><td>0.3270</td><td>0.5575</td><td>0.1082</td><td>0.1281</td><td>0.1297</td><td>0.6857</td></tr><tr><td>8</td><td class=m>GelGT</td><td class=rg>task-specific</td><td>0.2951</td><td>0.2479</td><td>0.0833</td><td>0.3784</td><td>0.3167</td><td>0.5315</td><td>0.1131</td><td>0.1270</td><td>0.1255</td><td>0.7324</td></tr><tr><td>9</td><td class=m>RelAgent (GPT-5.2 agent)</td><td class=rg>task-specific</td><td>0.2958</td><td>0.2426</td><td>0.0707</td><td class=best>0.3449</td><td>0.3150</td><td>0.5720</td><td>0.0707</td><td>0.1254</td><td class=best>0.1097</td><td>0.8112</td></tr><tr><td>10</td><td class=m>KumoRFM (in-context)</td><td class=rg>zero-shot</td><td>0.3036</td><td>0.2810</td><td>0.0935</td><td>0.3658</td><td>0.3450</td><td>0.3910</td><td>0.0808</td><td>0.1273</td><td>0.1717</td><td>0.8763</td></tr><tr><td>11</td><td class=m>Rel-LLM (Llama-3.2-1B + GNN soft prompts, fine-tuned)</td><td class=rg>task-specific</td><td>0.3106</td><td>0.2450</td><td>0.0816</td><td>0.3867</td><td>0.3280</td><td>0.5646</td><td>0.1050</td><td class=best>0.1215</td><td>0.1288</td><td>0.8343</td></tr><tr><td>12</td><td class=m>PluRel (synthetic only)</td><td class=rg>zero-shot</td><td>0.3110</td><td>0.3388</td><td>0.1154</td><td>0.4252</td><td>0.0878</td><td>0.5426</td><td>0.1749</td><td>0.1800</td><td>0.1889</td><td>0.7457</td></tr><tr><td>13</td><td class=m>RT (from scratch)</td><td class=rg>task-specific</td><td>0.3159</td><td>0.2590</td><td>0.0845</td><td>0.4064</td><td>0.5040</td><td>0.4775</td><td>0.1001</td><td>0.1471</td><td>0.1306</td><td>0.7341</td></tr><tr><td>14</td><td class=m>Data Scientist + LightGBM</td><td class=rg>task-specific</td><td>0.3202</td><td>0.2422</td><td class=best>0.0696</td><td>0.4599</td><td>0.3712</td><td>0.5641</td><td>0.0727</td><td>0.1273</td><td>0.1197</td><td>0.8553</td></tr><tr><td>15</td><td class=m>RDL (GraphSAGE)</td><td class=rg>task-specific</td><td>0.3204</td><td>0.2489</td><td>0.0847</td><td>0.4285</td><td>0.3372</td><td>0.5725</td><td>0.1131</td><td>0.1273</td><td>0.1311</td><td>0.8406</td></tr><tr><td>16</td><td class=m>GIN</td><td class=rg>task-specific</td><td>0.3214</td><td>0.2490</td><td>0.0848</td><td>0.4285</td><td>0.3450</td><td>0.5796</td><td>0.1110</td><td>0.1273</td><td>0.1309</td><td>0.8364</td></tr><tr><td>17</td><td class=m>Data Scientist + AutoGluon</td><td class=rg>task-specific</td><td>0.3325</td><td>0.2504</td><td>0.0768</td><td>0.4703</td><td>0.3346</td><td>0.6051</td><td>0.0868</td><td>0.1332</td><td>0.1318</td><td>0.9036</td></tr><tr><td>18</td><td class=m>GAT</td><td class=rg>task-specific</td><td>0.3382</td><td>0.2891</td><td>0.0997</td><td>0.4494</td><td>0.3437</td><td>0.6075</td><td>0.1595</td><td>0.1332</td><td>0.1357</td><td>0.8259</td></tr><tr><td>19</td><td class=m>LightGBM (raw entity features)</td><td class=rg>task-specific</td><td>0.3412</td><td>0.2919</td><td>0.1025</td><td>0.4285</td><td>0.3450</td><td>0.5935</td><td>0.1534</td><td>0.1332</td><td>0.1298</td><td>0.8931</td></tr><tr><td>20</td><td class=m>RT (zero-shot, leave-one-DB-out)</td><td class=rg>zero-shot</td><td>0.3461</td><td>0.3277</td><td>0.1029</td><td>0.6235</td><td>0.0662</td><td>0.4310</td><td>0.1719</td><td>0.2128</td><td>0.2233</td><td>0.9552</td></tr><tr><td>21</td><td class=m>HGT</td><td class=rg>task-specific</td><td>0.3464</td><td>0.2680</td><td>0.0945</td><td>0.4829</td><td>0.3444</td><td>0.6015</td><td>0.1294</td><td>0.1330</td><td>0.1332</td><td>0.9305</td></tr><tr><td>22</td><td class=m>HGT+PE (Laplacian positional encodings)</td><td class=rg>task-specific</td><td>0.3504</td><td>0.2759</td><td>0.0945</td><td>0.5048</td><td>0.3412</td><td>0.6251</td><td>0.1290</td><td>0.1332</td><td>0.1258</td><td>0.9238</td></tr><tr><td>23</td><td class=m>Griffin (fine-tuned)</td><td class=rg>task-specific</td><td>0.3686</td><td>0.3409</td><td>0.1130</td><td>0.4526</td><td>0.4846</td><td>0.5596</td><td>0.1205</td><td>0.2733</td><td>0.1743</td><td>0.7988</td></tr><tr><td>24</td><td class=m>Entity Median</td><td class=rg>task-specific</td><td>0.4278</td><td>0.3030</td><td>0.1124</td><td>0.4808</td><td>0.3516</td><td>1.2125</td><td>0.1575</td><td>0.1352</td><td>0.1708</td><td>0.9267</td></tr><tr><td>25</td><td class=m>Entity Mean</td><td class=rg>task-specific</td><td>0.4551</td><td>0.3314</td><td>0.1327</td><td>0.4808</td><td>0.3973</td><td>1.2100</td><td>0.2241</td><td>0.2077</td><td>0.1708</td><td>0.9414</td></tr></table></div><h2>Recommendation</h2><p class=metric>Metric: MAP on the official test split (higher is better).</p><div class=tablewrap><table><tr><th>#</th><th class=m>Method</th><th class=rg>Regime</th><th>Mean</th><th>rel-amazon<br>user-item-purchase</th><th>rel-amazon<br>user-item-rate</th><th>rel-amazon<br>user-item-review</th><th>rel-avito<br>user-ad-visit</th><th>rel-f1<br>driver-circuit-compete</th><th>rel-hm<br>user-item-purchase</th><th>rel-stack<br>user-post-comment</th><th>rel-stack<br>post-post-related</th><th>rel-trial<br>condition-sponsor-run</th><th>rel-trial<br>site-sponsor-run</th></tr><tr><td>1</td><td class=m>ID-GNN (4 layers)</td><td class=rg>task-specific</td><td class=best>14.0</td><td>0.1</td><td>0.1</td><td>0.1</td><td class=best>3.9</td><td class=best>76.2</td><td class=best>2.9</td><td class=best>13.8</td><td class=best>12.5</td><td>11.3</td><td class=best>19.0</td></tr><tr><td>2</td><td class=m>ID-GNN (2 layers)</td><td class=rg>task-specific</td><td>12.3</td><td>0.1</td><td>0.1</td><td>0.1</td><td>3.6</td><td>62.3</td><td>2.8</td><td>12.7</td><td>10.7</td><td class=best>11.4</td><td>19.0</td></tr><tr><td>3</td><td class=m>LightGBM (entity features + heuristic ranks)</td><td class=rg>task-specific</td><td>7.3</td><td>0.1</td><td>0.2</td><td>0.1</td><td>0.1</td><td>57.8</td><td>0.4</td><td>0.0</td><td>1.9</td><td>4.5</td><td>8.2</td></tr><tr><td>4</td><td class=m>Global Popularity</td><td class=rg>task-specific</td><td>5.9</td><td>0.2</td><td>0.1</td><td>0.1</td><td>0.0</td><td>50.1</td><td>0.3</td><td>0.0</td><td>1.5</td><td>2.5</td><td>3.8</td></tr><tr><td>5</td><td class=m>Past Visit</td><td class=rg>task-specific</td><td>5.3</td><td>0.1</td><td>0.1</td><td>0.0</td><td>1.9</td><td>20.8</td><td>0.9</td><td>1.4</td><td>1.7</td><td>8.4</td><td>17.3</td></tr><tr><td>6</td><td class=m>GraphSAGE (two-tower) (4 layers)</td><td class=rg>task-specific</td><td>3.4</td><td class=best>0.9</td><td class=best>1.0</td><td class=best>0.6</td><td>0.1</td><td>16.6</td><td>0.7</td><td>0.2</td><td>0.1</td><td>2.7</td><td>11.1</td></tr><tr><td>7</td><td class=m>GraphSAGE (two-tower) (2 layers)</td><td class=rg>task-specific</td><td>2.6</td><td>0.7</td><td>0.8</td><td>0.5</td><td>0.0</td><td>9.7</td><td>0.8</td><td>0.2</td><td>0.0</td><td>3.1</td><td>10.4</td></tr></table></div></body></html>
|