Expand baseline roster: all synthcity-available TabDiff/fair-tab-diffusion baselines + external-baseline framework

pipeline/arena.py

@@ -17,9 +17,13 @@ _CANDIDATES = {
     "datadesigner": [("datadesigner", "conditional-target"), ("datadesigner", "samplers")],
     "sdgym": [("sdgym", "data_identity"), ("sdgym", "column"), ("sdgym", "uniform")],
     "sdv": [("sdv", "gaussiancopula"), ("sdv", "ctgan"), ("sdv", "tvae")],
-    "synthcity": [("synthcity", "pategan"), ("synthcity", "tvae"), ("synthcity", "ddpm"),
-                  ("synthcity", "ctgan"), ("synthcity", "arf"), ("synthcity", "dpgan"),
-                  ("synthcity", "adsgan"), ("synthcity", "privbayes"), ("synthcity", "decaf")],
+    # TabDiff + fair-tab-diffusion baselines available as synthcity plugins.
+    "synthcity": [("synthcity", m) for m in (
+        # fidelity baselines
+        "ctgan", "tvae", "rtvae", "nflow", "ddpm", "arf", "goggle", "great",
+        # privacy / fairness baselines
+        "dpgan", "pategan", "adsgan", "privbayes", "decaf",
+    )],
 }
 
 # Acceptance thresholds — the "is it working well?" evaluation loop.

scripts/EXTERNAL_BASELINES.md

@@ -0,0 +1,55 @@
+# Baselines from TabDiff & fair-tab-diffusion
+
+Both papers benchmark a large set of generators. In MemisisLabs they fall into two tiers.
+
+## Tier 1 — available as synthcity plugins (run via the arena, no extra repos)
+These enter the arena directly through the `synthcity` backend once a `SYNTHCITY_VENV` is set
+(see `scripts/server_arena.py`). Install with `pip install "synthcity[all]"` on the GPU host to
+get the `goggle`/`great` extras.
+
+| Baseline (paper) | synthcity plugin |
+|---|---|
+| CTGAN | `ctgan` |
+| TVAE / RTVAE | `tvae`, `rtvae` |
+| TabDDPM | `ddpm` |
+| GOGGLE | `goggle` |
+| GReaT | `great` |
+| Normalizing flow | `nflow` |
+| ARF | `arf` |
+| DP-GAN / PATE-GAN / ADS-GAN | `dpgan`, `pategan`, `adsgan` |
+| PrivBayes | `privbayes` |
+| DECAF (fairness) | `decaf` |
+
+Run: `python scripts/server_arena.py` (uses these by default across all datasets).
+
+## Tier 2 — standalone research repos (external experiments)
+No pip/synthcity plugin — each is its own repo with its own env. Integrate them the same way as
+TabDiff: clone + set up, train + sample into a CSV, then score with **our** metric suite via a
+small evaluate step (mirror `scripts/tabdiff_evaluate.py`). This puts them on the shared
+leaderboard under our TabDiff-standard metrics.
+
+| Baseline | Repo | Notes |
+|---|---|---|
+| TabDiff | github.com/MinkaiXu/TabDiff | done — `scripts/tabdiff_prepare.py` + `tabdiff_evaluate.py` |
+| TabSyn | github.com/amazon-science/tabsyn | latent-space diffusion |
+| STaSy | github.com/JayoungKim408/STaSy | score-based |
+| CoDi | github.com/ChaejeongLee/CoDi | co-evolving diffusion |
+| FairTabDDPM | github.com/comp-well-org/fair-tab-diffusion | fairness-aware diffusion |
+| FairTabGAN / FairSMOTE | (fair-tab-diffusion `args/*/`) | fairness-aware baselines |
+
+### Generic pattern for a Tier-2 baseline
+1. Clone + create its env on the lab server.
+2. Prepare the dataset in its expected format (adapt `scripts/tabdiff_prepare.py` if it uses the
+   same Info-JSON layout — TabSyn/CoDi/STaSy/FairTabDDPM all descend from the TabSyn data format).
+3. Train + sample to a synthetic CSV.
+4. Score + record:
+   ```bash
+   python scripts/tabdiff_evaluate.py --dataset openml_45040 \
+       --synthetic /path/to/<method>_samples/ --record
+   ```
+   (change the recorded label in the script, or add a `--label <method>` flag.)
+
+## Fairness metrics
+fair-tab-diffusion's fairness = fairlearn `demographic_parity_ratio` + `equalized_odds_ratio` —
+already adopted as the arena's `fairness` dimension (`pipeline/fairness.py`), applied to every
+classification dataset with a sensitive attribute.

scripts/server_arena.py

@@ -24,9 +24,11 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
 from pipeline import arena, datasets, leaderboard  # noqa: E402
 from pipeline.metadata import build_metadata  # noqa: E402
 
-DEFAULT_MODELS = ["ctgan", "tvae", "arf", "nflow", "ddpm",
+# synthcity plugins matching the TabDiff + fair-tab-diffusion baselines.
+DEFAULT_MODELS = ["ctgan", "tvae", "rtvae", "nflow", "ddpm", "arf", "goggle", "great",
                   "dpgan", "pategan", "adsgan", "privbayes", "decaf"]
-DEFAULT_DATASETS = ["openml_45040", "adult", "german", "bank", "compas"]
+DEFAULT_DATASETS = ["openml_45040", "adult", "german", "bank", "compas",
+                    "shoppers", "magic", "default", "diabetes_pima", "beijing", "news"]
 
 
 def available_plugins(requested: list[str]) -> list[str]: