Spaces:
Sleeping
Sleeping
File size: 3,926 Bytes
8f559f1 c95367f 8f559f1 c95367f 8f559f1 c95367f 8f559f1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 | """Evaluate TabDiff's synthetic output with the full MemisisLabs metric suite.
TabDiff reports its own metrics (Density, MLE, C2ST, alpha-precision/beta-recall, DCR). This runs
OUR arena evaluators on the same synthetic table so TabDiff gets directly-comparable numbers —
fidelity (SDMetrics), privacy (NewRowSynthesis), ML utility (TSTR), fairness (DPR/EOR) — and lands
on the shared leaderboard alongside every other generator.
Usage:
python scripts/tabdiff_evaluate.py --dataset openml_45040 \
--synthetic /path/to/TabDiff/eval/report_runs/learnable_schedule/schizophrenia/sample_0.csv \
[--record]
"""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
import pandas as pd
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from pipeline import datasets, leaderboard # noqa: E402
from pipeline.evaluate import evaluate_all # noqa: E402
from pipeline.metadata import build_metadata # noqa: E402
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--dataset", default="openml_45040")
ap.add_argument("--synthetic", required=True, help="sampled CSV (or a dir of samples)")
ap.add_argument("--label", default="tabdiff · diffusion",
help="leaderboard model name (e.g. 'tabsyn · latent-diffusion')")
ap.add_argument("--record", action="store_true", help="append to the shared leaderboard")
args = ap.parse_args()
real, target, protected, task = datasets.load(args.dataset)
metadata = build_metadata(real)
# Accept a single CSV or a directory of TabDiff samples (evaluate each, average).
path = Path(args.synthetic)
files = sorted(path.glob("*.csv")) if path.is_dir() else [path]
if not files:
sys.exit(f"no CSVs found at {path}")
results = []
for f in files:
syn = pd.read_csv(f)
syn = syn[[c for c in real.columns if c in syn.columns]] # align columns
r = evaluate_all(real, syn, metadata, target=target, protected=protected, task=task)
r["synthetic"] = syn
results.append(r)
fair = r.get("fairness") or {}
print(f"{f.name}: fidelity={r['overall_score']:.3f} privacy={r['new_row_synthesis']} "
f"utility={r['ml_efficacy']} fairness={fair.get('fairness_score')}")
# Average across samples for the leaderboard row.
def avg(key):
vals = [x.get(key) for x in results if isinstance(x.get(key), (int, float))]
return sum(vals) / len(vals) if vals else None
fair_scores = [(x.get("fairness") or {}).get("fairness_score") for x in results]
fair_scores = [v for v in fair_scores if isinstance(v, (int, float))]
mean_metrics = {
"overall_score": avg("overall_score"),
"new_row_synthesis": avg("new_row_synthesis"),
"ml_efficacy": avg("ml_efficacy"),
"fairness": {"fairness_score": (sum(fair_scores) / len(fair_scores)) if fair_scores else None},
}
print("\n=== TabDiff · diffusion — averaged over "
f"{len(files)} sample(s) on {datasets.display_name(args.dataset)} ===")
print(f" fidelity : {mean_metrics['overall_score']}")
print(f" privacy : {mean_metrics['new_row_synthesis']}")
print(f" utility : {mean_metrics['ml_efficacy']}")
print(f" fairness : {mean_metrics['fairness']['fairness_score']}")
if args.record:
backend = args.label.split(" · ")[0].strip() or "external"
entry = {"backend": backend, "method": args.label.split(" · ")[-1].strip(),
"label": args.label, "metrics": mean_metrics}
leaderboard.record([entry], dataset=args.dataset,
run_id=args.label.replace(" ", "_").replace("·", ""),
num_records=len(results[0]["synthetic"]))
print(f"\nrecorded -> leaderboard ({args.label})")
if __name__ == "__main__":
main()
|