"""
DFL-Bench: Decision-Focused Learning Benchmark Leaderboard
==========================================================
A HuggingFace Spaces Gradio app for evaluating predict-then-optimize pipelines.
Storage: ground truth (gt.csv) and graded metrics (results.json) live in a
private HF dataset repo (default: GT-KOALA/DFL-Bench-Data). Submitted
prediction CSVs are evaluated in-memory and never persisted.
Deploy to HuggingFace Spaces:
1. Create a new Space with Gradio SDK
2. Upload app.py, utils.py, about.py, requirements.txt
3. Set Space secrets:
- HF_TOKEN â token with read+write access to the dataset repo
- HF_DATASET_REPO â optional override of the default repo id
4. Pre-populate the dataset repo with gt.csv (ground truth)
"""
import gradio as gr
from about import (
DATASET_DESCRIPTION_TEXT,
INTRODUCTION_TEXT,
TITLE,
)
from utils import (
TASKS,
format_time_ago,
handle_submission,
latest_submission_time,
load_results_as_dataframe,
load_task_figure,
)
TASK_DISPLAY_CHOICES = [f"{t['icon']} {t['label']}" for t in TASKS]
TASK_DISPLAY_TO_KEY = {f"{t['icon']} {t['label']}": t["key"] for t in TASKS}
def card_header_html(task: dict) -> str:
ago = format_time_ago(latest_submission_time(task["key"]))
return f"""
"""
def refresh_all():
out = []
for t in TASKS:
out.append(card_header_html(t))
out.append(load_results_as_dataframe(t["key"]))
out.append(load_task_figure(t["key"]))
return out
def submit(team_name, model_name, task_display, file_obj):
file_path = file_obj if isinstance(file_obj, str) else (file_obj.name if file_obj else None)
task_key = TASK_DISPLAY_TO_KEY.get(task_display, "")
return handle_submission(team_name, model_name, task_key, file_path)
custom_css = """
/* Page chrome ------------------------------------------------------------- */
.gradio-container {
max-width: 880px !important;
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "Inter", "Helvetica Neue", sans-serif !important;
color: #1f2937;
}
body { background: #fafafa; }
.page-heading h1 {
font-family: "Charter", "Iowan Old Style", "Palatino Linotype", "Georgia", serif !important;
font-weight: 600;
font-size: 2.0em;
margin: 0 0 0.4em 0;
letter-spacing: -0.01em;
}
.page-heading .page-subtitle {
color: #4b5563;
font-size: 1.0em;
line-height: 1.55;
margin: 0 0 1.4em 0;
max-width: 680px;
}
.page-heading .page-subtitle a,
.page-heading .page-subtitle em {
color: #2563eb;
font-style: normal;
text-decoration: none;
}
/* Leaderboard card -------------------------------------------------------- */
.lb-card {
background: #ffffff;
border: 1px solid #e5e7eb;
border-radius: 12px;
padding: 6px 18px 14px 18px !important;
margin-bottom: 16px;
}
.lb-card-header {
display: flex;
justify-content: space-between;
align-items: center;
padding: 14px 4px 12px 4px;
border-bottom: 1px solid #f3f4f6;
}
.lb-task {
font-weight: 600;
font-size: 1.05em;
color: #111827;
}
.lb-icon { margin-right: 8px; }
.lb-timestamp {
color: #6b7280;
font-size: 0.92em;
display: flex;
align-items: center;
gap: 6px;
}
/* Tighten the embedded Gradio dataframe styling --------------------------- */
.leaderboard-table { border: none !important; }
.leaderboard-table table {
border: none !important;
font-size: 0.95em;
}
.leaderboard-table table th {
font-weight: 600 !important;
color: #374151 !important;
background: transparent !important;
border-bottom: 1px solid #e5e7eb !important;
}
.leaderboard-table table td {
border-bottom: 1px solid #f3f4f6 !important;
}
/* Monospace for the Model column (3rd column: Rank | Team | Model | âĻ) */
.leaderboard-table table td:nth-child(3) {
font-family: ui-monospace, "JetBrains Mono", "SF Mono", "Fira Code", "Consolas", monospace;
font-size: 0.92em;
}
.leaderboard-table table td:first-child {
font-weight: 600;
color: #111827;
}
"""
with gr.Blocks(title="DFL-Bench", theme=gr.themes.Soft(), css=custom_css) as demo:
gr.HTML(TITLE)
with gr.Tabs():
# ââ Leaderboard Tab ââââââââââââââââââââââââââââââââââââââââââââââ
with gr.TabItem("đ Leaderboard"):
refresh_outputs = []
for _task in TASKS:
with gr.Column(elem_classes=["lb-card"]):
_h = gr.HTML(card_header_html(_task))
_tbl = gr.Dataframe(
value=load_results_as_dataframe(_task["key"]),
interactive=False,
elem_classes=["leaderboard-table"],
show_label=False,
)
_plot = gr.Plot(
value=load_task_figure(_task["key"]),
show_label=False,
)
refresh_outputs += [_h, _tbl, _plot]
refresh_btn = gr.Button("đ Refresh", scale=0)
refresh_btn.click(fn=refresh_all, inputs=[], outputs=refresh_outputs)
# ââ Submit Tab âââââââââââââââââââââââââââââââââââââââââââââââââââ
with gr.TabItem("đ¤ Submit"):
gr.Markdown(
"> **âšī¸ Note.** If the evaluation result is shown below "
"(\"â
Submission evaluated successfully!\" with the metrics "
"table), your submission **succeeded** and has been recorded. "
"Please do not resubmit the same result.\n"
">\n"
"> Click the **đ Refresh** button at the bottom left of the "
"Leaderboard tab to pull the latest results. The leaderboard "
"may take a little while to update (typically 0 to 5 mins)."
)
with gr.Row():
with gr.Column():
team_input = gr.Textbox(
label="Team Name",
placeholder="e.g. Team 1",
)
model_input = gr.Textbox(
label="Model Name",
placeholder="e.g. Model-A",
)
with gr.Column():
task_input = gr.Dropdown(
choices=TASK_DISPLAY_CHOICES,
value=TASK_DISPLAY_CHOICES[0],
label="Task",
interactive=True,
)
file_input = gr.File(
label="Submission JSON",
file_types=[".json"],
type="filepath",
)
submit_btn = gr.Button("đ Submit & Evaluate", variant="primary")
result_output = gr.Markdown(label="Result")
submit_btn.click(
fn=submit,
inputs=[team_input, model_input, task_input, file_input],
outputs=[result_output],
)
# ââ Dataset Tab ââââââââââââââââââââââââââââââââââââââââââââââââââ
with gr.TabItem("đ Dataset"):
gr.Markdown(DATASET_DESCRIPTION_TEXT)
# ââ About Tab ââââââââââââââââââââââââââââââââââââââââââââââââââââ
with gr.TabItem("đ About"):
# Enable inline ($...$) math in addition to the default block ($$...$$).
# Without this, KaTeX skips single-dollar spans and they render as literal text.
gr.Markdown(
INTRODUCTION_TEXT,
latex_delimiters=[
{"left": "$$", "right": "$$", "display": True},
{"left": "$", "right": "$", "display": False},
],
)
if __name__ == "__main__":
demo.launch()