""" DFL-Bench: Decision-Focused Learning Benchmark Leaderboard ========================================================== A HuggingFace Spaces Gradio app for evaluating predict-then-optimize pipelines. Storage: ground truth (gt.csv) and graded metrics (results.json) live in a private HF dataset repo (default: GT-KOALA/DFL-Bench-Data). Submitted prediction CSVs are evaluated in-memory and never persisted. Deploy to HuggingFace Spaces: 1. Create a new Space with Gradio SDK 2. Upload app.py, utils.py, about.py, requirements.txt 3. Set Space secrets: - HF_TOKEN — token with read+write access to the dataset repo - HF_DATASET_REPO — optional override of the default repo id 4. Pre-populate the dataset repo with gt.csv (ground truth) """ import gradio as gr from about import ( DATASET_DESCRIPTION_TEXT, INTRODUCTION_TEXT, TITLE, ) from utils import ( TASKS, format_time_ago, handle_submission, latest_submission_time, load_results_as_dataframe, load_task_figure, ) TASK_DISPLAY_CHOICES = [f"{t['icon']} {t['label']}" for t in TASKS] TASK_DISPLAY_TO_KEY = {f"{t['icon']} {t['label']}": t["key"] for t in TASKS} def card_header_html(task: dict) -> str: ago = format_time_ago(latest_submission_time(task["key"])) return f"""
{task['icon']} {task['label']}
🕐 {ago}
""" def refresh_all(): out = [] for t in TASKS: out.append(card_header_html(t)) out.append(load_results_as_dataframe(t["key"])) out.append(load_task_figure(t["key"])) return out def submit(team_name, model_name, task_display, file_obj): file_path = file_obj if isinstance(file_obj, str) else (file_obj.name if file_obj else None) task_key = TASK_DISPLAY_TO_KEY.get(task_display, "") return handle_submission(team_name, model_name, task_key, file_path) custom_css = """ /* Page chrome ------------------------------------------------------------- */ .gradio-container { max-width: 880px !important; font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "Inter", "Helvetica Neue", sans-serif !important; color: #1f2937; } body { background: #fafafa; } .page-heading h1 { font-family: "Charter", "Iowan Old Style", "Palatino Linotype", "Georgia", serif !important; font-weight: 600; font-size: 2.0em; margin: 0 0 0.4em 0; letter-spacing: -0.01em; } .page-heading .page-subtitle { color: #4b5563; font-size: 1.0em; line-height: 1.55; margin: 0 0 1.4em 0; max-width: 680px; } .page-heading .page-subtitle a, .page-heading .page-subtitle em { color: #2563eb; font-style: normal; text-decoration: none; } /* Leaderboard card -------------------------------------------------------- */ .lb-card { background: #ffffff; border: 1px solid #e5e7eb; border-radius: 12px; padding: 6px 18px 14px 18px !important; margin-bottom: 16px; } .lb-card-header { display: flex; justify-content: space-between; align-items: center; padding: 14px 4px 12px 4px; border-bottom: 1px solid #f3f4f6; } .lb-task { font-weight: 600; font-size: 1.05em; color: #111827; } .lb-icon { margin-right: 8px; } .lb-timestamp { color: #6b7280; font-size: 0.92em; display: flex; align-items: center; gap: 6px; } /* Tighten the embedded Gradio dataframe styling --------------------------- */ .leaderboard-table { border: none !important; } .leaderboard-table table { border: none !important; font-size: 0.95em; } .leaderboard-table table th { font-weight: 600 !important; color: #374151 !important; background: transparent !important; border-bottom: 1px solid #e5e7eb !important; } .leaderboard-table table td { border-bottom: 1px solid #f3f4f6 !important; } /* Monospace for the Model column (3rd column: Rank | Team | Model | â€Ļ) */ .leaderboard-table table td:nth-child(3) { font-family: ui-monospace, "JetBrains Mono", "SF Mono", "Fira Code", "Consolas", monospace; font-size: 0.92em; } .leaderboard-table table td:first-child { font-weight: 600; color: #111827; } """ with gr.Blocks(title="DFL-Bench", theme=gr.themes.Soft(), css=custom_css) as demo: gr.HTML(TITLE) with gr.Tabs(): # ── Leaderboard Tab ────────────────────────────────────────────── with gr.TabItem("🏆 Leaderboard"): refresh_outputs = [] for _task in TASKS: with gr.Column(elem_classes=["lb-card"]): _h = gr.HTML(card_header_html(_task)) _tbl = gr.Dataframe( value=load_results_as_dataframe(_task["key"]), interactive=False, elem_classes=["leaderboard-table"], show_label=False, ) _plot = gr.Plot( value=load_task_figure(_task["key"]), show_label=False, ) refresh_outputs += [_h, _tbl, _plot] refresh_btn = gr.Button("🔄 Refresh", scale=0) refresh_btn.click(fn=refresh_all, inputs=[], outputs=refresh_outputs) # ── Submit Tab ─────────────────────────────────────────────────── with gr.TabItem("📤 Submit"): gr.Markdown( "> **â„šī¸ Note.** If the evaluation result is shown below " "(\"✅ Submission evaluated successfully!\" with the metrics " "table), your submission **succeeded** and has been recorded. " "Please do not resubmit the same result.\n" ">\n" "> Click the **🔄 Refresh** button at the bottom left of the " "Leaderboard tab to pull the latest results. The leaderboard " "may take a little while to update (typically 0 to 5 mins)." ) with gr.Row(): with gr.Column(): team_input = gr.Textbox( label="Team Name", placeholder="e.g. Team 1", ) model_input = gr.Textbox( label="Model Name", placeholder="e.g. Model-A", ) with gr.Column(): task_input = gr.Dropdown( choices=TASK_DISPLAY_CHOICES, value=TASK_DISPLAY_CHOICES[0], label="Task", interactive=True, ) file_input = gr.File( label="Submission JSON", file_types=[".json"], type="filepath", ) submit_btn = gr.Button("🚀 Submit & Evaluate", variant="primary") result_output = gr.Markdown(label="Result") submit_btn.click( fn=submit, inputs=[team_input, model_input, task_input, file_input], outputs=[result_output], ) # ── Dataset Tab ────────────────────────────────────────────────── with gr.TabItem("📊 Dataset"): gr.Markdown(DATASET_DESCRIPTION_TEXT) # ── About Tab ──────────────────────────────────────────────────── with gr.TabItem("📖 About"): # Enable inline ($...$) math in addition to the default block ($$...$$). # Without this, KaTeX skips single-dollar spans and they render as literal text. gr.Markdown( INTRODUCTION_TEXT, latex_delimiters=[ {"left": "$$", "right": "$$", "display": True}, {"left": "$", "right": "$", "display": False}, ], ) if __name__ == "__main__": demo.launch()