from __future__ import annotations import base64 from pathlib import Path from typing import List, Tuple import gradio as gr import pandas as pd from src.data_loader import DataLoader from src.leaderboard import Leaderboard from src.plotter import Plotter from src.radar_plotter import RadarPlotter from src.styling import dataframe_to_html, get_academic_css DATA_FILE = "./data/leaderboards.json" APP_TITLE = "Urban Cup 2026 | EmbodiedCity Leaderboard" REGISTRATION_FORM_URL = "https://rcnc7uacrkc1.feishu.cn/share/base/shrcns37gJlku9CVRaS9lXBZDcb" SUBMISSION_UPLOAD_URL = "https://cloud.tsinghua.edu.cn/u/d/60c3058ed4ef4b8d90d4/" EVENT_WEBSITE_URL = "https://fi.ee.tsinghua.edu.cn/RSUSHD2026/" COMPETITION_GROUP_QR_PATH = Path(__file__).resolve().parent / "assets" / "competition_communication_group.png" CONFERENCE_GROUP_QR_PATH = Path(__file__).resolve().parent / "assets" / "conference_official_group.jpg" def build_title_markdown() -> str: return f""" # {APP_TITLE} Leaderboard-based evaluation for urban video question answering and goal-oriented embodied navigation """ def build_registration_html() -> str: return f"""
Registration & Submission
Register your team first, then upload a valid ZIP package. API submissions are preferred for faster organizer-side evaluation; model weights with inference code are also accepted.
Open Registration Form Upload ZIP Package
Tracks
Track 1: UrbanVideo-Bench for urban video QA.
Track 2: EmbodiedNav-Bench for goal-oriented embodied navigation.
ZIP Naming
Required format: 赛事一_赛道几_团队名称_模型名称_提交类型. For updated versions, append _V1, _V2, etc. to the model name in the ZIP filename; repeated registration is not required.
README Required
The ZIP must include a README with API endpoint/auth/calling instructions or model weights/inference code access, standard inference or evaluation entry, runtime requirements, and any rate-limit notes.
Official Evaluation
Final leaderboard results are determined by organizer evaluation on a hidden test set. A registration without a valid uploaded ZIP package will be treated as invalid.
Reference Baselines
Entries labeled Reference Baseline are not eligible for awards. Their reported results use the original benchmark test sets, which differ from the competition's final hidden test set, and are provided for reference only.
Submission Limit
Each team may make up to three submissions per track. If more than three packages are submitted, only the three most recent submissions will be retained; newer versions will be evaluated first.
API Deployment
If an internal GPU server (e.g., A800/H800) is not publicly reachable, use a pay-as-you-go VPS/ECS such as Alibaba Cloud or Volcengine as an HTTPS gateway, connected through WireGuard, FRP, or rathole. Keep inference and data processing on the backend; use a stable domain, TLS, sufficient video bandwidth, authentication, rate limits, and health checks. Keep the service available during evaluation, and never expose raw inference ports or model credentials.
Final submission deadline: July 30, 2026. Late submissions will not be evaluated. Leaderboard updates may be delayed by approximately one week. For version tracing, keep 赛道几_团队名称_模型名称 consistent across registration and ZIP uploads.
""" def build_image_data_uri(image_path: Path) -> str: mime_type = "image/jpeg" if image_path.suffix.lower() in {".jpg", ".jpeg"} else "image/png" encoded = base64.b64encode(image_path.read_bytes()).decode("ascii") return f"data:{mime_type};base64,{encoded}" def build_event_info_html() -> str: competition_qr_data_uri = build_image_data_uri(COMPETITION_GROUP_QR_PATH) conference_qr_data_uri = build_image_data_uri(CONFERENCE_GROUP_QR_PATH) return f"""
Event & Contact
Urban Cup 2026 Official Website
For conference information, schedules, and related updates, please visit the official website. Scan the QR codes for the competition communication group and the conference official group.
Conference Website Urban Cup 2026
Competition communication group QR code
Competition Communication Group
Conference official group QR code
Conference Official Group
""" def get_launch_kwargs() -> dict: return { "server_name": "0.0.0.0", "server_port": 7860, "prevent_thread_lock": False, "ssr_mode": False, } def build_info_html(loader: DataLoader) -> str: benchmark = loader.benchmark_config if benchmark is None: return "
Benchmark information unavailable.
" links_html = " ".join( f"{link['label']}" for link in benchmark.get("links", []) ) dimensions = ", ".join(benchmark.get("dimensions", [])) data_note = benchmark.get("dataNote", "") data_note_html = f"Metric Notes: {data_note}
" if data_note else "" award_note = benchmark.get("awardNote", "") award_html = f"Awards: {award_note}
" if award_note else "" return f"""
{benchmark['name']}
{benchmark['description']}
Scale: {benchmark['scale']}
Dimensions: {dimensions}
Evaluation: {benchmark['coverage']}
{data_note_html} {award_html} Links: {links_html}
""" def build_benchmark_tab(benchmark_id: str) -> Tuple: loader = DataLoader(benchmark_id=benchmark_id, data_file=DATA_FILE) loader.reload_data() leaderboard = Leaderboard(loader) plotter = Plotter(loader) radar_plotter = RadarPlotter(loader) benchmark = loader.benchmark_config metric_choices = loader.get_metric_choices() display_metric_choices = [loader.get_metric_label(metric) for metric in metric_choices] default_internal_metric = benchmark["primaryMetric"] default_display_metric = loader.get_metric_label(default_internal_metric) default_selected_metrics = [ loader.get_metric_label(metric) for metric in benchmark.get("defaultMetrics", []) if metric != default_internal_metric ] def to_internal_metric(metric_name: str) -> str: return loader.to_internal_metric(metric_name) def to_internal_metrics(metric_names: List[str]) -> List[str]: return [to_internal_metric(metric_name) for metric_name in metric_names] def build_radar_df(table_df: pd.DataFrame) -> pd.DataFrame: displayed_models = table_df["Model"].tolist() if not table_df.empty else [] return loader.get_dimension_dataframe(displayed_models) def render_table( metric_name: str, top_k: int, model_filter: str, entry_type_filter: str, sort_mode: str, selected_metrics: List[str], ) -> pd.DataFrame: clean_metric = to_internal_metric(metric_name) clean_selected_metrics = to_internal_metrics(selected_metrics) return leaderboard.update_leaderboard( metric=clean_metric, top_k=top_k, model_filter=model_filter, entry_type_filter=entry_type_filter, sort_mode=sort_mode, selected_metrics=clean_selected_metrics, ) def reload_data(): status_message = loader.reload_data() table_df = render_table( default_display_metric, 20, "", "All", "Auto", default_selected_metrics, ) html_table = dataframe_to_html( table_df, column_label_map=loader.metric_display_map, dimension_metrics=loader.dimension_metrics, primary_metric=default_internal_metric, ) entry_type_update = gr.update(choices=loader.get_entry_type_choices(), value="All", interactive=True) base_returns = ( status_message, entry_type_update, build_info_html(loader), html_table, ) if loader.dimension_metrics: radar_df = build_radar_df(table_df) radar_fig = radar_plotter.create_radar_chart(radar_df) return base_returns + (radar_fig,) return base_returns def update_leaderboard_wrapper(metric, top_k, model_filter, entry_type_filter, sort_mode, selected_metrics): table_df = render_table( metric, top_k, model_filter, entry_type_filter, sort_mode, selected_metrics, ) html_table = dataframe_to_html( table_df, column_label_map=loader.metric_display_map, dimension_metrics=loader.dimension_metrics, primary_metric=to_internal_metric(metric), ) if loader.dimension_metrics: radar_df = build_radar_df(table_df) radar_fig = radar_plotter.create_radar_chart(radar_df) return html_table, radar_fig return html_table def create_comparison_plot_wrapper(model_filter, entry_type_filter, selected_plot_metric, plot_sort_mode): internal_metric = to_internal_metric(selected_plot_metric) return plotter.create_comparison_plot( model_filter=model_filter, entry_type_filter=entry_type_filter, selected_plot_metric=internal_metric, plot_sort_mode=plot_sort_mode, display_metric_name=selected_plot_metric, ) status_box = gr.Markdown(f"Loading {benchmark['name']}...") benchmark_info = gr.HTML(build_info_html(loader)) with gr.Row(): with gr.Column(scale=2): metric_dropdown = gr.Dropdown( label="Primary Ranking Metric", choices=display_metric_choices, value=default_display_metric, interactive=True, ) with gr.Column(scale=1): sort_mode_radio = gr.Radio( label="Sort Order", choices=["Auto", "Ascending (low → high)", "Descending (high → low)"], value="Auto", interactive=True, ) topk_slider = gr.Slider( label="Display Top-K Models", minimum=3, maximum=max(20, len(loader.df_all) if loader.df_all is not None else 20), value=min(20, len(loader.df_all) if loader.df_all is not None else 20), step=1, interactive=True, ) with gr.Row(): metrics_select = gr.CheckboxGroup( label="Additional Metrics to Display (📊 indicates summary dimensions)", choices=[ f"📊 {loader.get_metric_label(metric)}" if metric in loader.dimension_metrics else loader.get_metric_label(metric) for metric in metric_choices if metric != default_internal_metric ], value=[ f"📊 {label}" if loader.to_internal_metric(label) in loader.dimension_metrics else label for label in default_selected_metrics ], interactive=True, ) def normalize_selected_metrics(metric_names: List[str]) -> List[str]: return [name.replace("📊 ", "") for name in metric_names] with gr.Row(): with gr.Column(scale=1): model_filter_box = gr.Textbox( label="Filter by Model Name", placeholder="Enter model name (partial match)", interactive=True, ) with gr.Column(scale=1): entry_type_dropdown = gr.Dropdown( label="Filter by Entry Type", choices=loader.get_entry_type_choices(), value="All", interactive=True, ) with gr.Row(): reload_button = gr.Button("🔄 Reload Data", variant="secondary", size="sm") update_button = gr.Button("✅ Update Leaderboard", variant="primary", size="sm") leaderboard_html = gr.HTML( label="Leaderboard Table", value="
Leaderboard will be displayed here...
", ) radar_plot = None if loader.dimension_metrics: with gr.Row(): radar_plot = gr.Plot(label="Dimension Radar Chart", format="png") with gr.Row(): with gr.Column(scale=2): plot_metric_radio = gr.Radio( label="Select Metric for Comparison Plot", choices=display_metric_choices, value=default_display_metric, interactive=True, ) with gr.Column(scale=1): plot_sort_radio = gr.Radio( label="Plot Sort Order", choices=["Ascending (low → high)", "Descending (high → low)"], value="Descending (high → low)", interactive=True, ) plot_update_button = gr.Button("📊 Generate Comparison Plot", variant="primary", size="sm") comparison_plot = gr.Plot(label="Model Comparison Visualization", format="png") reload_button.click( fn=reload_data, inputs=[], outputs=[status_box, entry_type_dropdown, benchmark_info, leaderboard_html] + ([radar_plot] if radar_plot is not None else []), ) update_button.click( fn=lambda metric, top_k, model_filter, entry_type_filter, sort_mode, selected_metrics: update_leaderboard_wrapper( metric, top_k, model_filter, entry_type_filter, sort_mode, normalize_selected_metrics(selected_metrics), ), inputs=[ metric_dropdown, topk_slider, model_filter_box, entry_type_dropdown, sort_mode_radio, metrics_select, ], outputs=[leaderboard_html] + ([radar_plot] if radar_plot is not None else []), ) plot_update_button.click( fn=create_comparison_plot_wrapper, inputs=[ model_filter_box, entry_type_dropdown, plot_metric_radio, plot_sort_radio, ], outputs=[comparison_plot], ) return reload_data, [status_box, entry_type_dropdown, benchmark_info, leaderboard_html] + ( [radar_plot] if radar_plot is not None else [] ) academic_css = get_academic_css() with gr.Blocks(css=academic_css) as demo: gr.Markdown(build_title_markdown(), elem_id="title") gr.HTML( """
Explore reference results and evaluation resources for the two EmbodiedCity competition tracks: UrbanVideo-Bench for urban video QA and EmbodiedNav-Bench for goal-oriented embodied navigation. The competition follows a leaderboard format, while official rankings are produced separately through organizer evaluation on a hidden test set.
""" ) gr.HTML(build_registration_html()) with gr.Tabs(): with gr.Tab("UrbanVideo-Bench"): urban_reload, urban_outputs = build_benchmark_tab("urbanvideo") with gr.Tab("EmbodiedNav-Bench"): nav_reload, nav_outputs = build_benchmark_tab("embodiednav") gr.HTML(build_event_info_html()) demo.load(fn=urban_reload, inputs=[], outputs=urban_outputs) demo.load(fn=nav_reload, inputs=[], outputs=nav_outputs) if __name__ == "__main__": demo.launch(**get_launch_kwargs())