from utils import * import utils_v3 as v3 import gradio as gr global data_component def update_table(query, min_size, max_size, selected_tasks=None): df = get_df() filtered_df = search_and_filter_models(df, query, min_size, max_size) if selected_tasks and len(selected_tasks) > 0: selected_columns = BASE_COLS + selected_tasks filtered_df = filtered_df[selected_columns] return filtered_df def update_table_v2(query, min_size, max_size): df = v3.get_df() filtered_df = v3.search_and_filter_models(df, query, min_size, max_size) return filtered_df with gr.Blocks() as block: gr.Markdown(LEADERBOARD_INTRODUCTION) with gr.Accordion("🔥 What's NEW:"): gr.Markdown(ANNOUNCEMENT) with gr.Tabs(elem_classes="tab-buttons") as tabs: # Table 1, the main leaderboard of overall scores with gr.TabItem("📊 MMEB (V3)[𝓝𝓔𝒲]", elem_id="tab-overall", id=1): with gr.Row(): with gr.Accordion("Citation", open=False): citation_button2 = gr.Textbox( value=v3.CITATION_BUTTON_TEXT_V3, label="Copy the following snippet to cite MMEB-V3", elem_id="citation-button3", lines=10, show_copy_button=True ) gr.Textbox( value=v3.CITATION_BUTTON_TEXT_V2, label="Copy the following snippet to cite VLM2Vec/MMEB-V2", elem_id="citation-button2", lines=10, show_copy_button=True ) gr.Textbox( value=CITATION_BUTTON_TEXT, label="Copy the following snippet to cite VLM2Vec/MMEB-V1", elem_id="citation-button1", lines=6, show_copy_button=True ) gr.Markdown(v3.TABLE_INTRODUCTION) with gr.Row(): search_bar2 = gr.Textbox( placeholder="Search models...", show_label=False, elem_id="search-bar" ) df = get_df() df['Date'] = 'unknown' df2 = v3.get_df('Overall') min_size2, max_size2 = get_size_range(df2) with gr.Row(): min_size_slider2 = gr.Slider( minimum=min_size2, maximum=max_size2, value=min_size2, step=0.1, label="Minimum number of parameters (B)", ) max_size_slider2 = gr.Slider( minimum=min_size2, maximum=max_size2, value=max_size2, step=0.1, label="Maximum number of parameters (B)", ) df2_all = df2[v3.COLUMN_NAMES_V3] data_component2 = gr.components.Dataframe( value=df2_all, headers=v3.COLUMN_NAMES_V3, type="pandas", datatype=v3.DATA_TITLE_TYPE_V3, interactive=False, visible=True, max_height=2400, ) refresh_button2 = gr.Button("Refresh") # save a summary of rankings v3.save_ranking_summary(df2_all, 'mmeb_ranking') download_overall_but = gr.DownloadButton("Download MMEB Ranking (CSV)", value=v3.download_ranking(df2_all, 'mmeb_ranking')) download_overall_but_json = gr.DownloadButton("Download MMEB Ranking (JSON)", value=v3.download_ranking(df2_all, 'mmeb_ranking', format='json')) def update_with_tasks_v2(*args): return update_table_v2(*args) search_bar2.change( fn=update_with_tasks_v2, inputs=[search_bar2, min_size_slider2, max_size_slider2], outputs=data_component2 ) min_size_slider2.change( fn=update_with_tasks_v2, inputs=[search_bar2, min_size_slider2, max_size_slider2], outputs=data_component2 ) max_size_slider2.change( fn=update_with_tasks_v2, inputs=[search_bar2, min_size_slider2, max_size_slider2], outputs=data_component2 ) refresh_button2.click(fn=v3.refresh_data, outputs=data_component2) # v2 overall table with gr.TabItem("📊 MMEB (V2)", elem_id="tab-overall-v2", id=10): gr.Markdown("Models are ranked based on **Overall-V2**.") overall_df_v2 = gr.components.Dataframe( value=v3.rank_models(df2[v3.COLUMN_NAMES_V2], 'Overall-V2'), headers=v3.COLUMN_NAMES_V2, type="pandas", datatype=v3.DATA_TITLE_TYPE_V2, interactive=False, visible=True, max_height=2400, ) # table 2, text scores with gr.TabItem("📝 Text [𝓝𝓔𝒲]", elem_id="tab-text", id=2): data_component_t = gr.components.Dataframe( value=v3.rank_models(df2[v3.COLUMN_NAMES_T], 'Text-Overall'), headers=v3.COLUMN_NAMES_T, type="pandas", datatype=v3.DATA_TITLE_TYPE_T, interactive=False, visible=True, max_height=2400, ) def get_special_processed_df2(): """Temporary special processing to merge v1 scores with v2 image scores. Will be removed later after v2 is fully adopted.""" df2_i = df2[v3.COLUMN_NAMES_I] df1 = df.rename(columns={'V1-Overall': 'Image-Overall'}) df1 = df1[v3.BASE_COLS + v3.SUB_TASKS_I + ['Image-Overall']] combined_df = pd.concat([df1, df2_i], ignore_index=True) for task in v3.TASKS_I: combined_df[task] = combined_df[task].apply(lambda score: '-' if pd.isna(score) else score) combined_df = v3.rank_models(combined_df, 'Image-Overall') return combined_df[v3.COLUMN_NAMES_I] # table 3, image scores only with gr.TabItem("🖼️ Image", elem_id="tab-image", id=3): gr.Markdown(v3.TABLE_INTRODUCTION_I) df2_i = get_special_processed_df2() data_component3 = gr.components.Dataframe( value=df2_i, headers=v3.COLUMN_NAMES_I, type="pandas", datatype=v3.DATA_TITLE_TYPE_I, interactive=False, visible=True, max_height=2400, ) v3.save_ranking_summary(df2_i, 'image_ranking') download_i_but = gr.DownloadButton("Download Image Ranking (CSV)", value=v3.download_ranking(df2_i, 'image_ranking')) download_i_but_json = gr.DownloadButton("Download Image Ranking (JSON)", value=v3.download_ranking(df2_i, 'image_ranking', format='json')) # table 4, video scores only with gr.TabItem("💽 Video", elem_id="tab-video", id=4): gr.Markdown(v3.TABLE_INTRODUCTION_V) df2_v = v3.rank_models(df2[v3.COLUMN_NAMES_V], 'Video-Overall') data_component4 = gr.components.Dataframe( value=df2_v, headers=v3.COLUMN_NAMES_V, type="pandas", datatype=v3.DATA_TITLE_TYPE_V, interactive=False, visible=True, max_height=2400, ) v3.save_ranking_summary(df2_v, 'video_ranking') download_v_but = gr.DownloadButton("Download Video Ranking (CSV)", value=v3.download_ranking(df2_v, 'video_ranking')) download_v_but_json = gr.DownloadButton("Download Video Ranking (JSON)", value=v3.download_ranking(df2_v, 'video_ranking', format='json')) # table 5, audio scores with gr.TabItem("🎵 Audio [𝓝𝓔𝒲]", elem_id="tab-audio", id=5): gr.Markdown(v3.TABLE_INTRODUCTION_A) data_component_a = gr.components.Dataframe( value=v3.rank_models(df2[v3.COLUMN_NAMES_A], 'Audio-Overall'), headers=v3.COLUMN_NAMES_A, type="pandas", datatype=v3.DATA_TITLE_TYPE_A, interactive=False, visible=True, max_height=2400, ) # table 6, visual document scores only with gr.TabItem("📑 Visual Doc", elem_id="tab-visdoc", id=6): gr.Markdown(v3.TABLE_INTRODUCTION_D) df2_d = v3.rank_models(df2[v3.COLUMN_NAMES_D], 'Visdoc-Overall') old_df_vd = pd.read_json('archive/cached_vd_scores_before_fix.jsonl', orient='records', lines=True) df2_d = df2_d.merge(old_df_vd, on='Models', how='left') df2_d = df2_d.fillna(0) temp_header = v3.COLUMN_NAMES_D[:5] + ['Visdoc-Overall-before-fix', 'VisDoc-OOD-before-fix'] + v3.COLUMN_NAMES_D[5:] + [ 'ViDoSeek-page-before-fix', 'MMLongBench-page-before-fix'] df2_d = df2_d[temp_header] data_component5 = gr.components.Dataframe( value=df2_d, headers=temp_header, type="pandas", datatype=v3.DATA_TITLE_TYPE_D+['number']*4, interactive=False, visible=True, max_height=2400, ) v3.save_ranking_summary(df2_d, 'visdoc_ranking') download_vd_but = gr.DownloadButton("Download Visual Document Ranking (CSV)", value=v3.download_ranking(df2_d, 'visdoc_ranking')) download_vd_but_json = gr.DownloadButton("Download Visual Document Ranking (JSON)", value=v3.download_ranking(df2_d, 'visdoc_ranking', format='json')) # table 7, agent scores with gr.TabItem("🤖 Agents [𝓝𝓔𝒲]", elem_id="tab-agents", id=7): gr.Markdown(v3.TABLE_INTRODUCTION_AG) data_component_ag = gr.components.Dataframe( value=v3.rank_models(df2[v3.COLUMN_NAMES_AG], 'Agent-Overall'), headers=v3.COLUMN_NAMES_AG, type="pandas", datatype=v3.DATA_TITLE_TYPE_AG, interactive=False, visible=True, max_height=2400, ) # table 8 with gr.TabItem("📰 About", elem_id="tab-about", id=8): gr.Image("overview.png", width=900, label="Dataset Overview") gr.Markdown(LEADERBOARD_INFO, elem_classes="markdown-text") # table 9 with gr.TabItem("🚀 Submit here! ", elem_id="tab-submit", id=9): with gr.Row(): gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text") if __name__ == "__main__": block.launch(share=True, ssr_mode=False)