MCMR added

app.py

@@ -129,8 +129,9 @@ with gr.Blocks() as block:
 
         # table 2, text scores
         with gr.TabItem("📝 Text [𝓝𝓔𝒲]", elem_id="tab-text", id=2):
+            df3_t = v3.rank_models(df2[v3.COLUMN_NAMES_T], 'Text-Overall')
             data_component_t = gr.components.Dataframe(
-                value=v3.rank_models(df2[v3.COLUMN_NAMES_T], 'Text-Overall'),
+                value=df3_t,
                 headers=v3.COLUMN_NAMES_T,
                 type="pandas",
                 datatype=v3.DATA_TITLE_TYPE_T,
@@ -138,6 +139,9 @@ with gr.Blocks() as block:
                 visible=True,
                 max_height=2400, 
             )
+            v3.save_ranking_summary(df3_t, 'text_ranking')
+            gr.DownloadButton("Download Text Ranking (CSV)", value=v3.download_ranking(df3_t, 'text_ranking'))
+            gr.DownloadButton("Download Text Ranking (JSON)", value=v3.download_ranking(df3_t, 'text_ranking', format='json'))
 
         def get_special_processed_df2():
             """Temporary special processing to merge v1 scores with v2 image scores.
@@ -189,8 +193,9 @@ with gr.Blocks() as block:
         # table 5, audio scores
         with gr.TabItem("🎵 Audio [𝓝𝓔𝒲]", elem_id="tab-audio", id=5):
             gr.Markdown(v3.TABLE_INTRODUCTION_A)
+            df3_a = v3.rank_models(df2[v3.COLUMN_NAMES_A], 'Audio-Overall')
             data_component_a = gr.components.Dataframe(
-                value=v3.rank_models(df2[v3.COLUMN_NAMES_A], 'Audio-Overall'),
+                value=df3_a,
                 headers=v3.COLUMN_NAMES_A,
                 type="pandas",
                 datatype=v3.DATA_TITLE_TYPE_A,
@@ -198,6 +203,10 @@ with gr.Blocks() as block:
                 visible=True,
                 max_height=2400, 
             )
+            v3.save_ranking_summary(df3_a, 'audio_ranking')
+            gr.DownloadButton("Download Audio Ranking (CSV)", value=v3.download_ranking(df3_a, 'audio_ranking'))
+            gr.DownloadButton("Download Audio Ranking (JSON)", value=v3.download_ranking(df3_a, 'audio_ranking', format='json'))
+
 
         # table 6, visual document scores only
         with gr.TabItem("📑 Visual Doc", elem_id="tab-visdoc", id=6):
@@ -224,8 +233,9 @@ with gr.Blocks() as block:
         # table 7, agent scores
         with gr.TabItem("🤖 Agents [𝓝𝓔𝒲]", elem_id="tab-agents", id=7):
             gr.Markdown(v3.TABLE_INTRODUCTION_AG)
+            df3_ag = v3.rank_models(df2[v3.COLUMN_NAMES_AG], 'Agent-Overall')
             data_component_ag = gr.components.Dataframe(
-                value=v3.rank_models(df2[v3.COLUMN_NAMES_AG], 'Agent-Overall'),
+                value=df3_ag,
                 headers=v3.COLUMN_NAMES_AG,
                 type="pandas",
                 datatype=v3.DATA_TITLE_TYPE_AG,
@@ -233,9 +243,13 @@ with gr.Blocks() as block:
                 visible=True,
                 max_height=2400, 
             )
+            v3.save_ranking_summary(df3_ag, 'agent_ranking')
+            gr.DownloadButton("Download Agent Ranking (CSV)", value=v3.download_ranking(df3_ag, 'agent_ranking'))
+            gr.DownloadButton("Download Agent Ranking (JSON)", value=v3.download_ranking(df3_ag, 'agent_ranking', format='json'))
 
         # table 8
         with gr.TabItem("📰 About", elem_id="tab-about", id=8):
+            gr.Markdown(SCORING_INFO, elem_classes="markdown-text")
             gr.Image("overview.png", width=900, label="Dataset Overview")
             gr.Markdown(LEADERBOARD_INFO, elem_classes="markdown-text")
 

datasets.py

@@ -20,7 +20,7 @@ DATASETS = {
     "image": {
         "I-CLS": ['VOC2007', 'N24News', 'SUN397', 'ObjectNet', 'Country211', 'Place365', 'ImageNet-1K', 'HatefulMemes', 'ImageNet-A', 'ImageNet-R'], 
         "I-QA": ['OK-VQA', 'A-OKVQA', 'DocVQA', 'InfographicsVQA', 'ChartQA', 'Visual7W', 'ScienceQA', 'GQA', 'TextVQA', 'VizWiz'], 
-        "I-RET": ['VisDial', 'CIRR', 'VisualNews_t2i', 'VisualNews_i2t', 'MSCOCO_t2i', 'MSCOCO_i2t', 'NIGHTS', 'WebQA', 'FashionIQ', 'Wiki-SS-NQ', 'OVEN', 'EDIS'],
+        "I-RET": ['VisDial', 'CIRR', 'VisualNews_t2i', 'VisualNews_i2t', 'MSCOCO_t2i', 'MSCOCO_i2t', 'NIGHTS', 'WebQA', 'FashionIQ', 'Wiki-SS-NQ', 'OVEN', 'EDIS', 'MCMR'],
         "I-VG": ['MSCOCO', 'RefCOCO', 'RefCOCO-Matching', 'Visual7W-Pointing']
         }, 
     "visdoc": {

utils.py

@@ -69,8 +69,23 @@ ANNOUNCEMENT = """
 <li>[2025-06] VLM2Vec/MMEB-V2 released!</li>
 </ul>"""
 
+SCORING_INFO = """
+# MMEB-V3 Datasets Overview and Introductions
+
+## Scoring
+Scores are computed as the unweighted average of all the datasets.\n
+Ex.:\n
+**Overall** is the average score of all the datasets included in MMEB-V3.\n
+**Image-Overall is the average score of all the image datasets.**\n
+**I-RET is the average of all the image retrieval datasets' scores.**\n
+
+Rankings in CSV/JSON is available at [rankings folder](https://huggingface.co/spaces/TIGER-Lab/MMEB-Leaderboard/tree/main/rankings). You may also scroll down to the bottom of each leaderboard and click the buttons to download. 
+
+## Appendix 1: Dataset Overview Visualization
+"""
+
 LEADERBOARD_INFO = f"""
-## Dataset Overview
+## Appendix 2: Dataset Overview in Text
 This is the dictionary of all datasets used in our code. Please make sure all datasets' scores are included in your submission. \n
 ```python
 {pp.pformat(DATASETS)}

utils_v3.py

@@ -1,5 +1,6 @@
 import json
 import os
+import re
 import pandas as pd
 from datetime import datetime
 from utils import create_hyperlinked_names, process_model_size, MODEL_SIZE_COL_NAME
@@ -8,12 +9,12 @@ from datasets import *
 BASE_COLS = ['Rank', 'Models', MODEL_SIZE_COL_NAME, 'Date']
 BASE_DATA_TITLE_TYPE = ['str', 'markdown', 'str', 'str']
 
-OVERALL_COLS_V2 = ["Overall-V2", 'Image-Overall', 'Video-Overall', 'Visdoc-Overall']
+OVERALL_COLS_V2 = ["Overall-V2", 'Image-Overall(V2)', 'Video-Overall', 'Visdoc-Overall']
 COLUMN_NAMES_V2 = BASE_COLS + OVERALL_COLS_V2
 DATA_TITLE_TYPE_V2 = BASE_DATA_TITLE_TYPE + \
                     ['number'] * len(OVERALL_COLS_V2)
 
-OVERALL_COLS_V3 = ["Overall", "Overall-V3🆕", "Text-Overall", "Audio-Overall", "Agent-Overall"]
+OVERALL_COLS_V3 = ["Overall", "Overall-V3🆕", "Text-Overall", "Audio-Overall", "Agent-Overall", 'Image-Overall', 'Video-Overall', 'Visdoc-Overall']
 COLUMN_NAMES_V3 = BASE_COLS + OVERALL_COLS_V3
 DATA_TITLE_TYPE_V3 = BASE_DATA_TITLE_TYPE + \
                     ['number'] * len(OVERALL_COLS_V3)
@@ -25,7 +26,7 @@ DATA_TITLE_TYPE_T = BASE_DATA_TITLE_TYPE + \
                     ['number'] * len(TASKS_T)
 
 SUB_TASKS_I = ["I-CLS", "I-QA", "I-RET", "I-VG"]
-TASKS_I = ['Image-Overall'] + SUB_TASKS_I + ALL_DATASETS_SPLITS['image']
+TASKS_I = ['Image-Overall', 'Image-Overall(V2)'] + SUB_TASKS_I + ALL_DATASETS_SPLITS['image']
 COLUMN_NAMES_I = BASE_COLS + TASKS_I
 DATA_TITLE_TYPE_I = BASE_DATA_TITLE_TYPE + \
                     ['number'] * len(TASKS_I)
@@ -55,12 +56,10 @@ DATA_TITLE_TYPE_AG = BASE_DATA_TITLE_TYPE + \
                     ['number'] * len(TASKS_AG)
 
 TABLE_INTRODUCTION = """**MMEB**: Massive MultiModal Embedding Benchmark \n
-                        Models are ranked based on **Overall**(V3-ALL). **Overall-V3🆕**: Newly added datasets in V3."""
+                        Models are ranked based on **Overall**(V3-ALL). **Overall-V3🆕**: Newly added datasets in V3 (e.g., agent, audio, text and MCMR)"""
 TABLE_INTRODUCTION_I = """**I-CLS**: Image Classification, **I-QA**: (Image) Visual Question Answering, **I-RET**: Image Retrieval, **I-VG**: (Image) Visual Grounding \n
-                        Models are ranked based on **Image-Overall**\n
-                        **Models from the old V1 leaderboard are missing detailed scores of each dataset. 
-                        We hope the authors of the models on V1 leaderboard could rerun your models using our updated V2 pipeline, 
-                        and provide us the scores sheet with the new format, so that we can make them consistent with the other models' formats.**"""
+                        Models are ranked based on **Image-Overall**, the V3 version which includes the newly added I-RET dataset MCMR.\n
+                        **Image-Overall(V2)** is the original V2 version that excludes MCMR.\n"""
 TABLE_INTRODUCTION_V = """**V-CLS**: Video Classification, **V-QA**: (Video) Visual Question Answering, **V-RET**: Video Retrieval, **V-MRET**: Video Moment Retrieval \n
                         Models are ranked based on **Video-Overall**"""
 TABLE_INTRODUCTION_A = """**A-CLS**: Audio Classification, **A-RET**: Audio Retrieval \n
@@ -110,10 +109,9 @@ def load_data(base_dir=SCORE_BASE_DIR):
 
 def load_scores(raw_scores={}):
     """This function loads the raw scores from the user provided scores summary and flattens them into a single dictionary."""
-    # temp fix, will figure out later ===========
+    # merge the three agent tasks into one 
     if any(_ in raw_scores for _ in ['tool', 'gui', 'memory']):
         raw_scores['agent'] = raw_scores.pop('tool', {}) | raw_scores.pop('gui', {}) | raw_scores.pop('memory', {})
-    # ===========================================
     all_scores = {}
     for modality, datasets_list in DATASETS.items(): # Ex.: ('image', {'I-CLS': [...], 'I-QA': [...]})
         for sub_task, datasets in datasets_list.items(): # Ex.: ('I-CLS', ['VOC2007', 'N24News', ...])
@@ -128,14 +126,14 @@ def load_scores(raw_scores={}):
                 all_scores[dataset] = round(score * 100.0, 2)
     return all_scores
 
+def get_avg(sum_score, leng):
+    avg = sum_score / leng if leng > 0 else 0.0
+    avg = round(avg, 2)  # Round to 2 decimal places
+    return avg
+
 def calculate_score(raw_scores=None):
     """This function calculates the overall average scores for all datasets as well as avg scores for each modality and sub-task based on the raw scores.
     """
-    def get_avg(sum_score, leng):
-        avg = sum_score / leng if leng > 0 else 0.0
-        avg = round(avg, 2)  # Round to 2 decimal places
-        return avg
-    
     all_scores = load_scores(raw_scores)
     avg_scores = {}
 
@@ -143,7 +141,7 @@ def calculate_score(raw_scores=None):
     avg_scores['Overall'] = get_avg(sum(all_scores.values()), len(ALL_DATASETS))
     v2_scores = {k:v for k,v in all_scores.items() if k in ALL_DATASETS_SPLITS['image'] or k in ALL_DATASETS_SPLITS['video'] or k in ALL_DATASETS_SPLITS['visdoc']}
     avg_scores['Overall-V2'] = get_avg(sum(v2_scores.values()), len(v2_scores))
-    v3_newonly_scores = {k:v for k,v in all_scores.items() if k in ALL_DATASETS_SPLITS['text'] or k in ALL_DATASETS_SPLITS['audio'] or k in ALL_DATASETS_SPLITS['agent']}
+    v3_newonly_scores = {k:v for k,v in all_scores.items() if k in ALL_DATASETS_SPLITS['text'] or k in ALL_DATASETS_SPLITS['audio'] or k in ALL_DATASETS_SPLITS['agent'] or k == "MCMR"}
     avg_scores['Overall-V3🆕'] = get_avg(sum(v3_newonly_scores.values()), len(v3_newonly_scores))
 
     # Calculate scores for each modality
@@ -153,6 +151,10 @@ def calculate_score(raw_scores=None):
             sum(all_scores.get(dataset, 0.0) for dataset in datasets_for_each_modality),
             len(datasets_for_each_modality)
         )
+    # special process for image overall v2, excluding MCMR
+    img_datasets_without_mcmr = [_ for _ in ALL_DATASETS_SPLITS["image"] if _ != "MCMR"] # exclude MCMR
+    assert len(img_datasets_without_mcmr) == len(ALL_DATASETS_SPLITS["image"])-1 and "MCMR" not in img_datasets_without_mcmr, f"MCMR not removed properly"
+    avg_scores["Image-Overall(V2)"] = get_avg(sum(all_scores.get(dataset, 0.0) for dataset in img_datasets_without_mcmr), len(img_datasets_without_mcmr))
     
     # Calculate scores for each sub-task
     for modality, datasets_list in DATASETS.items():
@@ -205,11 +207,11 @@ def get_df(rank_column='Overall'):
     df = rank_models(df, column=rank_column)
     return df
 
-def refresh_data():
+def refresh_data(columns = COLUMN_NAMES_V3):
     df = get_df()
-    return df[COLUMN_NAMES]
+    return df[columns]
 
-def search_and_filter_models(df, query, min_size, max_size):
+def search_and_filter_models(df, query, min_size, max_size, columns = COLUMN_NAMES_V3):
     filtered_df = df.copy()
     
     if query:
@@ -221,9 +223,22 @@ def search_and_filter_models(df, query, min_size, max_size):
     
     filtered_df = filtered_df[size_mask]
     
-    return filtered_df[COLUMN_NAMES]
+    return filtered_df[columns]
+
+def extract_link_data(text):
+    # Regex to capture content inside href="..." and inside <a>...</a>
+    pattern = r'href="([^"]+)".*?>(.*?)</a>'
+    match = re.search(pattern, text)
+    
+    if match:
+        # Found HTML: return (URL, Name)
+        return match.group(1), match.group(2)
+    
+    # No HTML found: return (None, Original String)
+    return None, text
 
 def save_ranking_summary(df, name, save_now=True, dir='rankings'):
+    # df[['url', 'name']] = df['Models'].apply(lambda x: pd.Series(extract_link_data(x)))
     csv_path, json_path = os.path.join(dir, f'{name}.csv'), os.path.join(dir, f'{name}.jsonl')
     if save_now:
         df.to_csv(csv_path, index=False)