mmeb v3 update

app.py

@@ -1,5 +1,5 @@
 from utils import *
-import utils_v2 as v2
+import utils_v3 as v3
 import gradio as gr
 
 global data_component
@@ -13,25 +13,38 @@ def update_table(query, min_size, max_size, selected_tasks=None):
     return filtered_df
 
 def update_table_v2(query, min_size, max_size):
-    df = v2.get_df()
-    filtered_df = v2.search_and_filter_models(df, query, min_size, max_size)
+    df = v3.get_df()
+    filtered_df = v3.search_and_filter_models(df, query, min_size, max_size)
     return filtered_df
 
 with gr.Blocks() as block:
     gr.Markdown(LEADERBOARD_INTRODUCTION)
-    
+    with gr.Accordion("🔥 What's NEW:"):
+        gr.Markdown(ANNOUNCEMENT)
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         # Table 1, the main leaderboard of overall scores
-        with gr.TabItem("📊 MMEB (V2)", elem_id="tab-overall", id=1):
+        with gr.TabItem("📊 MMEB (V3)", elem_id="tab-overall", id=1):
             with gr.Row():
                 with gr.Accordion("Citation", open=False):
                     citation_button2 = gr.Textbox(
-                        value=v2.CITATION_BUTTON_TEXT,
-                        label=CITATION_BUTTON_LABEL,
+                        value=v3.CITATION_BUTTON_TEXT_V3,
+                        label="Copy the following snippet to cite MMEB-V3",
                         elem_id="citation-button",
                         lines=10,
                     )
-            gr.Markdown(v2.TABLE_INTRODUCTION)
+                    gr.Textbox(
+                        value=v3.CITATION_BUTTON_TEXT_V2,
+                        label="Copy the following snippet to cite VLM2Vec/MMEB-V2",
+                        elem_id="citation-button",
+                        lines=10,
+                    )
+                    gr.Textbox(
+                        value=CITATION_BUTTON_TEXT,
+                        label="Copy the following snippet to cite VLM2Vec/MMEB-V1",
+                        elem_id="citation-button",
+                        lines=10,
+                    )
+            gr.Markdown(v3.TABLE_INTRODUCTION)
 
             with gr.Row():
                 search_bar2 = gr.Textbox(
@@ -42,7 +55,7 @@ with gr.Blocks() as block:
             
             df = get_df()
             df['Date'] = 'unknown'
-            df2 = v2.get_df()
+            df2 = v3.get_df('Overall-V2')
             min_size2, max_size2 = get_size_range(df2)
 
             with gr.Row():
@@ -60,12 +73,12 @@ with gr.Blocks() as block:
                     step=0.1,
                     label="Maximum number of parameters (B)",
                 )
-            df2_all = df2[v2.COLUMN_NAMES]
+            df2_all = df2[v3.COLUMN_NAMES]
             data_component2 = gr.components.Dataframe(
                 value=df2_all,
-                headers=v2.COLUMN_NAMES,
+                headers=v3.COLUMN_NAMES,
                 type="pandas",
-                datatype=v2.DATA_TITLE_TYPE,
+                datatype=v3.DATA_TITLE_TYPE,
                 interactive=False,
                 visible=True,
                 max_height=2400, 
@@ -74,9 +87,9 @@ with gr.Blocks() as block:
             refresh_button2 = gr.Button("Refresh")
 
             # save a summary of rankings
-            v2.save_ranking_summary(df2_all, 'mmeb_ranking')
-            download_overall_but = gr.DownloadButton("Download MMEB Ranking (CSV)", value=v2.download_ranking(df2_all, 'mmeb_ranking'))
-            download_overall_but_json = gr.DownloadButton("Download MMEB Ranking (JSON)", value=v2.download_ranking(df2_all, 'mmeb_ranking', format='json'))
+            v3.save_ranking_summary(df2_all, 'mmeb_ranking')
+            download_overall_but = gr.DownloadButton("Download MMEB Ranking (CSV)", value=v3.download_ranking(df2_all, 'mmeb_ranking'))
+            download_overall_but_json = gr.DownloadButton("Download MMEB Ranking (JSON)", value=v3.download_ranking(df2_all, 'mmeb_ranking', format='json'))
 
             def update_with_tasks_v2(*args):
                 return update_table_v2(*args)
@@ -96,84 +109,122 @@ with gr.Blocks() as block:
                 inputs=[search_bar2, min_size_slider2, max_size_slider2], 
                 outputs=data_component2
             )
-            refresh_button2.click(fn=v2.refresh_data, outputs=data_component2)
+            refresh_button2.click(fn=v3.refresh_data, outputs=data_component2)
 
+        # table 2, text scores
+        with gr.TabItem("📝 Text [NEW]", elem_id="tab-text", id=2):
+            data_component_t = gr.components.Dataframe(
+                value=v3.rank_models(df2[v3.COLUMN_NAMES_T], 'Text-Overall'),
+                headers=v3.COLUMN_NAMES_T,
+                type="pandas",
+                datatype=v3.DATA_TITLE_TYPE_T,
+                interactive=False,
+                visible=True,
+                max_height=2400, 
+            )
 
         def get_special_processed_df2():
             """Temporary special processing to merge v1 scores with v2 image scores.
             Will be removed later after v2 is fully adopted."""
-            df2_i = df2[v2.COLUMN_NAMES_I]
+            df2_i = df2[v3.COLUMN_NAMES_I]
             df1 = df.rename(columns={'V1-Overall': 'Image-Overall'})
-            df1 = df1[v2.BASE_COLS + v2.SUB_TASKS_I + ['Image-Overall']]
+            df1 = df1[v3.BASE_COLS + v3.SUB_TASKS_I + ['Image-Overall']]
             combined_df = pd.concat([df1, df2_i], ignore_index=True)
-            for task in v2.TASKS_I:
+            for task in v3.TASKS_I:
                 combined_df[task] = combined_df[task].apply(lambda score: '-' if pd.isna(score) else score)
-            combined_df = v2.rank_models(combined_df, 'Image-Overall')
-            return combined_df[v2.COLUMN_NAMES_I]
+            combined_df = v3.rank_models(combined_df, 'Image-Overall')
+            return combined_df[v3.COLUMN_NAMES_I]
+        
 
-        # table 2, image scores only
-        with gr.TabItem("🖼️ Image (Previously MMEB-V1)", elem_id="tab-image", id=2):
-            gr.Markdown(v2.TABLE_INTRODUCTION_I)
+        # table 3, image scores only
+        with gr.TabItem("🖼️ Image", elem_id="tab-image", id=3):
+            gr.Markdown(v3.TABLE_INTRODUCTION_I)
             df2_i = get_special_processed_df2()
             data_component3 = gr.components.Dataframe(
                 value=df2_i,
-                headers=v2.COLUMN_NAMES_I,
+                headers=v3.COLUMN_NAMES_I,
                 type="pandas",
-                datatype=v2.DATA_TITLE_TYPE_I,
+                datatype=v3.DATA_TITLE_TYPE_I,
                 interactive=False,
                 visible=True,
                 max_height=2400, 
             )
-            v2.save_ranking_summary(df2_i, 'image_ranking')
-            download_i_but = gr.DownloadButton("Download Image Ranking (CSV)", value=v2.download_ranking(df2_i, 'image_ranking'))
-            download_i_but_json = gr.DownloadButton("Download Image Ranking (JSON)", value=v2.download_ranking(df2_i, 'image_ranking', format='json'))
-
-        # table 3, video scores only
-        with gr.TabItem("💽 Video", elem_id="tab-video", id=3):
-            gr.Markdown(v2.TABLE_INTRODUCTION_V)
-            df2_v = v2.rank_models(df2[v2.COLUMN_NAMES_V], 'Video-Overall')
+            v3.save_ranking_summary(df2_i, 'image_ranking')
+            download_i_but = gr.DownloadButton("Download Image Ranking (CSV)", value=v3.download_ranking(df2_i, 'image_ranking'))
+            download_i_but_json = gr.DownloadButton("Download Image Ranking (JSON)", value=v3.download_ranking(df2_i, 'image_ranking', format='json'))
+
+        # table 4, video scores only
+        with gr.TabItem("💽 Video", elem_id="tab-video", id=4):
+            gr.Markdown(v3.TABLE_INTRODUCTION_V)
+            df2_v = v3.rank_models(df2[v3.COLUMN_NAMES_V], 'Video-Overall')
             data_component4 = gr.components.Dataframe(
                 value=df2_v,
-                headers=v2.COLUMN_NAMES_V,
+                headers=v3.COLUMN_NAMES_V,
                 type="pandas",
-                datatype=v2.DATA_TITLE_TYPE_V,
+                datatype=v3.DATA_TITLE_TYPE_V,
                 interactive=False,
                 visible=True,
                 max_height=2400, 
             )
-            v2.save_ranking_summary(df2_v, 'video_ranking')
-            download_v_but = gr.DownloadButton("Download Video Ranking (CSV)", value=v2.download_ranking(df2_v, 'video_ranking'))
-            download_v_but_json = gr.DownloadButton("Download Video Ranking (JSON)", value=v2.download_ranking(df2_v, 'video_ranking', format='json'))
-
-        # table 4, visual document scores only
-        with gr.TabItem("📑 Visual Doc", elem_id="tab-visdoc", id=4):
-            gr.Markdown(v2.TABLE_INTRODUCTION_D)
-            df2_d = v2.rank_models(df2[v2.COLUMN_NAMES_D], 'Visdoc-Overall')
+            v3.save_ranking_summary(df2_v, 'video_ranking')
+            download_v_but = gr.DownloadButton("Download Video Ranking (CSV)", value=v3.download_ranking(df2_v, 'video_ranking'))
+            download_v_but_json = gr.DownloadButton("Download Video Ranking (JSON)", value=v3.download_ranking(df2_v, 'video_ranking', format='json'))
+
+        # table 5, audio scores
+        with gr.TabItem("🎵 Audio [NEW]", elem_id="tab-audio", id=5):
+            gr.Markdown(v3.TABLE_INTRODUCTION_A)
+            data_component_a = gr.components.Dataframe(
+                value=v3.rank_models(df2[v3.COLUMN_NAMES_A], 'Audio-Overall'),
+                headers=v3.COLUMN_NAMES_A,
+                type="pandas",
+                datatype=v3.DATA_TITLE_TYPE_A,
+                interactive=False,
+                visible=True,
+                max_height=2400, 
+            )
+
+        # table 6, visual document scores only
+        with gr.TabItem("📑 Visual Doc", elem_id="tab-visdoc", id=6):
+            gr.Markdown(v3.TABLE_INTRODUCTION_D)
+            df2_d = v3.rank_models(df2[v3.COLUMN_NAMES_D], 'Visdoc-Overall')
             old_df_vd = pd.read_json('archive/cached_vd_scores_before_fix.jsonl', orient='records', lines=True)
             df2_d = df2_d.merge(old_df_vd, on='Models', how='left')
             df2_d = df2_d.fillna(0)
-            temp_header = v2.COLUMN_NAMES_D[:5] + ['Visdoc-Overall-before-fix', 'VisDoc-OOD-before-fix'] + v2.COLUMN_NAMES_D[5:] + [ 'ViDoSeek-page-before-fix', 'MMLongBench-page-before-fix']
+            temp_header = v3.COLUMN_NAMES_D[:5] + ['Visdoc-Overall-before-fix', 'VisDoc-OOD-before-fix'] + v3.COLUMN_NAMES_D[5:] + [ 'ViDoSeek-page-before-fix', 'MMLongBench-page-before-fix']
             df2_d = df2_d[temp_header]
             data_component5 = gr.components.Dataframe(
                 value=df2_d,
                 headers=temp_header,
                 type="pandas",
-                datatype=v2.DATA_TITLE_TYPE_D+['number']*4,
+                datatype=v3.DATA_TITLE_TYPE_D+['number']*4,
+                interactive=False,
+                visible=True,
+                max_height=2400, 
+            )
+            v3.save_ranking_summary(df2_d, 'visdoc_ranking')
+            download_vd_but = gr.DownloadButton("Download Visual Document Ranking (CSV)", value=v3.download_ranking(df2_d, 'visdoc_ranking'))
+            download_vd_but_json = gr.DownloadButton("Download Visual Document Ranking (JSON)", value=v3.download_ranking(df2_d, 'visdoc_ranking', format='json'))
+
+        # table 7, agent scores
+        with gr.TabItem("🤖 Agents [NEW]", elem_id="tab-agents", id=7):
+            gr.Markdown(v3.TABLE_INTRODUCTION_AG)
+            data_component_ag = gr.components.Dataframe(
+                value=v3.rank_models(df2[v3.COLUMN_NAMES_AG], 'Agent-Overall'),
+                headers=v3.COLUMN_NAMES_AG,
+                type="pandas",
+                datatype=v3.DATA_TITLE_TYPE_AG,
                 interactive=False,
                 visible=True,
                 max_height=2400, 
             )
-            v2.save_ranking_summary(df2_d, 'visdoc_ranking')
-            download_vd_but = gr.DownloadButton("Download Visual Document Ranking (CSV)", value=v2.download_ranking(df2_d, 'visdoc_ranking'))
-            download_vd_but_json = gr.DownloadButton("Download Visual Document Ranking (JSON)", value=v2.download_ranking(df2_d, 'visdoc_ranking', format='json'))
 
-        # table 5
-        with gr.TabItem("📝 About", elem_id="tab-about", id=5):
+        # table 8
+        with gr.TabItem("📰 About", elem_id="tab-about", id=8):
             gr.Image("overview.png", width=900, label="Dataset Overview")
             gr.Markdown(LEADERBOARD_INFO, elem_classes="markdown-text")
 
-        # table 6
-        with gr.TabItem("🚀 Submit here! ", elem_id="tab-submit", id=6):
+        # table 9
+        with gr.TabItem("🚀 Submit here! ", elem_id="tab-submit", id=9):
             with gr.Row():
                 gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")
 

datasets.py

@@ -8,6 +8,15 @@ def sum_lol(lol):
 SCORE_BASE_DIR = "scores"
 META_DATA = ["model_name", "model_size", "url", "report_generated_date"]
 DATASETS = {
+    "text": {
+        "FollowIR": ['core17-instructions', 'news21-instructions', 'robust04-instructions'],
+        "R2MED": ['Bioinformatics', 'Biology', 'IIYi-Clinical', 'MedQA-Diag', 'MedXpertQA-Exam', 'Medical-Sciences', 'PMC-Clinical', 'PMC-Treatment'],
+        "InfoSearch": ['Audience-v1', 'Clarity-v1', 'Format-v1', 'Language-v1', 'Length-v1', 'Source-v1'],
+        "BRIGHT": ['aops', 'biology', 'earth_science', 'economics', 'leetcode', 'pony', 'psychology', 'robotics', 'stackoverflow', 'sustainable_living', 'theoremqa_questions', 'theoremqa_theorems'],
+        "LongEmbed": ['2wikimqa', 'narrativeqa', 'needle', 'passkey', 'qmsum', 'summ_screen_fd'],
+        "MultiConIR": ['Books', 'Legal Document', 'Medical Case', 'Movies', 'People'],
+        "NanoBEIR": ['NanoArguAna', 'NanoClimateFEVER', 'NanoDBPedia', 'NanoFEVER', 'NanoFiQA2018', 'NanoHotpotQA', 'NanoMSMARCO', 'NanoNFCorpus', 'NanoNQ', 'NanoQuoraRetrieval', 'NanoSCIDOCS', 'NanoSciFact', 'NanoTouche2020']
+    }, 
     "image": {
         "I-CLS": ['VOC2007', 'N24News', 'SUN397', 'ObjectNet', 'Country211', 'Place365', 'ImageNet-1K', 'HatefulMemes', 'ImageNet-A', 'ImageNet-R'], 
         "I-QA": ['OK-VQA', 'A-OKVQA', 'DocVQA', 'InfographicsVQA', 'ChartQA', 'Visual7W', 'ScienceQA', 'GQA', 'TextVQA', 'VizWiz'], 
@@ -25,7 +34,18 @@ DATASETS = {
         "V-QA": ['Video-MME', 'MVBench', 'NExTQA', 'EgoSchema', 'ActivityNetQA'], 
         "V-RET": ['MSR-VTT', 'MSVD', 'DiDeMo', 'VATEX', 'YouCook2'], 
         "V-MRET": ['QVHighlight', 'Charades-STA', 'MomentSeeker']
-        }
+        }, 
+    "audio": {
+        "A-CLS": ["SpeechCommands", "UrbanSound8K", "ESC-50", "NSynth", "CREMA-D"], 
+        "A-RET": ['Clotho', 'SoundDescs', 'TUTSound', 'TUTSound(hard)', 'AVE', 'SpeechCOCO']
+        },
+    "agent": {
+        "Tool": ['apibank', 'apigen', 'mnms', 'reversechain', 'rotbench', 't-eval-dialog', 't-eval-step', 'taskbench-daily', 'toolace', 'toolbench', 'toolemu', 'tooleyes', 'toollens', 'ultratool', 'autotools-food', 'autotools-music', 'autotools-weather', 'restgpt-spotify', 'restgpt-tmdb',
+                  'craft-math-algebra', 'craft-tabmwp', 'craft-vqa', 'gorilla-huggingface', 'gorilla-pytorch', 'gorilla-tensor', 'toolink', 
+                  'appbench', 'gpt4tools', 'gta', 'taskbench-huggingface', 'taskbench-multimedia', 'metatool', 'tool-be-honest', 'toolalpaca', 'toolbench-sam'],
+        "GUI": ['GAE-GUIAct_q2t', 'GAE-GUIAct_q2s', 'GAE-GUIAct_s2s', 'GAE-GUIAct_t2s', 'GAE-Mind2Web_q2t', 'GAE-Mind2Web_q2s', 'GAE-Mind2Web_s2s', 'GAE-Mind2Web_t2s'],
+        "Memory": ["REALTALK", "KnowMeBench", "PeerQA", "DeepPlanning"]
+    }
 }
 ALL_DATASETS_SPLITS = {k: sum_lol(list(v.values())) for k, v in DATASETS.items()}
 ALL_DATASETS = sum_lol(list(ALL_DATASETS_SPLITS.values()))

report_score_v3.py

@@ -0,0 +1,250 @@
+"""
+report_score_v3.py: The official script to generate the final scores report (in JSON) for MMEB-V3 leaderboard submission. 
+You should only modify the Configuration section below to add your model's metadata. No other changes are needed.
+An example configuration is provided for your reference. 
+"""
+
+import os
+import json
+from datetime import datetime
+
+# ==============================================================================
+# Configuration
+# ==============================================================================
+
+# ==> Unified list of experiments to process.
+# Fill in the metadata for each experiment. `None` will become `null` in the JSON.
+
+EXAMPLES = [
+    {
+        "path": "vlm2vec_exps/VLM2Vec-Qwen2VL-V2.0-2B/",
+        "metadata": {
+            "model_name": "VLM2Vec-Qwen2VL-V2.0-2B",
+            "model_size": "2", # in B, digits only here. 2 for 2B, 0.4 for 400M, etc.
+            "embedding_dimension": None, # Please fill in
+            "max_length_tokens": None,   # Please fill in
+            "model_release_date": "2025-04-01", # Please adjust this date
+            "score_source": "",           # e.g., "Self-Reported" or "TIGER-Lab"
+            "url": ""                    # e.g., Paper, GitHub, or Hugging Face link
+        }
+    },
+    {
+        "path": "vlm2vec_exps/VLM2Vec-Qwen2VL-V2.1-2B/",
+        "metadata": {
+            "model_name": "VLM2Vec-Qwen2VL-V2.1-2B",
+            "model_size": "2", # in B, digits only here. 2 for 2B, 0.4 for 400M, etc.
+            "embedding_dimension": None, # Please fill in
+            "max_length_tokens": None,   # Please fill in
+            "model_release_date": "2025-05-15", # Please adjust this date
+            "score_source": "",           # e.g., "Self-Reported" or "TIGER-Lab"
+            "url": ""                    # e.g., Paper, GitHub, or Hugging Face link
+        }
+    },
+]
+
+
+# ==============================================================================
+# TODO: Your models' metadata goes here. Please fill in the required fields.
+# ==============================================================================
+
+EXPERIMENTS = [
+    {
+        "path": ...,
+        "metadata": {
+            "model_name": ..., # * Mandatory Field
+            "model_backbone": ...,
+            "model_size": ...,
+            "embedding_dimension": ...,
+            "max_length_tokens": ...,
+            "model_release_date": ...,
+            "data_source": "Self-Reported",
+            "url": ..., 
+            "contact": ...,
+        }
+    },
+    ...
+]
+
+
+# ==============================================================================
+# Main Processing Logic (No changes needed below this line)
+# ==============================================================================
+
+
+# Define the datasets grouped by modality
+modality2dataset = {
+    "audio": [
+        'SpeechCommand', 'UrbanSound8k', 'ESC-50', 'NSynth', 'CREMA-D', 'Clotho', 'SoundDescs', 'TUTSound', 'TUTSound(hard)', 'AVE', 'SpeechCOCO'
+        ], 
+    "image": [
+        "ImageNet-1K", "N24News", "HatefulMemes", "VOC2007", "SUN397", "Place365", "ImageNet-A", "ImageNet-R", "ObjectNet", "Country211",
+        "OK-VQA", "A-OKVQA", "DocVQA", "InfographicsVQA", "ChartQA", "Visual7W", "ScienceQA", "VizWiz", "GQA", "TextVQA",
+        "VisDial", "CIRR", "VisualNews_t2i", "VisualNews_i2t", "MSCOCO_t2i", "MSCOCO_i2t", "NIGHTS", "WebQA", "FashionIQ", "Wiki-SS-NQ", "OVEN", "EDIS",
+        "MSCOCO", "RefCOCO", "RefCOCO-Matching", "Visual7W-Pointing"
+        ],
+    "video": [
+        "K700", "SmthSmthV2", "HMDB51", "UCF101", "Breakfast",
+        "MVBench", "Video-MME", "NExTQA", "EgoSchema", "ActivityNetQA",
+        "DiDeMo", "MSR-VTT", "MSVD", "VATEX", "YouCook2",
+        "QVHighlight", "Charades-STA", "MomentSeeker",
+    ],
+    "visdoc": [
+        "ViDoRe_arxivqa", "ViDoRe_docvqa", "ViDoRe_infovqa", "ViDoRe_tabfquad", "ViDoRe_tatdqa", "ViDoRe_shiftproject",
+        "ViDoRe_syntheticDocQA_artificial_intelligence", "ViDoRe_syntheticDocQA_energy", "ViDoRe_syntheticDocQA_government_reports", "ViDoRe_syntheticDocQA_healthcare_industry",
+        "ViDoRe_esg_reports_human_labeled_v2", "ViDoRe_biomedical_lectures_v2_multilingual", "ViDoRe_economics_reports_v2_multilingual", "ViDoRe_esg_reports_v2_multilingual",
+        "VisRAG_ArxivQA", "VisRAG_ChartQA", "VisRAG_MP-DocVQA", "VisRAG_SlideVQA", "VisRAG_InfoVQA", "VisRAG_PlotQA",
+        "ViDoSeek-page", "ViDoSeek-doc", "MMLongBench-page", "MMLongBench-doc"
+    ], 
+    "text": [
+        'core17-instructions', 'news21-instructions', 'robust04-instructions', 'Bioinformatics', 'Biology', 'IIYi-Clinical', 'MedQA-Diag', 'MedXpertQA-Exam', 'Medical-Sciences', 'PMC-Clinical', 'PMC-Treatment', 'Audience-v1', 'Clarity-v1', 'Format-v1', 'Language-v1', 'Length-v1', 'Source-v1', 'aops', 'biology', 'earth_science', 'economics', 'leetcode', 'pony', 'psychology', 'robotics', 'stackoverflow', 'sustainable_living', 'theoremqa_questions', 'theoremqa_theorems', '2wikimqa', 'narrativeqa', 'needle', 'passkey', 'qmsum', 'summ_screen_fd', 'Books', 'Legal Document', 'Medical Case', 'Movies', 'People', 'NanoArguAna', 'NanoClimateFEVER', 'NanoDBPedia', 'NanoFEVER', 'NanoFiQA2018', 'NanoHotpotQA', 'NanoMSMARCO', 'NanoNFCorpus', 'NanoNQ', 'NanoQuoraRetrieval', 'NanoSCIDOCS', 'NanoSciFact', 'NanoTouche2020'
+        ], 
+    "tool": [
+        'apibank', 'apigen', 'mnms', 'reversechain', 'rotbench', 't-eval-dialog', 't-eval-step', 'taskbench-daily', 'toolace', 'toolbench', 'toolemu', 'tooleyes', 'toollens', 'ultratool', 'autotools-food', 'autotools-music', 'autotools-weather', 'restgpt-spotify', 'restgpt-tmdb', 'craft-math-algebra', 'craft-tabmwp', 'craft-vqa', 'gorilla-huggingface', 'gorilla-pytorch', 'gorilla-tensor', 'toolink', 'appbench', 'gpt4tools', 'gta', 'taskbench-huggingface', 'taskbench-multimedia', 'metatool', 'tool-be-honest', 'toolalpaca', 'toolbench-sam'
+        ], 
+    "gui": [
+        'GAE-GUIAct_q2t', 'GAE-GUIAct_q2s', 'GAE-GUIAct_s2s', 'GAE-GUIAct_t2s', 'GAE-Mind2Web_q2t', 'GAE-Mind2Web_q2s', 'GAE-Mind2Web_s2s', 'GAE-Mind2Web_t2s'
+        ], 
+    "memory": [
+        'REALTALK', 'KnowMeBench', 'PeerQA', 'DeepPlanning'
+        ]
+}
+modality2metric = {
+    "image": "hit@1",
+    "video": "hit@1",
+    "visdoc": "ndcg_linear@5",
+    "audio": "hit@1", 
+    "text": "hit@1",
+    "tool": "hit@1",
+    "gui": "hit@1",
+    "memory": "hit@1"
+}
+modalities = ["image", "video", "visdoc", "audio", "text", "tool", "gui", "memory"] # Process in this order
+
+for experiment in EXPERIMENTS:
+    base_path = experiment['path']
+    experiment_metadata = experiment['metadata']
+    experiment_name_for_log = os.path.basename(base_path.strip('/'))
+
+    current_experiment_scores = {}
+
+    print(f"\nProcessing experiment: {experiment_name_for_log}")
+    print(f"Path: {base_path}")
+
+    for modality in modalities:
+        current_experiment_scores[modality] = {}
+        modality_specific_result_dir = os.path.join(base_path, modality)
+
+        for dataset_name in modality2dataset.get(modality, []):
+            current_experiment_scores[modality][dataset_name] = "FILE_N/A" # Initialize
+
+        if not os.path.isdir(modality_specific_result_dir):
+            print(f"    Directory not found: {modality_specific_result_dir}")
+            for dataset_name in modality2dataset.get(modality, []):
+                current_experiment_scores[modality][dataset_name] = "DIR_N/A"
+            continue
+
+        for filename in os.listdir(modality_specific_result_dir):
+            if filename.endswith("_score.json"):
+                score_file_path = os.path.join(modality_specific_result_dir, filename)
+                dataset_name_from_file = None
+                for known_dataset in modality2dataset.get(modality, []):
+                    if filename == f"{known_dataset}_score.json":
+                        dataset_name_from_file = known_dataset
+                        break
+
+                if dataset_name_from_file:
+                    try:
+                        with open(score_file_path, "r") as f:
+                            score_data = json.load(f)
+                            current_experiment_scores[modality][dataset_name_from_file] = score_data
+                    except json.JSONDecodeError:
+                        print(f"      Error decoding JSON from {score_file_path}")
+                        current_experiment_scores[modality][dataset_name_from_file] = "JSON_ERROR"
+                    except Exception as e:
+                        print(f"      Error reading file {score_file_path}: {e}")
+                        current_experiment_scores[modality][dataset_name_from_file] = "READ_ERROR"
+
+    # --- Construct and Save the Final JSON Report ---
+    final_metadata = experiment_metadata.copy()
+    final_metadata['report_generated_date'] = datetime.now().isoformat()
+    # --- Special handling ---
+    # Special handling for VisDoc datasets with "-fixed" suffix
+    current_experiment_scores['visdoc'] = {(f'{k}-fixed' if k in ['MMLongBench-page', 'ViDoSeek-page'] else k): v 
+                                           for k, v in current_experiment_scores['visdoc'].items()} # temp fix
+    # merge "tool", "gui", and "memory" into "agent"
+    # current_experiment_scores['agent'] = current_experiment_scores.pop('tool', {}) | current_experiment_scores.pop('gui', {}) | current_experiment_scores.pop('memory', {})
+    # -------------------------
+
+    final_output = {
+        "metadata": final_metadata,
+        "metrics": current_experiment_scores
+    }
+
+    output_json_path = os.path.join(base_path, f"{final_metadata['model_name']}.json")
+    try:
+        with open(output_json_path, "w") as f:
+            json.dump(final_output, f, indent=4)
+        print(f"  Report for '{experiment_name_for_log}' saved to: {output_json_path}")
+    except Exception as e:
+        print(f"  Error saving JSON report for '{experiment_name_for_log}' to {output_json_path}: {e}")
+
+
+    # --- Print detailed main scores per dataset for easy copy to spreadsheet ---
+    print(f"\n  --- Detailed Main Scores for Spreadsheet (Experiment: {experiment_name_for_log}) ---")
+    for modality in modalities:
+        main_metric_key = modality2metric[modality]
+        for dataset_name in modality2dataset.get(modality, []):
+            score_to_print_val = "NOT_FOUND_IN_RESULTS"
+            modality_data = current_experiment_scores.get(modality, {})
+            score_info = modality_data.get(dataset_name)
+
+            if isinstance(score_info, dict):
+                metric_value = score_info.get(main_metric_key)
+                if isinstance(metric_value, (int, float)):
+                    score_to_print_val = f"{metric_value:.4f}"
+                else:
+                    score_to_print_val = f"METRIC_KEY_MISSING ({main_metric_key})"
+            elif isinstance(score_info, str):
+                score_to_print_val = score_info
+
+            print(f"{dataset_name}\t{score_to_print_val}")
+        print("")
+
+    # --- Print average scores and missing datasets per modality ---
+    print(f"\n  --- Summary for Experiment: {experiment_name_for_log} ---")
+    for modality in modalities:
+        if modality not in current_experiment_scores:
+            print(f"    Modality '{modality.upper()}' not processed.")
+            continue
+        main_metric_key = modality2metric[modality]
+        modality_data = current_experiment_scores[modality]
+        collected_metric_values = []
+        datasets_missing_score_file = []
+        datasets_file_found_metric_missing = []
+
+        for dataset_name in modality2dataset.get(modality, []):
+            score_info = modality_data.get(dataset_name)
+            if isinstance(score_info, dict):
+                metric_value = score_info.get(main_metric_key)
+                if isinstance(metric_value, (int, float)):
+                    collected_metric_values.append(metric_value)
+                else:
+                    datasets_file_found_metric_missing.append(f"{dataset_name} (metric '{main_metric_key}' missing/invalid)")
+            else:
+                datasets_missing_score_file.append(f"{dataset_name} (status: {score_info if score_info else 'Not Processed'})")
+
+        if collected_metric_values:
+            average_score = sum(collected_metric_values) / len(collected_metric_values)
+            print(f"      Average of {modality.upper()}\t- {main_metric_key}:\t{average_score:.4f} (from {len(collected_metric_values)} datasets)")
+        else:
+            print(f"      Average of {modality.upper()}\t-  {main_metric_key}:\tN/A (no valid scores found)")
+
+        if datasets_missing_score_file:
+            print(f"      Datasets with missing/errored score files:")
+            for ds_status in datasets_missing_score_file: print(f"        - {ds_status}")
+        if datasets_file_found_metric_missing:
+            print(f"      Score files found but main metric ('{main_metric_key}') missing/invalid:")
+            for ds_status in datasets_file_found_metric_missing: print(f"        - {ds_status}")
+
+
+print("\nProcessing complete.")

utils.py

@@ -18,43 +18,51 @@ LEADERBOARD_INTRODUCTION = """
 # 📊 **MMEB LEADERBOARD (VLM2Vec)**
 
 ## Introduction
-We introduce a novel benchmark, **MMEB-V1 (Massive Multimodal Embedding Benchmark)**, 
-which includes 36 datasets spanning four meta-task categories: classification, visual question answering, retrieval, and visual grounding. MMEB provides a comprehensive framework for training
-and evaluating embedding models across various combinations of text and image modalities. 
-All tasks are reformulated as ranking tasks, where the model follows instructions, processes a query, and selects the correct target from a set of candidates. The query and target can be an image, text,
-or a combination of both. MMEB-V1 is divided into 20 in-distribution datasets, which can be used for
-training, and 16 out-of-distribution datasets, reserved for evaluation.
-
-Building upon on **MMEB-V1**, **MMEB-V2** expands the evaluation scope to include five new tasks: four video-based tasks 
-— Video Retrieval, Moment Retrieval, Video Classification, and Video Question Answering — and one task focused on visual documents, Visual Document Retrieval. 
-This comprehensive suite enables robust evaluation of multimodal embedding models across static, temporal, and structured visual data settings.
-
-**⚠️ Your attention please:==================================================================================**  
-We have fixed the errors found in ViDoSeek-page and MMLongBench-page datasets. Detailed information about the issues can be viewed [HERE](https://github.com/TIGER-AI-Lab/VLM2Vec/issues/167).   
-‼️ CALL TO ACTION: Please verify that your model's scores are accurate by following the instructions below. ‼️
-
-The scores submitted before the fix have been renamed to **ViDoSeek-page-before-fix** and **MMLongBench-page-before-fix**, and the current **Overall** and **Visdoc-Overall** are using the **fixed versions of these two datasets (eg. ViDoSeek-page-fixed and MMLongBench-page-fixed)**, which means the models submitted before the fix might now have a lower ranking. 
-
-Here is the list of models affected by this fix: [File](https://huggingface.co/spaces/TIGER-Lab/MMEB-Leaderboard/blob/main/archive/cached_vd_scores_before_fix.jsonl). If your model is on this list, we kindly ask you to visit the VisDoc leaderboard and verify if your model is missing the scores for ViDoSeek-page-fixed and MMLongBench-page-fixed, and then re-evaluate your model on these two datasets and submit the updated scores to ensure accurate ranking on the leaderboard. 
-
-In the generated report sheet, please double check that ViDoSeek-page and MMLongBench-page have been renamed to ViDoSeek-page-fixed and MMLongBench-page-fixed respectively. 
-
-Please let us know if you have any questions or concerns, or there is an issue with your model's scores. We appreciate your understanding and cooperation in maintaining the integrity of the benchmark.
-**===========================================================================================================**
+We introduce **Massive Multimodal Embedding Benchmark (MMEB)**, a novel comprehensive benchmark for evaluating omni-modality embedding models across text, image, video, audio, visual document, and agent-centric retrieval scenarios.  
+
+**MMEB-V1** includes 36 datasets spanning four image-text meta-task categories: classification, visual question answering, retrieval, visual grounding.
+
+**MMEB-V2** expands the evaluation scope to include five new tasks: 
+- four video-based tasks: Video Retrieval, Moment Retrieval, Video Classification, and Video Question Answering 
+- one task focused on visual documents: Visual Document Retrieval. 
+
+**MMEB-V3** further extends to a fuller modality setting by adding three major new evaluation categories:
+- Audio Tasks: audio classification, cross-modal audio retrieval, and audio temporal grounding.
+- Text Retrieval: instruction-following retrieval, reasoning retrieval, long-context retrieval, multi-condition retrieval, and general text retrieval.
+- Agent Tasks: tool retrieval, GUI control, and agent memory retrieval.
+
+<div style="display:inline-flex; flex-wrap:wrap; gap:6px; align-items:center; margin:8px 0;">
+<a target="_blank" href="https://tiger-ai-lab.github.io/VLM2Vec/">
+<img style="height:17pt" src="https://img.shields.io/badge/-🌐%20Website-red?style=flat"></a>
+<a target="_blank" href="https://github.com/TIGER-AI-Lab/VLM2Vec">
+<img style="height:17pt" src="https://img.shields.io/badge/-MMEB--V3%20Code-green?style=flat&logo=github"></a>
+<a target="_blank" href="https://huggingface.co/datasets/VLM2Vec/MMEB-V3">
+<img style="height:17pt" src="https://img.shields.io/badge/-🤗%20Dataset(MMEB--V3)-red?style=flat"></a>
+<a target="_blank" href="https://arxiv.org/abs/2604.23321">
+<img style="height:17pt" src="https://img.shields.io/badge/-V3 Paper-black?style=flat&logo=arxiv"></a>
+<a target="_blank" href="https://arxiv.org/abs/2507.04590">
+<img style="height:17pt" src="https://img.shields.io/badge/-V2 Paper-black?style=flat&logo=arxiv"></a>
+<a target="_blank" href="https://arxiv.org/abs/2410.05160">
+<img style="height:17pt" src="https://img.shields.io/badge/-V1 Paper-black?style=flat&logo=arxiv"></a>
+<a target="_blank" href="https://huggingface.co/datasets/TIGER-Lab/MMEB-V2">
+<img style="height:17pt" src="https://img.shields.io/badge/-🤗%20Dataset(V2)-red?style=flat"></a>
+<a target="_blank" href="https://huggingface.co/datasets/TIGER-Lab/MMEB-eval">
+<img style="height:17pt" src="https://img.shields.io/badge/-🤗%20Dataset(V1)-red?style=flat"></a>
+<a target="_blank" href="https://huggingface.co/VLM2Vec">
+<img style="height:17pt" src="https://img.shields.io/badge/-🤗%20Models-red?style=flat"></a>
+<a target="_blank" href="https://huggingface.co/spaces/TIGER-Lab/MMEB-Leaderboard">
+<img style="height:17pt" src="https://img.shields.io/badge/-🤗%20Leaderboard-red?style=flat"></a>
+</div>
+"""
 
-<summary><span style='font-weight:bold'>🔥 What's NEW: </span></summary>
+ANNOUNCEMENT = """
 <ul>
-<li>[2026-01] ⚠️ The issues found in ViDoSeek-page and MMLongBench-page datasets have been fixed.</li>
-<li>[2025-11] The leaderboards' rankings can be directly downloaded in csv/json format. Scroll down to the bottom of this page and click the button to download.</li>
-<li>[2025-06] MMEB-V2 released!</li>
-</ul>
-
-| [**📈Overview**](https://tiger-ai-lab.github.io/VLM2Vec/) | [**Github**](https://github.com/TIGER-AI-Lab/VLM2Vec) 
-| [**📖MMEB-V2/VLM2Vec-V2 Paper**](https://arxiv.org/abs/2507.04590) 
-| [**📖MMEB-V1/VLM2Vec-V1 Paper**](https://arxiv.org/abs/2410.05160) 
-| [**🤗Hugging Face**](https://huggingface.co/datasets/TIGER-Lab/MMEB-V2) 
-| [**Discord**](https://discord.gg/njyKubdtry) |
-"""
+<li>[2026-07] MMEB-V3 Leaderboard is now live!</li>
+<li>[2026-07] MMEB-V3 is accepted to COLM 2026!</li>
+<li>[2026-04] MMEB-V3 released!</li>
+<li>[2026-01] VLM2Vec/MMEB-V2 is accepted to TMLR 2026!</li>
+<li>[2025-06] VLM2Vec/MMEB-V2 released!</li>
+</ul>"""
 
 LEADERBOARD_INFO = f"""
 ## Dataset Overview
@@ -73,27 +81,32 @@ CITATION_BUTTON_TEXT = r"""@article{jiang2024vlm2vec,
 }"""
 
 SUBMIT_INTRODUCTION = """# Submit on MMEB Leaderboard Introduction \n
-## We have provided detailed step-by-step guides on how to submit your model. Please read carefully in order to submit successfully. \n
+## Please follow the guidelines in order to submit successfully. \n
 1. **Step  1️⃣:** Please refer to the [**GitHub page**](https://github.com/TIGER-AI-Lab/VLM2Vec) for detailed instructions about evaluating your model. \n
-2. **Step  2️⃣:** After running the evaluation pipelines, please use the script we provided **(e.g., [report_score_v2.py](https://github.com/TIGER-AI-Lab/VLM2Vec/blob/main/experiments/report_score_v2.py))** to generate the unified score sheet. 
-    - Reminder: Adjust your model's configurations in the script, including model name, URL, model size, and data source.
-        - Note the "model size" field is digits-only and is by default in Billions (B), so please convert it accordingly if your model size is in different units/formats (e.x., "8" for 8 billion, "0.5" for 500 million, don't include non-digits in it).
+2. **Step  2️⃣:** After running the evaluation pipelines, please use the provided script **(e.g., [report_score_v2.py](https://github.com/TIGER-AI-Lab/VLM2Vec/blob/main/experiments/report_score_v2.py))** to generate the final score sheet. (Use [report_score_v3.py](https://github.com/TIGER-AI-Lab/VLM2Vec/blob/main/experiments/report_score_v3.py) for v3 submission).
+    - Adjust your model's configurations in the script before running it
+        - Note the "model size" field is digits-only and is in Billions (B), so please convert it if yours is in different units/formats (e.x., "8" for 8 billion, "0.5" for 500 million).
         - If possible, please also add a contact method in case we want to reach you in the future
 3. **Step  3️⃣:** Finally, create a pull request and upload the generated JSON file to the ***scores*** folder. 
-    - You can directly use the Hugging Face Space's web UI to upload your files:
+    - If directly using web UI:
         - Go to the [scores folder](https://huggingface.co/spaces/TIGER-Lab/MMEB-Leaderboard/upload/main/scores)
         - Select "Upload file" and upload your JSON files.
-    - Please inform us on [our discord server](https://discord.gg/njyKubdtry), or send us an email at m7su@uwaterloo.ca, or leave a simple message (@ us) in the PR to indicate that you are ready to merge your PR. 
-    - We will then review your submission and update the leaderboard accordingly. \n\n
-## Please double check your score sheet have the following JSON format ⬇️: \n
+    - If by git command line: refer to the [PR documentation](https://huggingface.co/docs/hub/repositories-pull-requests-discussions#pull-requests-advanced-usage).
+    - Submit the PR and leave any comments if any. We will then review and update the leaderboard accordingly.\n
+    - To delete or modify your submission, submit a new PR with the updated file.\n\n
+
+## 🐞 Bug reporting and feedback
+If you encounter any issues or have improvement feedback regarding the leaderboard, please report them in Discussion.\n
+If you cannot reach us via above methods, email us at **m7su@uwaterloo.ca**. 
+
+## Appendix 1: Example: valid score sheet format ⬇️: \n
 ```json
 {
     "metadata": {
         "model_name": "<Model Name>",
         "url": "<Model URL>" or null,
         "model_size": <Model Size> or null,
-        "data_source": "Self-Reported",
-        "contact": xxx@gmail.com
+        "contact": xxx@xxxxx.com
         ... ...
     },
     "metrics": {
@@ -116,11 +129,9 @@ SUBMIT_INTRODUCTION = """# Submit on MMEB Leaderboard Introduction \n
 }
 ```
 ## ⚠️ Special Instructions for submitting to MMEB Image (Previously MMEB-V1) Leaderboard
-We understand that some researchers want to exclusively submit to the Image leaderboard, but unfortunately our current leaderboard system cannot exclude your model from other modalities' leaderboards. 
-To do so, please run the 36 image datasets only and simply ignore the other datasets.
-The leaderboard will automatically assign a 0 to the missing datasets. 
-We might be able to hide your model from the other leaderboards in the next leaderboard updates, but for now your model will be shown on all leaderboards and might have a lower rank. \n
-We highly recommend joining our [discord server](https://discord.gg/njyKubdtry), which provides a convenient way to stay informed with latest updates, or share any feedback you have for improving the leaderboard experience. We appreciate your contributions to the MMEB community!
+We understand that some researchers want to exclusively submit to the Image leaderboard, but unfortunately our current leaderboard cannot exclude your model from other modalities' leaderboards. 
+To do so, run the 36 image datasets only and simply ignore other datasets.
+The leaderboard will automatically assign a 0 to the missing datasets and your model will be shown on all leaderboards, and might have a lower rank. \n
 """
 
 def create_hyperlinked_names(df):

utils_v2.py → utils_v3.py

@@ -6,11 +6,18 @@ from utils import create_hyperlinked_names, process_model_size
 from datasets import *
 
 BASE_COLS = ['Rank', 'Models', 'Model Size(B)', 'Date']
-BASE_DATA_TITLE_TYPE = ['number', 'markdown', 'str', 'str']
+BASE_DATA_TITLE_TYPE = ['str', 'markdown', 'str', 'str']
 
-COLUMN_NAMES = BASE_COLS + ["Overall", 'Image-Overall', 'Video-Overall', 'Visdoc-Overall']
+OVERALL_COLS = ["Overall", "Overall-V2", "Text-Overall", 'Image-Overall', 'Video-Overall', 'Audio-Overall', 'Visdoc-Overall', "Agent-Overall"]
+COLUMN_NAMES = BASE_COLS + OVERALL_COLS
 DATA_TITLE_TYPE = BASE_DATA_TITLE_TYPE + \
-                    ['number'] * 4
+                    ['number'] * len(OVERALL_COLS)
+
+SUB_TASKS_T = ["FollowIR", "R2MED", "InfoSearch", "BRIGHT", "LongEmbed", "MultiConIR", "NanoBEIR"]
+TASKS_T = ['Text-Overall'] + SUB_TASKS_T + ALL_DATASETS_SPLITS['text']
+COLUMN_NAMES_T = BASE_COLS + TASKS_T
+DATA_TITLE_TYPE_T = BASE_DATA_TITLE_TYPE + \
+                    ['number'] * len(TASKS_T)
 
 SUB_TASKS_I = ["I-CLS", "I-QA", "I-RET", "I-VG"]
 TASKS_I = ['Image-Overall'] + SUB_TASKS_I + ALL_DATASETS_SPLITS['image']
@@ -24,14 +31,26 @@ COLUMN_NAMES_V = BASE_COLS + TASKS_V
 DATA_TITLE_TYPE_V = BASE_DATA_TITLE_TYPE + \
                     ['number'] * len(TASKS_V)
 
+SUB_TASKS_A = ["A-CLS", "A-RET"]
+TASKS_A = ['Audio-Overall'] + SUB_TASKS_A + ALL_DATASETS_SPLITS['audio']
+COLUMN_NAMES_A = BASE_COLS + TASKS_A
+DATA_TITLE_TYPE_A = BASE_DATA_TITLE_TYPE + \
+                    ['number'] * len(TASKS_A)
+
 SUB_TASKS_D = ['ViDoRe-V1', 'ViDoRe-V2', 'VisRAG', 'VisDoc-OOD']
 TASKS_D = ['Visdoc-Overall'] + SUB_TASKS_D + ALL_DATASETS_SPLITS['visdoc']
 COLUMN_NAMES_D = BASE_COLS + TASKS_D
 DATA_TITLE_TYPE_D = BASE_DATA_TITLE_TYPE + \
                     ['number'] * len(TASKS_D)
 
+SUB_TASKS_AG = ['Tool', 'GUI', 'Memory']
+TASKS_AG = ['Agent-Overall'] + SUB_TASKS_AG + ALL_DATASETS_SPLITS['agent']
+COLUMN_NAMES_AG = BASE_COLS + TASKS_AG
+DATA_TITLE_TYPE_AG = BASE_DATA_TITLE_TYPE + \
+                    ['number'] * len(TASKS_AG)
+
 TABLE_INTRODUCTION = """**MMEB**: Massive MultiModal Embedding Benchmark \n
-                        Models are ranked based on **Overall**"""
+                        Models are ranked based on **Overall-V2**."""
 TABLE_INTRODUCTION_I = """**I-CLS**: Image Classification, **I-QA**: (Image) Visual Question Answering, **I-RET**: Image Retrieval, **I-VG**: (Image) Visual Grounding \n
                         Models are ranked based on **Image-Overall**\n
                         **Models from the old V1 leaderboard are missing detailed scores of each dataset. 
@@ -39,15 +58,19 @@ TABLE_INTRODUCTION_I = """**I-CLS**: Image Classification, **I-QA**: (Image) Vis
                         and provide us the scores sheet with the new format, so that we can make them consistent with the other models' formats.**"""
 TABLE_INTRODUCTION_V = """**V-CLS**: Video Classification, **V-QA**: (Video) Visual Question Answering, **V-RET**: Video Retrieval, **V-MRET**: Video Moment Retrieval \n
                         Models are ranked based on **Video-Overall**"""
+TABLE_INTRODUCTION_A = """**A-CLS**: Audio Classification, **A-RET**: Audio Retrieval \n
+                        Models are ranked based on **Audio-Overall**"""
 TABLE_INTRODUCTION_D = """⚠️ Please re-evaluate your models if you see a 0 on ViDoSeek-page-fixed or MMLongBench-page-fixed datasets. \n
 **VisDoc**: Visual Document Understanding \n
                         Models are ranked based on **Visdoc-Overall**"""
+TABLE_INTRODUCTION_AG = """**Tool**: Tool Retrieval, **GUI**: GUI Control, **Memory**: Agent Memory Retrieval \n
+                        Models are ranked based on **Agent-Overall**"""
 
 LEADERBOARD_INFO = """
 ## Dataset Summary
 """
 
-CITATION_BUTTON_TEXT = r"""@misc{meng2025vlm2vecv2advancingmultimodalembedding,
+CITATION_BUTTON_TEXT_V2 = r"""@misc{meng2025vlm2vecv2advancingmultimodalembedding,
       title={VLM2Vec-V2: Advancing Multimodal Embedding for Videos, Images, and Visual Documents}, 
       author={Rui Meng and Ziyan Jiang and Ye Liu and Mingyi Su and Xinyi Yang and Yuepeng Fu and Can Qin and Zeyuan Chen and Ran Xu and Caiming Xiong and Yingbo Zhou and Wenhu Chen and Semih Yavuz},
       year={2025},
@@ -56,6 +79,15 @@ CITATION_BUTTON_TEXT = r"""@misc{meng2025vlm2vecv2advancingmultimodalembedding,
       primaryClass={cs.CV},
       url={https://arxiv.org/abs/2507.04590}, 
 }"""
+CITATION_BUTTON_TEXT_V3 = r"""@misc{huang2026mmebv3measuringperformancegaps,
+      title={MMEB-V3: Measuring the Performance Gaps of Omni-Modality Embedding Models}, 
+      author={Haohang Huang and Xuan Lu and Mingyi Su and Xuan Zhang and Ziyan Jiang and Ping Nie and Kai Zou and Tomas Pfister and Wenhu Chen and Wei Zhang and Xiaoyu Shen and Rui Meng},
+      year={2026},
+      eprint={2604.23321},
+      archivePrefix={arXiv},
+      primaryClass={cs.IR},
+      url={https://arxiv.org/abs/2604.23321}, 
+}"""
 
 def load_single_json(file_path):
     with open(file_path, 'r') as file:
@@ -71,8 +103,12 @@ def load_data(base_dir=SCORE_BASE_DIR):
             all_data.append(data)
     return all_data
 
-def load_scores(raw_scores=None):
+def load_scores(raw_scores={}):
     """This function loads the raw scores from the user provided scores summary and flattens them into a single dictionary."""
+    # temp fix, will figure out later ===========
+    if any(_ in raw_scores for _ in ['tool', 'gui', 'memory']):
+        raw_scores['agent'] = raw_scores.pop('tool', {}) | raw_scores.pop('gui', {}) | raw_scores.pop('memory', {})
+    # ===========================================
     all_scores = {}
     for modality, datasets_list in DATASETS.items(): # Ex.: ('image', {'I-CLS': [...], 'I-QA': [...]})
         for sub_task, datasets in datasets_list.items(): # Ex.: ('I-CLS', ['VOC2007', 'N24News', ...])
@@ -100,6 +136,8 @@ def calculate_score(raw_scores=None):
 
     # Calculate overall score for all datasets
     avg_scores['Overall'] = get_avg(sum(all_scores.values()), len(ALL_DATASETS))
+    v2_scores = {k:v for k,v in all_scores.items() if k in ALL_DATASETS_SPLITS['image'] or k in ALL_DATASETS_SPLITS['video'] or k in ALL_DATASETS_SPLITS['visdoc']}
+    avg_scores['Overall-V2'] = get_avg(sum(v2_scores.values()), len(v2_scores))
 
     # Calculate scores for each modality
     for modality in MODALITIES:
@@ -134,17 +172,22 @@ def generate_model_row(data):
 def print_time(time: str|None):
     try:
         dt = datetime.strptime(time, "%Y-%m-%dT%H:%M:%S.%f")
-        return dt.strftime("%Y-%m-%d")
+        return dt.strftime("%y-%m-%d")
     except (ValueError, TypeError):
         return 'unknown'
 
+medal_map = {
+    "1": "🏆",
+    "2": "🥈",
+    "3": "🥉"
+}
 def rank_models(df, column='Overall', rank_name='Rank'):
     """Ranks the models based on the specific score."""
     df = df.sort_values(by=column, ascending=False).reset_index(drop=True)
-    df[rank_name] = range(1, len(df) + 1)
+    df[rank_name] = df[column].rank(method='min', ascending=False).astype(int).astype(str).map(lambda x: medal_map.get(x, x))
     return df
 
-def get_df():
+def get_df(rank_column='Overall'):
     """Generates a DataFrame from the loaded data."""
     all_data = load_data()
     rows = [generate_model_row(data) for data in all_data]
@@ -152,7 +195,7 @@ def get_df():
     df['Model Size(B)'] = df['Model Size(B)'].apply(process_model_size)
     df['Date'] = df['Date'].apply(print_time)
     df = create_hyperlinked_names(df)
-    df = rank_models(df, column='Overall')
+    df = rank_models(df, column=rank_column)
     return df
 
 def refresh_data():