minor update

app.py

@@ -23,26 +23,29 @@ with gr.Blocks() as block:
         gr.Markdown(ANNOUNCEMENT)
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         # Table 1, the main leaderboard of overall scores
-        with gr.TabItem("📊 MMEB (V3)", elem_id="tab-overall", id=1):
+        with gr.TabItem("📊 MMEB (V3)[𝓝𝓔𝒲]", elem_id="tab-overall", id=1):
             with gr.Row():
                 with gr.Accordion("Citation", open=False):
                     citation_button2 = gr.Textbox(
                         value=v3.CITATION_BUTTON_TEXT_V3,
                         label="Copy the following snippet to cite MMEB-V3",
-                        elem_id="citation-button",
+                        elem_id="citation-button3",
                         lines=10,
+                        show_copy_button=True
                     )
                     gr.Textbox(
                         value=v3.CITATION_BUTTON_TEXT_V2,
                         label="Copy the following snippet to cite VLM2Vec/MMEB-V2",
-                        elem_id="citation-button",
+                        elem_id="citation-button2",
                         lines=10,
+                        show_copy_button=True
                     )
                     gr.Textbox(
                         value=CITATION_BUTTON_TEXT,
                         label="Copy the following snippet to cite VLM2Vec/MMEB-V1",
-                        elem_id="citation-button",
-                        lines=10,
+                        elem_id="citation-button1",
+                        lines=6,
+                        show_copy_button=True
                     )
             gr.Markdown(v3.TABLE_INTRODUCTION)
 
@@ -73,12 +76,12 @@ with gr.Blocks() as block:
                     step=0.1,
                     label="Maximum number of parameters (B)",
                 )
-            df2_all = df2[v3.COLUMN_NAMES]
+            df2_all = df2[v3.COLUMN_NAMES_V3]
             data_component2 = gr.components.Dataframe(
                 value=df2_all,
-                headers=v3.COLUMN_NAMES,
+                headers=v3.COLUMN_NAMES_V3,
                 type="pandas",
-                datatype=v3.DATA_TITLE_TYPE,
+                datatype=v3.DATA_TITLE_TYPE_V3,
                 interactive=False,
                 visible=True,
                 max_height=2400, 
@@ -111,8 +114,21 @@ with gr.Blocks() as block:
             )
             refresh_button2.click(fn=v3.refresh_data, outputs=data_component2)
 
+        # v2 overall table
+        with gr.TabItem("📊 MMEB (V2)", elem_id="tab-overall-v2", id=10):
+            gr.Markdown("Models are ranked based on **Overall-V2**.")
+            overall_df_v2 = gr.components.Dataframe(
+                value=v3.rank_models(df2[v3.COLUMN_NAMES_V2], 'Overall-V2'),
+                headers=v3.COLUMN_NAMES_V2,
+                type="pandas",
+                datatype=v3.DATA_TITLE_TYPE_V2,
+                interactive=False,
+                visible=True,
+                max_height=2400, 
+            )
+
         # table 2, text scores
-        with gr.TabItem("📝 Text [NEW]", elem_id="tab-text", id=2):
+        with gr.TabItem("📝 Text [𝓝𝓔𝒲]", elem_id="tab-text", id=2):
             data_component_t = gr.components.Dataframe(
                 value=v3.rank_models(df2[v3.COLUMN_NAMES_T], 'Text-Overall'),
                 headers=v3.COLUMN_NAMES_T,
@@ -171,7 +187,7 @@ with gr.Blocks() as block:
             download_v_but_json = gr.DownloadButton("Download Video Ranking (JSON)", value=v3.download_ranking(df2_v, 'video_ranking', format='json'))
 
         # table 5, audio scores
-        with gr.TabItem("🎵 Audio [NEW]", elem_id="tab-audio", id=5):
+        with gr.TabItem("🎵 Audio [𝓝𝓔𝒲]", elem_id="tab-audio", id=5):
             gr.Markdown(v3.TABLE_INTRODUCTION_A)
             data_component_a = gr.components.Dataframe(
                 value=v3.rank_models(df2[v3.COLUMN_NAMES_A], 'Audio-Overall'),
@@ -206,7 +222,7 @@ with gr.Blocks() as block:
             download_vd_but_json = gr.DownloadButton("Download Visual Document Ranking (JSON)", value=v3.download_ranking(df2_d, 'visdoc_ranking', format='json'))
 
         # table 7, agent scores
-        with gr.TabItem("🤖 Agents [NEW]", elem_id="tab-agents", id=7):
+        with gr.TabItem("🤖 Agents [𝓝𝓔𝒲]", elem_id="tab-agents", id=7):
             gr.Markdown(v3.TABLE_INTRODUCTION_AG)
             data_component_ag = gr.components.Dataframe(
                 value=v3.rank_models(df2[v3.COLUMN_NAMES_AG], 'Agent-Overall'),

scores/LCO-Embedding-Omni-7B.json

@@ -3,8 +3,8 @@
         "model_name": "LCO-Embedding-Omni-7B",
         "model_size": 7,
         "embedding_dimension": 0,
-        "url": "",
-        "data_source": "Reproduced",
+        "url": "https://huggingface.co/LCO-Embedding/LCO-Embedding-Omni-7B",
+        "data_source": "Reproduced by TIGER-Lab",
         "report_generated_date": "2026-07-23T16:25:55.268061"
     },
     "metrics": {

scores/e5-omni-3B.json

@@ -3,8 +3,8 @@
         "model_name": "e5-omni-3B",
         "model_size": 3,
         "embedding_dimension": 0,
-        "url": "",
-        "data_source": "Reproduced",
+        "url": "https://huggingface.co/Haon-Chen/e5-omni-3B",
+        "data_source": "Reproduced by TIGER-Lab",
         "report_generated_date": "2026-07-23T17:40:39.795658"
     },
     "metrics": {

scores/e5-omni-7B.json

@@ -3,8 +3,8 @@
         "model_name": "e5-omni-7B",
         "model_size": 7,
         "embedding_dimension": 0,
-        "url": "",
-        "data_source": "Reproduced",
+        "url": "https://huggingface.co/Haon-Chen/e5-omni-7B",
+        "data_source": "Reproduced by TIGER-Lab",
         "report_generated_date": "2026-07-20T13:57:44.725269"
     },
     "metrics": {

scores/omni-embed-nemotron-3b.json

@@ -3,8 +3,8 @@
         "model_name": "omni-embed-nemotron-3b",
         "model_size": 3,
         "embedding_dimension": 0,
-        "url": "",
-        "data_source": "Reproduced",
+        "url": "https://huggingface.co/nvidia/omni-embed-nemotron-3b",
+        "data_source": "Reproduced by TIGER-Lab",
         "report_generated_date": "2026-07-20T13:57:43.169818"
     },
     "metrics": {

utils.py

@@ -3,11 +3,16 @@ import os
 import pprint as pp
 # import requests
 
+# ==========================
+# Define some column names
+MODEL_SIZE_COL_NAME = 'Size(B)'
+# =========================
+
 from datasets import DATASETS
 
 HF_TOKEN = os.environ.get("HF_TOKEN")
 
-BASE_COLS = ["Rank", "Models", "Model Size(B)", "Data Source"]
+BASE_COLS = ["Rank", "Models", MODEL_SIZE_COL_NAME, "Data Source"]
 TASKS_V1 = ["V1-Overall", "I-CLS", "I-QA", "I-RET", "I-VG"]
 COLUMN_NAMES = BASE_COLS + TASKS_V1
 
@@ -24,7 +29,7 @@ We introduce **Massive Multimodal Embedding Benchmark (MMEB)**, a novel comprehe
 
 **MMEB-V2** expands the evaluation scope to include five new tasks: 
 - four video-based tasks: Video Retrieval, Moment Retrieval, Video Classification, and Video Question Answering 
-- one task focused on visual documents: Visual Document Retrieval. 
+- one visual documents task: Visual Document Retrieval. 
 
 **MMEB-V3** further extends to a fuller modality setting by adding three major new evaluation categories:
 - Audio Tasks: audio classification, cross-modal audio retrieval, and audio temporal grounding.
@@ -83,23 +88,23 @@ CITATION_BUTTON_TEXT = r"""@article{jiang2024vlm2vec,
 SUBMIT_INTRODUCTION = """# Submit on MMEB Leaderboard Introduction \n
 ## Please follow the guidelines in order to submit successfully. \n
 1. **Step  1️⃣:** Please refer to the [**GitHub page**](https://github.com/TIGER-AI-Lab/VLM2Vec) for detailed instructions about evaluating your model. \n
-2. **Step  2️⃣:** After running the evaluation pipelines, please use the provided script **(e.g., [report_score_v2.py](https://github.com/TIGER-AI-Lab/VLM2Vec/blob/main/experiments/report_score_v2.py))** to generate the final score sheet. (Use [report_score_v3.py](https://github.com/TIGER-AI-Lab/VLM2Vec/blob/main/experiments/report_score_v3.py) for v3 submission).
+    - If you want to submit to a specific modality leaderboard, such as MMEB Image, only run your model on the corresponding datasets and ignore the remaining. 
+    - However, your model will still be shown on all leaderboards and might have a lower rank since missing datasets will be assigned a 0. \n
+2. **Step  2️⃣:** After running the evaluation pipelines, please use the provided script **(e.g., [report_score_v3.py](https://github.com/TIGER-AI-Lab/VLM2Vec/blob/main/experiments/report_score_v3.py))** to generate the final score sheet.
     - Adjust your model's configurations in the script before running it
-        - Note the "model size" field is digits-only and is in Billions (B), so please convert it if yours is in different units/formats (e.x., "8" for 8 billion, "0.5" for 500 million).
-        - If possible, please also add a contact method in case we want to reach you in the future
+        - Note the "model size" field is digits-only and is in Billions (B) (ex., "8" for 8 billions, "0.5" for 500 millions).
+        - If possible, please also add a contact method in case we want to reach you in the future.
 3. **Step  3️⃣:** Finally, create a pull request and upload the generated JSON file to the ***scores*** folder. 
-    - If directly using web UI:
-        - Go to the [scores folder](https://huggingface.co/spaces/TIGER-Lab/MMEB-Leaderboard/upload/main/scores)
-        - Select "Upload file" and upload your JSON files.
+    - If directly using web UI: Go to the [scores folder](https://huggingface.co/spaces/TIGER-Lab/MMEB-Leaderboard/upload/main/scores), select "Upload file" and upload your JSON files.
     - If by git command line: refer to the [PR documentation](https://huggingface.co/docs/hub/repositories-pull-requests-discussions#pull-requests-advanced-usage).
-    - Submit the PR and leave any comments if any. We will then review and update the leaderboard accordingly.\n
+    - Submit the PR and leave comments if any. We will then review and update the leaderboard accordingly.\n
     - To delete or modify your submission, submit a new PR with the updated file.\n\n
 
 ## 🐞 Bug reporting and feedback
-If you encounter any issues or have improvement feedback regarding the leaderboard, please report them in Discussion.\n
+If you encounter any issues or have feedback for improvement regarding the leaderboard, please report them in [Discussion](https://huggingface.co/spaces/TIGER-Lab/MMEB-Leaderboard/discussions).\n
 If you cannot reach us via above methods, email us at **m7su@uwaterloo.ca**. 
 
-## Appendix 1: Example: valid score sheet format ⬇️: \n
+## Appendix 1: Example valid score sheet format ⬇️: \n
 ```json
 {
     "metadata": {
@@ -128,10 +133,6 @@ If you cannot reach us via above methods, email us at **m7su@uwaterloo.ca**.
     }
 }
 ```
-## ⚠️ Special Instructions for submitting to MMEB Image (Previously MMEB-V1) Leaderboard
-We understand that some researchers want to exclusively submit to the Image leaderboard, but unfortunately our current leaderboard cannot exclude your model from other modalities' leaderboards. 
-To do so, run the 36 image datasets only and simply ignore other datasets.
-The leaderboard will automatically assign a 0 to the missing datasets and your model will be shown on all leaderboards, and might have a lower rank. \n
 """
 
 def create_hyperlinked_names(df):
@@ -159,7 +160,7 @@ def create_hyperlinked_names(df):
 
 def get_df(file="results.jsonl"):
     df = pd.read_json(file, orient='records', lines=True)
-    df['Model Size(B)'] = df['Model Size(B)'].apply(process_model_size)
+    df[MODEL_SIZE_COL_NAME] = df['Model Size(B)'].apply(process_model_size)
     for task in TASKS_V1:
         if df[task].isnull().any():
             df[task] = df[task].apply(lambda score: '-' if pd.isna(score) else score)
@@ -178,7 +179,7 @@ def search_and_filter_models(df, query, min_size, max_size):
     if query:
         filtered_df = filtered_df[filtered_df['Models'].str.contains(query, case=False, na=False)]
 
-    size_mask = filtered_df['Model Size(B)'].apply(lambda x: 
+    size_mask = filtered_df[MODEL_SIZE_COL_NAME].apply(lambda x: 
         (min_size <= 1000.0 <= max_size) if x == 'unknown' 
         else (min_size <= x <= max_size))
     
@@ -193,7 +194,7 @@ def search_models(df, query):
     return df
 
 def get_size_range(df):
-    sizes = df['Model Size(B)'].apply(lambda x: 0.0 if x == 'unknown' else x)
+    sizes = df[MODEL_SIZE_COL_NAME].apply(lambda x: 0.0 if x == 'unknown' else x)
     if (sizes == 0.0).all():
         return 0.0, 1000.0
     return float(sizes.min()), float(sizes.max())
@@ -212,7 +213,7 @@ def filter_columns_by_tasks(df, selected_tasks=None):
     if selected_tasks is None or len(selected_tasks) == 0:
         return df[COLUMN_NAMES]
     
-    base_columns = ['Models', 'Model Size(B)', 'Data Source', 'Overall']
+    base_columns = ['Models', MODEL_SIZE_COL_NAME, 'Data Source', 'Overall']
     selected_columns = base_columns + selected_tasks
     
     available_columns = [col for col in selected_columns if col in df.columns]

utils_v3.py

@@ -2,16 +2,21 @@ import json
 import os
 import pandas as pd
 from datetime import datetime
-from utils import create_hyperlinked_names, process_model_size
+from utils import create_hyperlinked_names, process_model_size, MODEL_SIZE_COL_NAME
 from datasets import *
 
-BASE_COLS = ['Rank', 'Models', 'Model Size(B)', 'Date']
+BASE_COLS = ['Rank', 'Models', MODEL_SIZE_COL_NAME, 'Date']
 BASE_DATA_TITLE_TYPE = ['str', 'markdown', 'str', 'str']
 
-OVERALL_COLS = ["Overall", "Overall-V2", "Text-Overall", 'Image-Overall', 'Video-Overall', 'Audio-Overall', 'Visdoc-Overall', "Agent-Overall"]
-COLUMN_NAMES = BASE_COLS + OVERALL_COLS
-DATA_TITLE_TYPE = BASE_DATA_TITLE_TYPE + \
-                    ['number'] * len(OVERALL_COLS)
+OVERALL_COLS_V2 = ["Overall-V2", 'Image-Overall', 'Video-Overall', 'Visdoc-Overall']
+COLUMN_NAMES_V2 = BASE_COLS + OVERALL_COLS_V2
+DATA_TITLE_TYPE_V2 = BASE_DATA_TITLE_TYPE + \
+                    ['number'] * len(OVERALL_COLS_V2)
+
+OVERALL_COLS_V3 = ["Overall", "Overall-V3🆕", "Text-Overall", "Audio-Overall", "Agent-Overall"]
+COLUMN_NAMES_V3 = BASE_COLS + OVERALL_COLS_V3
+DATA_TITLE_TYPE_V3 = BASE_DATA_TITLE_TYPE + \
+                    ['number'] * len(OVERALL_COLS_V3)
 
 SUB_TASKS_T = ["FollowIR", "R2MED", "InfoSearch", "BRIGHT", "LongEmbed", "MultiConIR", "NanoBEIR"]
 TASKS_T = ['Text-Overall'] + SUB_TASKS_T + ALL_DATASETS_SPLITS['text']
@@ -50,7 +55,7 @@ DATA_TITLE_TYPE_AG = BASE_DATA_TITLE_TYPE + \
                     ['number'] * len(TASKS_AG)
 
 TABLE_INTRODUCTION = """**MMEB**: Massive MultiModal Embedding Benchmark \n
-                        Models are ranked based on **Overall**(V3)."""
+                        Models are ranked based on **Overall**(V3-ALL). **Overall-V3🆕**: Newly added datasets in V3."""
 TABLE_INTRODUCTION_I = """**I-CLS**: Image Classification, **I-QA**: (Image) Visual Question Answering, **I-RET**: Image Retrieval, **I-VG**: (Image) Visual Grounding \n
                         Models are ranked based on **Image-Overall**\n
                         **Models from the old V1 leaderboard are missing detailed scores of each dataset. 
@@ -138,6 +143,8 @@ def calculate_score(raw_scores=None):
     avg_scores['Overall'] = get_avg(sum(all_scores.values()), len(ALL_DATASETS))
     v2_scores = {k:v for k,v in all_scores.items() if k in ALL_DATASETS_SPLITS['image'] or k in ALL_DATASETS_SPLITS['video'] or k in ALL_DATASETS_SPLITS['visdoc']}
     avg_scores['Overall-V2'] = get_avg(sum(v2_scores.values()), len(v2_scores))
+    v3_newonly_scores = {k:v for k,v in all_scores.items() if k in ALL_DATASETS_SPLITS['text'] or k in ALL_DATASETS_SPLITS['audio'] or k in ALL_DATASETS_SPLITS['agent']}
+    avg_scores['Overall-V3🆕'] = get_avg(sum(v3_newonly_scores.values()), len(v3_newonly_scores))
 
     # Calculate scores for each modality
     for modality in MODALITIES:
@@ -160,9 +167,9 @@ def generate_model_row(data):
     metadata = data['metadata']
     row = {
         'Models': metadata.get('model_name', None), 
-        'Model Size(B)': metadata.get('model_size', None),
+        MODEL_SIZE_COL_NAME: metadata.get('model_size', None),
         'URL': metadata.get('url', None), 
-        'Data Source': metadata.get('data_source', 'Self-Reported'),
+        'Submitted by': metadata.get('data_source', 'Self-Reported'),
         'Date': metadata.get('report_generated_date', None)
     }
     scores = calculate_score(data['metrics'])
@@ -192,7 +199,7 @@ def get_df(rank_column='Overall'):
     all_data = load_data()
     rows = [generate_model_row(data) for data in all_data]
     df = pd.DataFrame(rows)
-    df['Model Size(B)'] = df['Model Size(B)'].apply(process_model_size)
+    df[MODEL_SIZE_COL_NAME] = df[MODEL_SIZE_COL_NAME].apply(process_model_size)
     df['Date'] = df['Date'].apply(print_time)
     df = create_hyperlinked_names(df)
     df = rank_models(df, column=rank_column)
@@ -208,7 +215,7 @@ def search_and_filter_models(df, query, min_size, max_size):
     if query:
         filtered_df = filtered_df[filtered_df['Models'].str.contains(query, case=False, na=False)]
 
-    size_mask = filtered_df['Model Size(B)'].apply(lambda x: 
+    size_mask = filtered_df[MODEL_SIZE_COL_NAME].apply(lambda x: 
         (min_size <= 1000.0 <= max_size) if x == 'unknown' 
         else (min_size <= x <= max_size))