utils.py

import pandas as pd
import os
import pprint as pp
# import requests

# ==========================
# Define some column names
MODEL_SIZE_COL_NAME = 'Size(B)'
# =========================

from datasets import DATASETS

HF_TOKEN = os.environ.get("HF_TOKEN")

BASE_COLS = ["Rank", "Models", MODEL_SIZE_COL_NAME, "Data Source"]
TASKS_V1 = ["V1-Overall", "I-CLS", "I-QA", "I-RET", "I-VG"]
COLUMN_NAMES = BASE_COLS + TASKS_V1

DATA_TITLE_TYPE = ['number', 'markdown', 'str', 'markdown'] + \
                    ['number'] * len(TASKS_V1)

LEADERBOARD_INTRODUCTION = """
# 📊 **MMEB LEADERBOARD (VLM2Vec)**

## Introduction
We introduce **Massive Multimodal Embedding Benchmark (MMEB)**, a novel comprehensive benchmark for evaluating omni-modality embedding models across text, image, video, audio, visual document, and agent-centric retrieval scenarios.  

**MMEB-V1** includes 36 datasets spanning four image-text meta-task categories: classification, visual question answering, retrieval, visual grounding.

**MMEB-V2** expands the evaluation scope to include five new tasks: 
- four video-based tasks: Video Retrieval, Moment Retrieval, Video Classification, and Video Question Answering 
- one visual documents task: Visual Document Retrieval. 

**MMEB-V3** further extends to a fuller modality setting by adding three major new evaluation categories:
- Audio Tasks: audio classification, cross-modal audio retrieval, and audio temporal grounding.
- Text Retrieval: instruction-following retrieval, reasoning retrieval, long-context retrieval, multi-condition retrieval, and general text retrieval.
- Agent Tasks: tool retrieval, GUI control, and agent memory retrieval.

<div style="display:inline-flex; flex-wrap:wrap; gap:6px; align-items:center; margin:8px 0;">
<a target="_blank" href="https://tiger-ai-lab.github.io/VLM2Vec/">
<img style="height:17pt" src="https://img.shields.io/badge/-🌐%20Website-red?style=flat"></a>
<a target="_blank" href="https://github.com/TIGER-AI-Lab/VLM2Vec">
<img style="height:17pt" src="https://img.shields.io/badge/-MMEB--V3%20Code-green?style=flat&logo=github"></a>
<a target="_blank" href="https://huggingface.co/datasets/VLM2Vec/MMEB-V3">
<img style="height:17pt" src="https://img.shields.io/badge/-🤗%20Dataset(MMEB--V3)-red?style=flat"></a>
<a target="_blank" href="https://arxiv.org/abs/2604.23321">
<img style="height:17pt" src="https://img.shields.io/badge/-V3 Paper-black?style=flat&logo=arxiv"></a>
<a target="_blank" href="https://arxiv.org/abs/2507.04590">
<img style="height:17pt" src="https://img.shields.io/badge/-V2 Paper-black?style=flat&logo=arxiv"></a>
<a target="_blank" href="https://arxiv.org/abs/2410.05160">
<img style="height:17pt" src="https://img.shields.io/badge/-V1 Paper-black?style=flat&logo=arxiv"></a>
<a target="_blank" href="https://huggingface.co/datasets/TIGER-Lab/MMEB-V2">
<img style="height:17pt" src="https://img.shields.io/badge/-🤗%20Dataset(V2)-red?style=flat"></a>
<a target="_blank" href="https://huggingface.co/datasets/TIGER-Lab/MMEB-eval">
<img style="height:17pt" src="https://img.shields.io/badge/-🤗%20Dataset(V1)-red?style=flat"></a>
<a target="_blank" href="https://huggingface.co/VLM2Vec">
<img style="height:17pt" src="https://img.shields.io/badge/-🤗%20Models-red?style=flat"></a>
<a target="_blank" href="https://huggingface.co/spaces/TIGER-Lab/MMEB-Leaderboard">
<img style="height:17pt" src="https://img.shields.io/badge/-🤗%20Leaderboard-red?style=flat"></a>
</div>
"""

ANNOUNCEMENT = """
<ul>
<li>[2026-07] MMEB-V3 Leaderboard is now live!</li>
<li>[2026-07] MMEB-V3 is accepted to COLM 2026!</li>
<li>[2026-04] MMEB-V3 released!</li>
<li>[2026-01] VLM2Vec/MMEB-V2 is accepted to TMLR 2026!</li>
<li>[2025-06] VLM2Vec/MMEB-V2 released!</li>
</ul>"""

LEADERBOARD_INFO = f"""
## Dataset Overview
This is the dictionary of all datasets used in our code. Please make sure all datasets' scores are included in your submission. \n
```python
{pp.pformat(DATASETS)}
```
"""

CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
CITATION_BUTTON_TEXT = r"""@article{jiang2024vlm2vec,
  title={VLM2Vec: Training Vision-Language Models for Massive Multimodal Embedding Tasks},
  author={Jiang, Ziyan and Meng, Rui and Yang, Xinyi and Yavuz, Semih and Zhou, Yingbo and Chen, Wenhu},
  journal={arXiv preprint arXiv:2410.05160},
  year={2024}
}"""

SUBMIT_INTRODUCTION = """# Submit on MMEB Leaderboard Introduction \n
## Please follow the guidelines in order to submit successfully. \n
1. **Step  1️⃣:** Please refer to the [**GitHub page**](https://github.com/TIGER-AI-Lab/VLM2Vec) for detailed instructions about evaluating your model. \n
    - If you want to submit to a specific modality leaderboard, such as MMEB Image, only run your model on the corresponding datasets and ignore the remaining. 
    - However, your model will still be shown on all leaderboards and might have a lower rank since missing datasets will be assigned a 0. \n
2. **Step  2️⃣:** After running the evaluation pipelines, please use the provided script **(e.g., [report_score_v3.py](https://github.com/TIGER-AI-Lab/VLM2Vec/blob/main/experiments/report_score_v3.py))** to generate the final score sheet.
    - Adjust your model's configurations in the script before running it
        - Note the "model size" field is digits-only and is in Billions (B) (ex., "8" for 8 billions, "0.5" for 500 millions).
        - If possible, please also add a contact method in case we want to reach you in the future.
3. **Step  3️⃣:** Finally, create a pull request and upload the generated JSON file to the ***scores*** folder. 
    - If directly using web UI: Go to the [scores folder](https://huggingface.co/spaces/TIGER-Lab/MMEB-Leaderboard/upload/main/scores), select "Upload file" and upload your JSON files.
    - If by git command line: refer to the [PR documentation](https://huggingface.co/docs/hub/repositories-pull-requests-discussions#pull-requests-advanced-usage).
    - Submit the PR and leave comments if any. We will then review and update the leaderboard accordingly.\n
    - To delete or modify your submission, submit a new PR with the updated file.\n\n

## 🐞 Bug reporting and feedback
If you encounter any issues or have feedback for improvement regarding the leaderboard, please report them in [Discussion](https://huggingface.co/spaces/TIGER-Lab/MMEB-Leaderboard/discussions).\n
If you cannot reach us via above methods, email us at **m7su@uwaterloo.ca**. 

## Appendix 1: Example valid score sheet format ⬇️: \n
```json
{
    "metadata": {
        "model_name": "<Model Name>",
        "url": "<Model URL>" or null,
        "model_size": <Model Size> or null,
        "contact": xxx@xxxxx.com
        ... ...
    },
    "metrics": {
        "image": {
            "ImageNet-1K": {
                "hit@1": 0.5,
                "ndcg@1": 0.5,
                ... ...
            }, 
            "N24News": {
                ... ...
            }, 
            ... ...
        }, 
        "video": {
            ... ...
        },
        ... ...
    }
}
```
"""

def create_hyperlinked_names(df):
    def convert_url(url, model_name):
        return f'<a href="{url}">{model_name}</a>' if url else model_name

    def add_link_to_model_name(row):
        row['Models'] = convert_url(row['URL'], row['Models'])
        return row
    
    df = df.copy()
    df = df.apply(add_link_to_model_name, axis=1)
    return df

# def fetch_data(file: str) -> pd.DataFrame:
#     # fetch the leaderboard data from remote
#     if file is None:
#         raise ValueError("URL Not Provided")
#     url = f"https://huggingface.co/spaces/TIGER-Lab/MMEB/resolve/main/{file}"
#     print(f"Fetching data from {url}")
#     response = requests.get(url)
#     if response.status_code != 200:
#         raise requests.HTTPError(f"Failed to fetch data: HTTP status code {response.status_code}")
#     return pd.read_json(io.StringIO(response.text), orient='records', lines=True)

def get_df(file="results.jsonl"):
    df = pd.read_json(file, orient='records', lines=True)
    df[MODEL_SIZE_COL_NAME] = df['Model Size(B)'].apply(process_model_size)
    for task in TASKS_V1:
        if df[task].isnull().any():
            df[task] = df[task].apply(lambda score: '-' if pd.isna(score) else score)
    df = df.sort_values(by=['V1-Overall'], ascending=False)
    df = create_hyperlinked_names(df)
    df['Rank'] = range(1, len(df) + 1)
    return df

def refresh_data():
    df = get_df()
    return df[COLUMN_NAMES]

def search_and_filter_models(df, query, min_size, max_size):
    filtered_df = df.copy()
    
    if query:
        filtered_df = filtered_df[filtered_df['Models'].str.contains(query, case=False, na=False)]

    size_mask = filtered_df[MODEL_SIZE_COL_NAME].apply(lambda x: 
        (min_size <= 1000.0 <= max_size) if x == 'unknown' 
        else (min_size <= x <= max_size))
    
    filtered_df = filtered_df[size_mask]
    
    return filtered_df[COLUMN_NAMES]


def search_models(df, query):
    if query:
        return df[df['Models'].str.contains(query, case=False, na=False)]
    return df

def get_size_range(df):
    sizes = df[MODEL_SIZE_COL_NAME].apply(lambda x: 0.0 if x == 'unknown' else x)
    if (sizes == 0.0).all():
        return 0.0, 1000.0
    return float(sizes.min()), float(sizes.max())


def process_model_size(size):
    if pd.isna(size) or size == 'unk':
        return 'unknown'
    try:
        val = float(size)
        return round(val, 3)
    except (ValueError, TypeError):
        return 'unknown'

def filter_columns_by_tasks(df, selected_tasks=None):
    if selected_tasks is None or len(selected_tasks) == 0:
        return df[COLUMN_NAMES]
    
    base_columns = ['Models', MODEL_SIZE_COL_NAME, 'Data Source', 'Overall']
    selected_columns = base_columns + selected_tasks
    
    available_columns = [col for col in selected_columns if col in df.columns]
    return df[available_columns]